diff --git a/.github/workflows/zig-tests.yml b/.github/workflows/zig-tests.yml index ad8c1dc862..7f65b3cafc 100644 --- a/.github/workflows/zig-tests.yml +++ b/.github/workflows/zig-tests.yml @@ -412,9 +412,10 @@ jobs: run: python3 scripts/packaging/test_reproducible_tar.py # ARC may expose the shared node's memory total when the runner cgroup is - # unlimited. Cap the aggregate scheduler budget at 20 GiB so concurrent + # unlimited. Cap the aggregate scheduler budget at 22 GiB so concurrent # test runtimes cannot overcommit the runner while still admitting the - # largest individual compile step, which reserves 20 GiB. The patched + # largest individual compile step, which reserves 22 GiB. Cgroup detection + # still reserves 20% headroom and never raises a smaller host's budget. The patched # 0.16 runner correctly reserves each ready step's max_rss claim as it # wakes it. - name: Configure bounded parallel unit tests @@ -424,7 +425,7 @@ jobs: jobs=$(nproc) if [ "$jobs" -gt 8 ]; then jobs=8; fi echo "list=0-$((jobs - 1))" >> "$GITHUB_OUTPUT" - echo "max_rss=$(python3 zig/tools/run_bounded_zig_build.py --print-max-rss --max-rss-cap 21474836480)" >> "$GITHUB_OUTPUT" + echo "max_rss=$(python3 zig/tools/run_bounded_zig_build.py --print-max-rss --max-rss-cap 23622320128)" >> "$GITHUB_OUTPUT" zig_lib_dir="$(zig env | sed -n 's/^[[:space:]]*\.lib_dir = "\(.*\)",$/\1/p')" patched_runner="${RUNNER_TEMP}/zig-build-runner-maxrss.zig" python3 zig/tools/patch_zig_0_16_build_runner_maxrss.py \ @@ -674,14 +675,14 @@ jobs: python3 -m unittest discover -s scripts/bench/pdf -p 'test_*.py' # Keep normal build-step parallelism while bounding aggregate compiler - # memory to the pod's cgroup budget. + # memory to the pod's cgroup budget and the 22 GiB largest-step claim. - name: Configure bounded parallel heavy tests id: cpus run: | jobs=$(nproc) if [ "$jobs" -gt 8 ]; then jobs=8; fi echo "list=0-$((jobs - 1))" >> "$GITHUB_OUTPUT" - echo "max_rss=$(python3 zig/tools/run_bounded_zig_build.py --print-max-rss --max-rss-cap 21474836480)" >> "$GITHUB_OUTPUT" + echo "max_rss=$(python3 zig/tools/run_bounded_zig_build.py --print-max-rss --max-rss-cap 23622320128)" >> "$GITHUB_OUTPUT" zig_lib_dir="$(zig env | sed -n 's/^[[:space:]]*\.lib_dir = "\(.*\)",$/\1/p')" patched_runner="${RUNNER_TEMP}/zig-build-runner-maxrss.zig" python3 zig/tools/patch_zig_0_16_build_runner_maxrss.py \ diff --git a/PAGERANK.md b/PAGERANK.md new file mode 100644 index 0000000000..13ada1803d --- /dev/null +++ b/PAGERANK.md @@ -0,0 +1,5820 @@ +# PageRank Graph Metric Design + +## Goal + +Add PageRank-style graph centrality as a materialized graph index metric. The +metric should be eventually fresh, observable, safe to query during rebuilds, +and cheap on the write path. + +This should be implemented as a generic graph metric framework with PageRank as +the first supported metric. Eigenvector centrality, HITS authorities, and HITS +hubs can reuse the same storage, job, and query surfaces later. + +## User Model + +Service-targeted maintenance roles (including `supervise` and `launch`) require +`ANTFLY_INTERNAL_SERVICE_SECRET` (at least 32 bytes) and +`ANTFLY_INTERNAL_SERVICE_ISSUER`, matching the owning service's +`antfly.internal_service.secret` and `antfly.internal_service.issuer` credentials. +Supply these through the deployment's secret environment, never process arguments. +Roles validate credentials before requesting work; supervisors and launchers +validate them before spawning children. Children inherit the environment, and +the shared HTTP client signs only internal API requests with short-lived tokens. +The process harness exercises enforced authentication without an unsigned bypass. + +Users opt in through graph index configuration. A graph metric is owned by the +graph index because it is derived index state: the graph index tracks dirtiness, +runs maintenance, stores scores, and exposes query access. Table schema may +validate or surface the config, but it should not own the materialization. + +Writes to graph edges mark the metric stale. Background work computes a new +score generation. Queries read the last completely published generation by +default. + +The core promise is: + +- Writes do not wait for PageRank recomputation. +- Queries do not read partial scores. +- Failed rebuilds leave the prior published generation usable. +- Freshness and progress are visible through status APIs. + +## Index Configuration + +Graph metrics should live under graph index configuration: + +```json +{ + "name": "knowledge_graph", + "type": "graph", + "metrics": { + "pagerank": { + "enabled": true, + "damping": 0.85, + "max_iterations": 50, + "tolerance": 0.000001, + "refresh": "background", + "edge_filter": { + "mode": "all" + } + } + } +} +``` + +`max_iterations` must be between 1 and 1,000. The bound applies at both the +configuration parser and graph runtime boundary so malformed internal config +cannot create unbounded foreground or background work. HITS authority and hub +metrics form a pair only when their iteration limit, tolerance, refresh mode, +and edge filter match; ambiguous aliases and mixed manual/background pairs are +rejected during index validation. + +The internal representation should keep the surface generic: + +```zig +pub const GraphMetricKind = enum { + pagerank, + degree, + eigenvector, + hits_authority, + hits_hub, +}; + +pub const GraphMetricRefreshMode = enum { + background, + manual, +}; + +pub const GraphMetricConfig = struct { + name: []const u8, + kind: GraphMetricKind, + damping: f64 = 0.85, + tolerance: f64 = 0.000001, + max_iterations: u32 = 50, + refresh: GraphMetricRefreshMode = .background, + edge_filter: GraphMetricEdgeFilter = .{ .mode = .all }, +}; +``` + +PageRank should ship first. Additional metric kinds should stay opt-in and must +reuse the same graph-index-owned materialization, status, and query framework. + +The resolved config should always record the edge scope. The ergonomic default +is all edges in the graph index, but users need a first-class way to restrict +the metric to specific edge families because PageRank over mixed semantic edges +can be misleading: + +For v1, support `mode: "all"` and typed edge include lists: + +```json +{ + "edge_filter": { + "types": ["mentions", "cites"] + } +} +``` + +Typed include lists should be implemented only against the graph index's +existing edge type or edge family metadata. V1 should not add a separate +predicate engine for graph metric filtering. + +Long-term, edge scope can grow into labels and field predicates: + +```json +{ + "edge_filter": { + "types": ["mentions", "cites"], + "labels": ["references"], + "where": [ + { "field": "confidence", "op": ">=", "value": 0.8 } + ] + } +} +``` + +Arbitrary predicates should wait until the core materialization and generation +contract is stable. + +## SQL DDL + +If SQL DDL is exposed for graph metrics, it should lower into the same graph +index metric config: + +```sql +CREATE GRAPH METRIC pagerank +ON knowledge_graph +WITH ( + damping = 0.85, + tolerance = 0.000001, + max_iterations = 50, + refresh = 'background' +); +``` + +This is optional for the first implementation. The JSON/schema path is enough to +prove the storage and query contract. + +## Query API + +Graph traversal and graph search APIs should be able to return and order by a +published graph metric: + +```json +{ + "graph": { + "index": "knowledge_graph", + "start": { "label": "Person", "id": "alice" }, + "traverse": { "max_depth": 2 }, + "order_by": [ + { "metric": "pagerank", "direction": "desc" } + ], + "return": ["id", "label", "pagerank"] + } +} +``` + +Direct top-k metric reads should also be supported: + +```json +{ + "graph_metric": { + "index": "knowledge_graph", + "metric": "pagerank", + "top_k": 100 + } +} +``` + +Direct metric endpoints should always include metric status in their responses. +Graph traversal and graph search responses should include metric status only +when requested: + +```json +{ + "graph": { + "index": "knowledge_graph", + "traverse": { "max_depth": 2 }, + "return": ["id", "pagerank"], + "include_metric_status": true + } +} +``` + +Metric status is a map keyed by metric name. That keeps responses easy to join +against requested metric fields when a query includes multiple metrics. + +Response shape: + +```json +{ + "metric_status": { + "pagerank": { + "state": "stale", + "published_generation": 42, + "edge_generation": 43, + "converged": true, + "iterations_completed": 31, + "computed_at_ms": 1780000000000 + }, + "authority": { + "state": "stale", + "published_generation": 8, + "edge_generation": 11, + "converged": false, + "iterations_completed": 50, + "computed_at_ms": 1779999900000 + } + } +} +``` + +By default, queries read the last published complete generation. If a metric has +never been published, behavior depends on how the metric is used: + +- Projecting a metric field returns `null`. +- Ordering by a metric fails with `MetricNotReady`. +- Filtering by a metric fails with `MetricNotReady`. +- Direct top-k metric reads fail with `MetricNotReady`. + +Projection can tolerate missing derived data; ranking and filtering cannot +because they imply meaningful score semantics. + +Useful query freshness modes: + +```json +{ + "metric_freshness": "published" +} +``` + +Initial modes: + +- `published`: read the last complete generation, even if stale. +- `fresh`: require the published generation to match the current edge + generation; fail with `MetricNotReady` or `MetricStale` otherwise. + +Use two distinct freshness errors: + +- `MetricNotReady`: no published generation exists. +- `MetricStale`: a published generation exists, but the caller requested + `fresh` and it does not match the current edge generation. + +Avoid blocking query execution for a rebuild in the first implementation. + +## Status API + +Expose metric progress and freshness: + +```json +{ + "index": "knowledge_graph", + "metric": "pagerank", + "state": "stale", + "published_generation": 42, + "building_generation": 43, + "edge_generation": 43, + "progress": 0.61, + "iterations_completed": 18, + "last_error": null +} +``` + +Suggested states: + +- `disabled` +- `not_ready` +- `fresh` +- `stale` +- `building` +- `failed` + +The status API should distinguish "queryable but stale" from "not queryable". + +## Storage Layout + +Use generationed materialization so publishing is atomic: + +```text +graph_metric_dirty:: -> edge_generation +graph_metric_published:: -> score_generation +graph_metric_build::: -> build metadata +graph_metric_score:::: -> f64 +graph_metric_meta::: -> stats/convergence metadata +``` + +Queries only resolve scores through `graph_metric_published`. Build jobs write +to a private `building_generation` and flip the published pointer only after the +generation converges or reaches the configured iteration cap. + +Score generations are private storage epochs, not edge generations. A manual +refresh or config-only rebuild may target the same edge snapshot more than once, +so every attempt gets a new durable score namespace. Status and API responses +continue to expose the edge snapshot as `published_generation`; the private +score epoch is only used to locate immutable materialized rows. + +Publication enqueues the superseded score epoch for bounded background cleanup. +Cleanup uses durable phase/cursor state and small transactions, which avoids +unbounded write batches and remains safe for readers holding an older storage +snapshot. Failed builds enqueue only their unpublished output and never remove +the last good published generation. Operator deletion similarly tombstones the +metric immediately, then removes scores, ranks, metadata, and job state in +bounded maintenance pages. + +The durable cleanup state is internal: + +```text +graph_metric_retired:: -> score_generation +graph_metric_cleanup_phase:: -> scores | ranks | metadata +graph_metric_cleanup_cursor:: -> last_key +``` + +The queue is deliberately bounded. If maintenance cannot retire generations as +fast as new materializations are requested, control requests apply backpressure +instead of accumulating unbounded disk usage. + +Future versions can add retention controls for debugging or rollback: + +```json +{ + "retention": { + "mode": "count", + "retained_generations": 2 + } +} +``` + +Do not expose retention as a first-version user-facing option. Keep v1 +latest-only. If this becomes useful later, prefer a retention object with modes: + +- `latest`: keep only the latest published generation. +- `count`: keep the last N published generations. +- `duration`: keep generations for a time window. + +## Write Path + +Graph edge writes should only mark metric dirtiness: + +```text +edge write commits + -> edge_generation advances + -> graph_metric_dirty::pagerank = edge_generation + -> background job is scheduled best-effort +``` + +The write path must not compute PageRank, scan the graph, or wait for a metric +job. If scheduling fails, the dirty marker remains durable and a later +maintenance round can recover. + +## Local Job Execution + +For a single local graph index, follow the algebraic HLL maintenance pattern: + +- Dirty marker is persisted. +- A durable maintenance lane runs the rebuild off the write path. +- Redundant jobs collapse by re-checking dirty state under the index write lock. +- A failed maintenance attempt leaves the dirty marker intact. + +This is the simplest first implementation and keeps the first PageRank version +small. + +## Distributed Job Execution + +For distributed graph indexes, use the relational job pattern: + +- durable job id +- worker lease +- phase +- cursor/progress key +- requeue support +- progress endpoint +- idempotent page writes + +PageRank is iterative, so the distributed job is a multi-phase job rather than a +single range repair pass. + +Suggested phases: + +```text +prepare_generation +scan_edges_and_out_degree +initialize_ranks +iterate_contributions +reduce_ranks +check_convergence +publish_generation +cleanup_old_generations +``` + +The important distributed invariant is that partial contributions and partial +scores are never visible through the query path. Only the final publish step +changes the generation pointer that queries read. + +## PageRank Algorithm + +Initial algorithm: + +```text +rank_next(node) = + (1 - damping) / node_count + + damping * sum(rank_prev(src) / out_degree(src)) + + damping * sink_mass / node_count +``` + +Each iteration computes: + +- contribution records from source nodes to destination nodes +- sink mass from zero-out-degree nodes +- reduced next-rank values per destination node +- convergence delta, for example L1 norm + +Stop when either: + +- `delta <= tolerance` +- `iterations_completed == max_iterations` + +If the iteration cap is reached without convergence, publish the bounded result +by default and mark the metadata as approximate: + +```json +{ + "converged": false, + "iterations_completed": 50, + "delta": 0.000034 +} +``` + +PageRank is commonly used as a bounded iterative approximation, so a valid +non-converged fixed-iteration result is still useful. The job should fail and +preserve the prior published generation only for invalid output or corrupt +state, such as NaN scores, infinities, missing graph metadata, or incomplete +iteration output. + +## Execution and Memory Architecture + +Graph metric kernels consume an immutable ordinal topology whose adjacency +lanes are selected by capability: degree retains counts only, PageRank retains +incoming neighbors plus outgoing counts, eigenvector retains incoming +neighbors, and HITS retains both neighbor lanes. A compatible metric group +builds the union once. Endpoint ordinals are validated during construction so +reused topology is not rescanned for every metric. + +Serverless compilation enforces the peak-memory limit with a live allocation +limiter after charging the decoded source graph. Admission therefore follows +observed vertex, edge, and distinct-edge-type cardinality instead of rejecting +low-cardinality graphs using a pessimistic per-edge string estimate. Encoders +borrow canonical node IDs, and paired HITS outputs are encoded and published +one at a time. + +Large dense and adjacency passes use the caller's shared `std.Io` runtime. +Floating-point reductions use fixed logical partitions and merge them in a +stable order, so changing execution parallelism does not change score bits. +The embedded compatibility runner uses the same storage-independent kernels in +serial mode; it is an oracle and rollback path rather than a second algorithm. +Serverless operators can bound per-materialization CPU fanout with +`--graph-metric-max-parallelism` or +`ANTFLY_SERVERLESS_GRAPH_METRIC_MAX_PARALLELISM` (range 1-16, default 4). +Admission and execution share one algorithm-specific logical cost model: +PageRank, eigenvector, and HITS account separately for adjacency passes, dense +reductions, normalization/scaling, and vector setup. This keeps the configured +work ceiling meaningful for sparse graphs and prevents HITS from being admitted +using a cheaper single-vector estimate. + +The durable planned runner treats attempt records and per-page contribution +records as recovery journals, not history. Iterative producers write immutable, +attempt-tagged ordinal shards directly in reducer order; the final checkpoint +and producer completion commit atomically. There is no contribution copy/delete +adoption pass. Every iterative build uses checkpointed summary leaves and a +scalar-only root, regardless of node count, because small graphs can still have +dense edge sets. Consumers select only completed producer attempts and retain +inputs until the entire consumer phase finishes. Bounded, cursor-based barrier +cleanup then retires all winning and abandoned shards. The working set does not +grow with the configured iteration count; final cleanup removes job state. + +Scan-page adoption also maintains one target-owned out-degree total per node. +The retained per-page value is an idempotency ledger: a reclaimed attempt +replaces that page's value and adjusts the total by its delta. Initialization +therefore performs one point lookup per node instead of probing every scan +partition, while retry safety remains independent of worker identity. + +Published native generations keep the complete node-keyed score vector for +point reads, but their score-ordered secondary index retains only the best +10,000 entries, matching the public `top_k` ceiling. Each publication page +merges its local top prefix with that bounded durable prefix, so top-k remains +`O(K)` to read without doubling persistent score storage. Native reranking and +graph projections resolve every dependency column through one stable read +snapshot and reuse retained cursor storage across maintenance pages. Graph filtering, ordering, and limiting +then stay columnar: clause names are resolved once, bounded result pages use +`O(N log K)` selection, and surviving rows are views into one contiguous +metric-value slab with one owned copy of each metric name. Native and serverless execution call the same +storage-independent selector, so ordering, null placement, filtering, limits, +and deterministic tie-breaking cannot drift between deployments. + +Search reranking is a bounded two-stage retrieval operation. `candidate_count` +controls the first-stage window (default `offset + 4 * limit`, maximum 10,000), +the graph metric scores and sorts that window, and only then are `offset` and +`limit` applied. An explicit window must cover the requested page. This avoids +the misleading UX of reranking only an already-truncated page while keeping +score reads and latency predictable for operators. + +Serverless point, projection, rerank, and direct top-k reads share one budget +ledger on the pinned request session. The ledger composes authenticated range +operations, transferred bytes, decoded blocks/work, and retained result-column +and status memory across every named operation and metric dependency. This closes the +per-metric-limit loophole where a valid request could multiply the allowed I/O +by its dependency count. Exhaustion fails before the next backend range read +and is returned as an actionable, non-retryable HTTP 422. The unreleased +serverless format has one accepted wire version: discarded pre-release layouts +are rejected rather than carried as a permanent compatibility surface. +Independent immutable score ranges use bounded eight-way `std.Io` +fanout, while every child view shares the same synchronized request ledger and +pinned manifest. Range payloads are decoded and released one fanout batch at a +time, so peak temporary memory is bounded by concurrency rather than the total +number of planned ranges. Parallel metric workers write into disjoint +request-owned result columns, avoiding a second full-column clone from the +thread-safe transient allocator. Point-score planning uses a sparse sorted +worklist, so the common path allocates in proportion to requested nodes and +touched blocks instead of the complete routing table. Public graph shaping +carries stable source-row +ordinals through filter and order stages, fetches later dependencies only for +surviving rows, and moves nodes once when the final projection is materialized. + +Planned native maintenance pins the index catalog with a shared lifetime guard +for each bounded scheduler unit. It does not hold the database-wide apply fence: +graph storage transactions, page leases, attempt identities, and publication +generation fences are the concurrency boundary. A short cooperative pause after +durable progress protects foreground storage latency without serializing graph +writes behind a complete maintenance page. + +Each computational native phase also maintains an order-independent progress +record in the same transaction as its page mutation. Claims advance a durable +round-robin cursor, idle coordinator ticks can decide incomplete/failed state +without a page scan, and exhausted-page scans are skipped unless a page has +reached the attempt ceiling. Cleanup instead uses its bounded page/job cursor, +so every retirement path remains a single job-namespace protocol. Once every +computational page is complete, the coordinator performs one page-key-ordered +floating-point reduction and caches the phase summary. This keeps publication +bit-deterministic without recomputing floating aggregates after every +completion. + +Planned score workers likewise write only their disjoint node-keyed score +pages. They do not maintain the generation-wide score-ranked keyspace at every +checkpoint. After verification, the coordinator selects the exact supported +top-10k once from the immutable generation and installs that bounded secondary +index in the same transaction that flips the publication pointer (and both +indexes/pointers for paired HITS). Readers therefore retain O(K) top-K access +without turning every 4k-score checkpoint into another generation scan and +sort. + +Public serverless query shaping treats the selector's returned parent indexes +as the row-lineage authority. Resident metric columns are transactionally +rebased with direct indexing after each filter/order transition, filter-only +and order-only columns are released before the next transition, and a failed +allocation leaves the entire cache on its prior lineage. Cross-column I/O also +preallocates every caller-owned result buffer before scheduling the first +worker, so no error path can release stack state or score storage still used by +an asynchronous child. + +External serverless reconciliation accepts the same caller-owned compute +runtime as ordinary lake builds, keeping CPU fanout under one operator-visible +limit. PageRank accepts authenticated, ordinal-aligned seeds from +the last compatible publication. Both document-backed publication and lake +reconciliation use the same seed admission and mapping path. It linearly maps the prior +node-sorted vector onto the new projection, assigns zero to new nodes, skips +deleted nodes, and lets the kernel validate and normalize the result. A +disjoint, rejected, over-budget, incompatible, missing, or corrupt prior artifact +cold-starts. Unauthenticated seeds are never used; optional fetch failures do not +block publication from authoritative topology. Cancellation and allocation +failure still propagate. Seed admission covers both preparation +(including decoded routing memory) and execution alongside kernel/output memory. +Cold kernel work is admitted before optional seed I/O. A separate per-publication +seed budget caps input at 64 MiB and decode/mapping work at 67,108,864 units +(one prior byte plus one current node per unit). Every actual read is charged, +including repeated identities; no seed cache is implied. Zero allowance disables +warm starts. Exhausting this optional budget cold-starts without consuming later +metrics' cold execution allowance. Materializer epoch 10 fingerprints this policy +and the independently authenticated paged routing format. +An optional seed never turns an admissible cold build into a budget rejection. +Native PageRank pins its seed generation and configuration for the job and computes +the surviving seed mass through the bounded, checkpointed initialization summary. +Every worker uses that same scalar before writing ranks and source factors. Zero +surviving mass falls back to the uniform vector. Execution epoch 9 rejects older +in-flight jobs: rebuild them with upgraded workers. Their previously published +score generation remains readable; partially computed old seeds are not resumed. +The same job/configuration/epoch fence applies to workers, coordinator phase +transitions, and the final publication transaction, including paired HITS. +Continuing an unexpired owned lease does not consume a retry; the final attempt +may finish any number of bounded checkpoints, but cannot be reclaimed after expiry. +Warm starts preserve PageRank's probability normalization, but finite-iteration +results and tolerance-based stopping can depend on the seed. + +All iterative native jobs assign stable, partition-local ordinals in +the initialization producers. The dictionary is immutable for the job and is +retired with job state. PageRank ranks/source factors and both spectral vectors +use 256-slot blocks with explicit presence bits; zero and missing output are +distinct. Sorted multi-get resolves dictionary slots, deduplicates block reads, +and preserves caller order. Attempt-fenced checkpoint writes update each block +once and retire obsolete iterations by block. Small graphs use the same bounded +pipeline, with one summary leaf, one scalar root, and one data partition. + +Iterative native jobs compile immutable row-oriented adjacency during the first +contribution phase of each lane. HITS stores both orientations, so hub reduction +does not repeatedly scatter across target-sorted physical edges. Each compile +checkpoint scans at most 4,096 edges and emits tiles of at most 256 neighbor/output +ordinal pairs, grouped by output vector block. Later contribution phases are +metadata-only barriers. Reducers gather the current input vector directly; no +numeric contribution shuffle is written on any iteration. Adjacency identities +include the first-iteration producer attempt and checkpoint. Before the first +fold, each output chunk packs only winning producer fragments into dense 256-edge +tiles. Compaction checkpoints bound both physical records (512) and edges (4,096), +persisting the input cursor and partial tile atomically. A completion receipt +selects one immutable packed attempt; replacement attempts cannot mix abandoned +output, and missing or truncated tiles fail closed. Document IDs remain at the +input/output boundary; node-oriented reducers resolve ordinals by sorted multi-get. +A worker step may run one bounded packing pass followed by one bounded fold, +so small chunks do not wait another maintenance tick after receipt publication. +Receipt preflight prevents repeated numeric work while another chunk is packing. + +All native reducers retain packed immutable adjacency until job cleanup. +Checkpointed output may be replayed safely after lease takeover. Once every +consumer finishes, the phase barrier deletes at most 512 temporary records per +transaction, persisting a last-deleted-key cursor atomically with each batch. +Restart seeks beyond that cursor instead of rescanning LSM tombstones. Completed +namespaces retain a durable sentinel until job cleanup. Retirement counts flow +through DB maintenance sweeps, idle detection and runtime status +(`total_retired_input_records` / `last_retired_input_records`). The barrier advances +only after cleanup finishes. Both HITS lanes retire producer fragments after +iteration zero and raw vectors each iteration. +Retained topology is proportional to edges and bounded producer attempts, not +the number of power iterations; temporary vectors remain iteration-bounded. + +Large reduction passes reuse the immutable node quantiles: independently leased +producers fold at most 16 physical adjacency tiles (at most 4,096 edges) for a group of up to +256 nodes per checkpoint. A checkpoint-local vector-block cache deduplicates +neighbor reads across all its tiles without retaining state across transactions. +Their attempt-fenced cursor and compensated sums +survive restart; a replacement attempt recomputes without mixing old state. +Completed folds atomically publish raw vector blocks and partial spectral norms +or PageRank dangling mass. The root deterministically combines at most 256 +completed scalar records. Data reducers stay behind that barrier and read only +raw vector blocks, avoiding a second adjacency traversal for normalization. +Raw vectors retire at the consumer barrier, bounding temporary state by an +iteration. Filtered-out ranges contribute empty leaves. + +Serverless metric wire version 9 separates the ranked routing root, a sparse +directory, and 64-entry primary routing pages. Manifest version 18 authenticates +the root, which authenticates the directory; each directory entry authenticates +one routing page, whose entries authenticate score blocks. Point reads load +only selected pages, with global block ordinals preserving score-cache identity. +Indexes fitting in one routing page or a 64 KiB footer retain a single footer fetch and decoded +cache lease, avoiding extra network round trips for small metrics. +Only missing contiguous pages coalesce into authenticated ranges; independent runs +use the same bounded parallel executor as score ranges (per column: eight reads and 32 MiB +in flight, with each coalesced range capped at 8 MiB). A query prepares every +column's control and routing, consuming authenticated cached score blocks before +admitting score network reads. Contiguous misses form zero-overfetch runs capped +at 8 MiB, sharing the remaining request budget without per-column quotas. +When those runs exceed the request allowance, an exact bounded partition planner +minimizes bytes across all columns. It considers partial merges, splits within +miss runs, and ranges crossing former fixed-window boundaries. A monotone queue +keeps planning work O(missed blocks × remaining requests); scratch memory and work +are admitted against the shared query limits before allocation. Ranges omit unused +boundary blocks and bridge authenticated cached gaps only when required to fit +the request budget. They never bridge a discontinuity in authenticated extents; +partial cache warmth must not reduce admission feasibility. +The complete score request and byte totals are reserved atomically before +bounded parallel execution. An uneven query therefore does not fail merely because +one column needs more than an equal share. The shared budget remains authoritative +across all metric surfaces of the pinned query. +Top-K reads fetch only the root (at most 1,872 bytes) and requested ranked blocks, +never the point directory or pages. The root also binds primary vector extents +and the complete point-index digest for full-artifact/warm-start validation. +Decoders validate the cross-tier relationship. Unreleased older +graph-metric wire versions are rejected rather than migrated at query time. + +Eigenvector and HITS always use canonical cold seeds in both runtimes. An old +spectral vector may have zero support on a newly dominant disconnected component; +normalization alone cannot make that a safe warm start. A future spectral restart +policy needs component/support guarantees and oracle tests for topology changes. + +Serverless query caches retain authenticated decoded metric roots and directories under +an independent 16 MiB process-memory budget (configurable with +`max_graph_metric_routing_bytes`). Identity includes the immutable artifact, +checksums, footer extent, and wire version. Leases keep borrowed IDs alive; +eviction skips pinned entries and saturated caches bypass retention. +Misses reserve bounded per-identity fill ownership before fetching or decoding. +Waiters yield through caller-owned `std.Io`, check cancellation independently, +and can take over when a failed/canceled producer releases ownership. At most +64 distinct fills are admitted concurrently; cache saturation applies backpressure. +Registered waiters pin a shared result independently of LRU admission, including +when retention is disabled or existing entries are pinned. The producer may drop +its lease before waiters wake without losing the result. Cancellation releases +the waiter's registration; the last reference releases a bypassed result. This +does not create another unbounded cache: fill registrations have fixed capacity, +and every consuming query applies its retained-memory admission to the lease. +Routing pages use the authenticated block cache and are decoded only for selected +pages, so a large primary index cannot repeatedly bypass the decoded-index cache. +Routing pages and primary score blocks have canonical cache identities using the +immutable artifact, global block ordinal, and exact extent, independent of the +candidate set or transport batch. Readers probe those units before planning +transport, authenticate the entire coalesced response before publishing any unit, +and never retain candidate-specific combined routing or primary-score ranges. +Publication groups at most 32 canonical blocks (and 8 MiB) per batch, bounding +simultaneously open per-key leases. A cache-wide 32-publication admission limit +also bounds descriptor use across parallel queries; saturation bypasses optional +retention without waiting or evicting useful entries. Each batch +reserves capacity and performs eviction once, writes payloads outside global +maintenance/coordination locks, then commits independent canonical entries. +Small caches retain a fitting subset instead of bypassing the entire batch. +Existing per-block durable reservations, nonblocking publication ownership, +cross-process usage reconciliation, and abandoned-write recovery remain intact. +Point, routing-page, and ranked top-K reads also share a process-local canonical +block pool, bounded to 4,096 live blocks and 64 MiB of payload. Identity binds the +artifact, checksum, extent, and authenticated block digest, never K or a candidate +set. Missing block sets are claimed atomically before transport: a reader waits +without holding other fill claims, preventing overlapping-query deadlocks. +Registered waiters pin completed or failed producer state; cancellation does not +cancel another reader's producer, and producer failure permits takeover. +Failed pinned entries still consume both admission limits. Ready unpinned entries +are evictable, and warm point preparation can copy them without disk I/O or +re-authentication. Payload bytes, live entries, and waiters are reported in +query-cache statistics. +Only missing contiguous runs are downloaded. If newly shared interior hits would +require more requests than the already-admitted range budget allows, execution +keeps its original bounded range rather than failing a query due to cache warmth. +Ranked blocks use canonical disk identities too: changing top-K across a block +boundary fetches the new suffix, rather than retaining overlapping prefix ranges. +Fetch temporaries retire before contiguous result allocation, preserving the +per-query in-flight payload bound independently of the shared pool's fixed budget. +Overlapping candidate sets therefore reuse already-fetched blocks even when their +routing-page sets or coalescing boundaries differ. Cached scores are decoded into +the final result during preparation without retaining all cached payloads; they +consume decode/work budgets but no score network request/byte reservation. The +latest artifact's existing authenticated blocks define these units; no additional +wire format or legacy cache reader is needed. +Parallel range payloads use a thread-safe allocator and transfer ownership +explicitly, independently of the allocator owning returned scores or per-request +routing arrays. +PageRank's fixed logical reduction partitions account for edge work as well as +vertex count. Edge-heavy graphs below the vector-parallelism threshold therefore +use configured compute workers, while logical reduction order stays identical +across serial and parallel execution. +Stateful reverse-edge probes validate the catalog's index incarnation and config +fingerprint, plus the source shard's read generation, under the storage apply +lease that protects the reverse snapshot, including probes without hydration. +An index replacement cannot certify old negative answers under a new routing +cache identity. Index-incarnation mismatches retain their identity across native +and remote HTTP boundaries. Public queries release the failed snapshot and retry the +whole query at most once, respecting cancellation and deadlines; continued +reconciliation returns the existing retryable `index_rebuilding` response. +No storage or decode operation runs under the cache lock. Warm point +and top-k reads avoid footer I/O and full-index decoding. Decode-cache hits, +misses, and retained bytes are exposed in query-cache statistics. + +Ordinal dictionaries are scoped to an exact edge-filter projection; graph-wide +ordinals are not interchangeable because filters change the active node set. +The compute lifecycle treats that projection as the family boundary and shares +its topology across every compatible column. Durable score artifacts remain +independently addressable by column: point reads are the primary serving shape, +so they must not fetch unrelated columns or couple cache eviction and rebuild +failure domains. Paired HITS shares one kernel execution while publishing its +two columns independently. A packed multi-column wire is therefore gated on a +scale benchmark that proves its storage savings exceed its extra random-read +and lifecycle costs; it is not assumed to be an unconditional optimization. +PageRank contributions remain target/page qualified so retry adoption and +reclaimed-attempt replacement stay deterministic. Serverless is unreleased and +accepts no compatibility readers for discarded graph-metric prototypes; +native in-flight attempt changes still require an explicit rolling-upgrade +bridge. + +Native durable cross-job topology reuse remains a benchmark-gated follow-up, not +an implicit extension of the compute-family sharing above. Measure repeated +compatible metric jobs against the current job-scoped packed topology, including +cold/warm wall time, edge-scan bytes, topology bytes written, peak RSS, and cleanup +cost at representative graph sizes. A shared artifact must identify the exact +edge generation, filter, orientation, and ordinal/layout version; publish only +complete immutable data; and acquire durable job pins atomically with attachment. +Concurrent builders, failed publication, restart recovery, and last-pin garbage +collection need fault-injection coverage before replacing job-scoped ownership. +Sharing ordinal arrays by metric configuration alone is not safe. No durable +cross-job cache or measured speedup is claimed by this PR. + +## Query Integration + +At query time: + +1. Resolve the graph metric config. +2. Resolve the published metric generation. +3. Read scores for returned candidate nodes. +4. Apply metric ordering if requested. +5. Include freshness metadata when requested. + +Internal graph query types likely need fields similar to: + +```zig +pub const GraphMetricRead = struct { + name: []const u8, + freshness: GraphMetricFreshnessMode = .published, +}; + +pub const GraphOrder = union(enum) { + field: GraphFieldOrder, + metric: GraphMetricOrder, +}; + +pub const GraphMetricOrder = struct { + name: []const u8, + direction: OrderDirection = .desc, +}; +``` + +Metric values should be returned as nullable floats. Missing scores can occur +for nodes introduced after the last publish. Ordering should treat missing +scores explicitly, with the default being `nulls_last`. + +Before the first publish, direct metric ranking paths should not try to rank all +nodes as missing. They should fail with `MetricNotReady`. + +## Progress and Recovery + +Jobs should persist enough state to recover after restart: + +- job id +- metric name +- target edge generation +- building score generation +- current phase +- iteration number +- page cursor +- accumulated convergence delta +- worker lease owner and expiration +- last error + +Recovery should be idempotent: + +- Re-running a contribution page overwrites the same generation/iteration output. +- Re-running a reduce page overwrites the same next-rank output. +- Re-running publish checks that the target generation is complete before + flipping the pointer. + +If graph writes happen during a build, the active build can still publish its +target generation. The dirty marker remains newer than the published generation, +which schedules a later rebuild. + +## Why PageRank First + +PageRank is the best first centrality metric because it behaves well on directed +graphs, disconnected graphs, and sink-heavy graphs. The damping factor gives +stable results even when the graph does not have the structural properties raw +eigenvector centrality expects. + +Eigenvector centrality can reuse the same materialization framework later, but +it needs more careful handling for disconnected components, reducibility, and +non-convergence. + +## Implementation Plan + +1. Add graph metric config and schema parsing for `pagerank`. +2. Add dirty and published generation metadata keys. +3. Add local PageRank maintenance using the durable background lane. +4. Store generationed scores and publish through a metadata pointer flip. +5. Add query support for returning metric scores. +6. Add direct top-k metric reads. +7. Add status reporting for freshness and build progress. +8. Add tests for write dirtiness, successful publish, failed rebuild preserving + prior generation, stale reads, and ordering by score. +9. Add distributed phased jobs after the local implementation is stable. + +## Long-Term Design Roadmap + +The first PageRank implementation should prove the core generation contract: +graph-index-owned metric config, durable dirty state, complete-generation +publish, direct metric reads, and observable freshness. The roadmap below is the +production shape beyond that first slice. + +This roadmap treats PageRank as the first product surface for a broader graph +metric subsystem. The long-term design goal is that future centrality metrics, +distributed execution, richer edge scopes, and retrieval composition extend the +same index-owned lifecycle instead of creating parallel APIs. + +Long-term graph metrics should evolve as an index subsystem, not as one-off +query operators. Every new metric should reuse the same lifecycle: + +```text +graph writes + -> durable metric dirtiness + -> local or distributed metric job + -> generationed score writes + -> atomic publish pointer flip + -> snapshot-safe cleanup + -> query/status/read APIs +``` + +The sequencing principle is to expand one axis at a time: + +- First make one metric reliable on one node. +- Then make that metric compose with graph queries. +- Then distribute the same lifecycle. +- Then add more metrics through the same framework. +- Then add richer scopes and operational controls. + +Avoid adding new metric-specific query surfaces after PageRank. Eigenvector, +HITS, degree, personalized PageRank, and future centrality metrics should all +look like named graph metrics to users and should differ only in config, +metadata, and documented convergence behavior. + +Roadmap summary: + +| Phase | User-facing outcome | Main implementation work | +| --- | --- | --- | +| 0. Framework baseline | PageRank is queryable as a named graph metric. | Local dirty tracking, generationed score storage, atomic publish, status, and cleanup. | +| 1. Query integration | Traversals/searches can project, order, and filter by metrics. | Metric lookup in graph execution, freshness checks, deterministic ordering, and OpenAPI/client updates. | +| 2. Distributed jobs | Large graphs rebuild metrics durably across workers. | Job records, leases, resumable phases, idempotent pages, progress, retries, and verified publish. | +| 3. Metric families | Degree, eigenvector, HITS, and later personalized PageRank reuse the same surface. | Shared algorithm runner contracts, metric-specific convergence metadata, paired HITS publish, and high-cardinality safeguards. | +| 4. Edge scope | Users can maintain scoped metrics for specific edge families. | Resolved edge-filter metadata, typed validation, materialization invalidation, and status/explain output. | +| 5. Operations | Admins can refresh, rebuild, pause, resume, delete, and observe metrics. | Idempotent controls, retention policy, event logs, queue visibility, and safe cleanup. | +| 6. Retrieval composition | Graph metrics can contribute to explicit reranking and planner features. | Score features, freshness-aware planning, cross-shard top-k merge, and explain/profile integration. | +| 7. Compatibility | Existing PageRank APIs keep working as the framework grows. | Metadata versioning, compatibility aliases, migrations, and deprecation windows. | + +### Phase 0: Framework Baseline + +The baseline is the minimum viable graph metric framework: + +- graph index config owns `metrics` +- metric config resolves to a stable name, kind, refresh mode, and edge filter +- edge writes persist dirty state +- maintenance computes a complete score generation +- publishing is a metadata pointer flip +- queries only read the published generation +- failed builds preserve the prior published generation +- direct metric reads expose status + +This phase should stay intentionally local. Its job is to prove the storage and +freshness contract, not to optimize for very large graphs. + +Milestones: + +- Persist dirty and published generation metadata. +- Implement local PageRank over all edges and typed edge include lists. +- Publish fixed-iteration non-converged PageRank with `converged: false`. +- Reject invalid score output such as NaN or infinity. +- Clean up old generations immediately when reads are snapshot-safe. +- Add public tests for first publish, stale publish, top-k reads, and status. + +Exit criteria: + +- A graph can accept writes while metric maintenance runs. +- A query never observes partially written metric scores. +- Restart preserves dirty state and the last published generation. +- The user can distinguish `not_ready`, `fresh`, `stale`, `building`, and + `failed`. + +Current implementation progress: + +- Local PageRank dirty-state handling now has restart coverage: after a fresh + publish, a later edge write persists stale/queued status across graph/store + reopen, continues serving direct metric reads from the prior published + generation, and rebuilds the later edge generation when maintenance runs. +- Local graph metric failure handling now has restart coverage for the critical + publish invariant: after a successful PageRank publish, a later invalid + rebuild records durable failed status, preserves the prior published + generation across graph/store reopen, and continues serving direct top-k reads + from the last complete generation until a later successful rebuild clears the + failure. + +### Phase 1: Complete Query Integration + +Direct metric reads are useful for top-k centrality lists, but graph metrics +become more valuable when they compose with graph traversal, graph search, and +ordinary retrieval. + +Add metric projection to graph query nodes: + +```json +{ + "graph_searches": { + "related": { + "type": "traverse", + "index_name": "knowledge_graph", + "start_nodes": { "keys": ["doc:alice"] }, + "params": { "edge_types": ["mentions"], "max_depth": 2 }, + "metrics": ["pagerank"], + "include_metric_status": true + } + } +} +``` + +Response nodes should expose metric values as nullable floats: + +```json +{ + "key": "doc:bob", + "depth": 1, + "metrics": { + "pagerank": 0.0182 + } +} +``` + +Projection semantics: + +- Missing metric generation returns `null`. +- Missing node score in an existing generation returns `null`. +- `metric_freshness: "published"` allows stale projected scores. +- `metric_freshness: "fresh"` fails with `MetricNotReady` or `MetricStale`. + +Add metric ordering for graph traversal/search results: + +```json +{ + "order_by": [ + { "metric": "pagerank", "direction": "desc", "nulls": "last" } + ] +} +``` + +Ordering semantics: + +- Ordering by an unpublished metric fails with `MetricNotReady`. +- Ordering with `fresh` over stale scores fails with `MetricStale`. +- Missing scores sort with explicit `nulls_first` or `nulls_last`; default + `nulls_last`. +- Ties break by the graph query's existing deterministic node order, then key. + +Filtering by metric should come after ordering support: + +```json +{ + "where_metric": [ + { "metric": "pagerank", "op": "gte", "value": 0.01 } + ] +} +``` + +Filtering must fail closed if the metric is not published or if `fresh` is +requested and stale. + +Implementation milestones: + +- Add metric projection to graph traversal/search result nodes. +- Add optional `metric_status` to graph query responses. +- Add metric ordering with deterministic tie-breaking. +- Add metric filtering after ordering semantics are stable. +- Thread `metric_freshness` through local and remote table-read paths. +- Regenerate OpenAPI/client types for the public query surface. + +Do not add planner-level score blending in this phase. Keep metric usage +explicit and local to graph query results until freshness and distributed merge +semantics are tested. + +Current implementation progress: + +- Direct DB graph metric reads now have focused freshness coverage for stale + materializations: `metric_freshness: "published"` returns the last complete + generation with stale status, while `metric_freshness: "fresh"` fails closed + with `MetricStale` after a later graph write advances the edge generation. + Active planned-rebuild coverage now also proves public direct metric top-k + queries with `published` use the prior score generation while reporting + building status, and `fresh` fails closed before the planned generation + publishes. Failed planned-rebuild coverage now proves the same public direct + top-k path continues to serve the prior published generation while reporting + failed status, and `fresh` remains fail-closed. Public DB search coverage now + also proves direct graph metric top-k returns `MetricNotReady` for both + `published` and `fresh` reads before the first generation publishes. +- DB graph traversal/search metric reads now have matching stale/fresh coverage: + published projection can return stale metric scores and status, while fresh + projection, ordering, and filtering over stale graph metric materializations + fail closed with `MetricStale`. The DB graph query conversion path also + releases temporary graph metric status ownership after cloning it into the + public search result. Active planned-rebuild coverage now also proves + traversal projection with `published` reads the prior score generation while + reporting building status, and `fresh` projection/order/filter fail closed + while the planned generation is unpublished. Failed planned-rebuild coverage + now proves the same traversal projection path keeps serving the prior + generation while reporting failed status, and `fresh` projection/order/filter + remain fail-closed. These traversal projection/order/filter not-ready, stale, + building, failed, and fresh-failure DB cases now run in the fast root suite. +- DB search rerank now has active planned-rebuild freshness coverage: + `metric_freshness: "published"` continues to rerank with the last complete + score generation while reporting building status, and `fresh` fails closed + while the planned generation is still unpublished. Failed planned-rebuild + coverage now also proves published rerank preserves the prior generation while + reporting failed status, and fresh rerank remains fail-closed. Public DB + search coverage now also proves search rerank returns `MetricNotReady` for + both `published` and `fresh` reads before the first generation publishes. +- DB graph traversal/search metric reads now also cover unpublished metrics: + published projection returns a null metric score plus `not_ready` status, + while fresh projection and published ranking/filtering fail closed with + `MetricNotReady` before the first metric generation is published. +- Public HTTP API graph metric e2e coverage now has a focused build step. It + creates a table, a graph index with PageRank, degree, eigenvector, and + compatible HITS metrics, first writes typed edges with `sync_level: "write"`, + verifies public status reports `not_ready` with no published generation, and + proves direct metric reads plus search rerank fail closed while traversal + projection can expose null scores with not-ready metric status. It then writes + the same graph with `sync_level: "full_index"`, verifies public metric status + metadata and edge filters, and exercises fresh direct graph metric top-k plus + graph traversal projection, ordering, and filtering through the public query + route. The same public e2e now follows with a later `sync_level: "write"` + graph update and verifies `published` direct graph metric top-k and traversal + projection keep serving the prior published generation with `stale` status, + published search rerank exposes stale status in profile output and + prior-generation score details, while `fresh` direct, traversal, and rerank + reads fail closed. The broader `public-api-parity-test` also remains green with + generated OpenAPI query bodies that include explicit `null` graph metric + fields, so graph metric raw-body parsing now matches ordinary optional-field + semantics. + +### Phase 2: Distributed Metric Jobs + +Local maintenance is enough to validate storage and API semantics. Production +large-graph metrics need the same durable distributed job shape used by +relational rebuild work. + +Add a graph metric job table with: + +- metric name and graph index name +- target edge generation +- building score generation +- phase and iteration number +- cursor/progress keys +- worker lease owner and expiration +- retry count and last error +- publish eligibility metadata + +Recommended phases: + +```text +prepare_generation +scan_edges_and_out_degree +initialize_ranks +iterate_contributions +reduce_ranks +check_convergence +publish_generation +cleanup_old_generations +``` + +Distributed invariants: + +- Partial scores are never visible to queries. +- Every intermediate key includes metric name, target generation, and iteration. +- Contribution and reduce pages are idempotent overwrites. +- Publish verifies all required pages for the target generation are complete. +- Writes during a build do not cancel the build; they leave the dirty generation + newer than the published generation and schedule a follow-up build. +- Worker loss only abandons a lease, not the generation. + +The status API should expose distributed progress without leaking internal page +keys: + +```json +{ + "state": "building", + "phase": "iterate_contributions", + "iteration": 12, + "progress": 0.64, + "target_edge_generation": 84, + "published_generation": 72 +} +``` + +Implementation milestones: + +- Add durable graph metric job records and worker leases. +- Split PageRank into resumable phases with idempotent page writes. +- Track contribution, reduce, and convergence metadata per iteration. +- Make publish verify completion before flipping the pointer. +- Expose status by phase, iteration, target generation, and progress. +- Add restart/retry tests that kill work between phases. + +The distributed implementation should not introduce a different public API. +Users should see better scale and progress visibility, not a new way to request +PageRank. + +Current implementation progress: + +- Local graph metric builds now write a durable per-metric build job record + alongside the active lease. The record captures the build job id, target edge + generation, building score generation, start/update timestamps, lease + expiration, worker id, phase, iteration, opaque build cursor, completed/total + work units, retry count, and last error. Active local builds now report the + same long-term phase vocabulary planned for resumable workers, including + `prepare_generation`, `scan_edges_and_out_degree`, `initialize_ranks`, + `iterate_contributions`, `check_convergence`, and `publish_generation`. The + job record is updated with the same phase/iteration progress used by active + status, persists cursor/unit progress across graph-index reopen while a build + lease is active, records failed build retry/error details durably, and is + marked `complete` with no active lease expiration or failure details after a + successful publish. The status and OpenAPI surfaces now expose cursor and + work-unit fields for active builds so distributed workers can reuse the + observable shape later. This is still a local job-table primitive for the + future distributed/resumable worker implementation; it does not yet split + PageRank into distributed pages or change the public metric API. +- Active local metric builds now also plan a durable per-job manifest under the + graph metric control namespace. The first manifest version records the job id, + target edge generation, building score generation, metric config fingerprint, + planned edge/node counts, phase count, and page count. The planner writes one + deterministic pending page per phase for the selected metric kind: degree uses + the short prepare/scan/publish/cleanup phase list, PageRank and eigenvector + use the iterative prepare/scan/initialize/contribute/reduce/check/publish/ + cleanup list, and HITS uses the paired + prepare/scan/initialize/authority-contribute/authority-reduce/ + hub-contribute/hub-reduce/check/publish/cleanup list. Re-planning the same + manifest is + idempotent and does not reset an already completed page, and restart coverage + now verifies that the manifest and page records survive graph-index reopen. + Build pages now persist versioned range metadata: range kind, opaque lower + bound, opaque upper bound, and output namespace prefix. Degree and PageRank + now both plan deterministic key-range pages for scan and node phases, and + PageRank later-iteration pages are appended by the convergence coordinator + when another iteration is required. The PageRank manifest now increments its + durable page count for newly appended later-iteration pages and keeps + re-planning idempotent; applying the same dynamic accounting model to later + iterative metrics remains future work. Phase barriers and iteration summaries + are now covered by the durable coordinator primitives below. +- Durable build pages now have the first lease lifecycle primitives. A worker + can claim a pending or failed page, same-worker claims renew the active lease, + other workers are refused while the lease is active, expired leases can be + reclaimed with the attempt counter incremented, and completed pages cannot be + claimed again. Page completion is idempotent for the same output fingerprint + and rejects conflicting fingerprints. Failed leased pages persist the worker, + attempt count, and last error, and can be claimed again by a later worker. + Leased pages can also persist progress cursor and completed/total unit + updates without completing the page; active job status mirrors the current + page cursor, same-worker lease renewal preserves it, and restart coverage + verifies that page/job cursor progress survives graph-index reopen. Workers + can also claim the next eligible page by scanning the durable page prefix for + the current phase/iteration; the scheduler skips active leases, claims pending + or failed pages, and reclaims expired leases through the same attempt-counting + lifecycle until an internal bounded retry policy is exhausted. Exhausted pages + now make the coordinator record a failed active build instead of leaving the + planned scheduler idle on an unclaimable page. This establishes the + recovery/retry/progress state machine needed by distributed page workers. + PageRank now uses this path for scan/out-degree, + initialize, contribution, reduce, convergence, publish, and cleanup pages; + scan, contribution, reduce, and convergence pages can persist cursor progress, + and contribution/reduce/convergence pages now have reopen-and-resume coverage + on initial and dynamically planned later-iteration pages. Dynamically planned + later-iteration contribution/reduce/convergence pages also have failed-page + retry coverage. Cleanup prefix deletion now has durable cursor progress and + same-worker renewal coverage across graph-index reopen. Reclaimed PageRank + scan, initialize, contribution, and reduce pages now have coverage that + expired partial output is recomputed without double-counting or preserving + stale out-degree, node, contribution, or rank records, and reclaimed + convergence pages now clear stale partial summaries before recompute. + Remaining work is broader production expired/reclaimed coverage across all + phases and later iterative metrics. +- Build jobs now have a durable phase summary and barrier primitive. The + coordinator can summarize a job phase into expected/completed/failed page + counts, completed/total units, output fingerprint, and reserved convergence + fields (`max_delta`, `total_delta`, `rank_sum`, `converged`) by enumerating + every durable page record for the requested phase and iteration. The summary + is persisted under the job's phase namespace. Phase advancement is gated on + the summary being complete: incomplete or failed pages keep the job and active + lease on the current phase, while a complete summary advances both the build + job and active lease to the next planned phase. This gives the future + distributed PageRank executor a restart-safe multi-page barrier contract + before iteration-specific contribution/reduce/check executors are added. +- Convergence metadata is now part of durable page and iteration state. + `check_convergence` pages can persist `max_delta`, `total_delta`, `rank_sum`, + `converged`, and an output fingerprint. The phase barrier aggregates those + fields across completed check pages, and the coordinator can persist a + per-iteration summary with expected/completed page counts, convergence + metrics, convergence status, bounded fixed-iteration status, and output + fingerprint. This establishes the durable iteration summary contract needed to + decide whether a distributed iterative metric should publish, schedule another + iteration, or publish bounded non-converged output at `max_iterations`. +- Verified planned metric publish now flips the public generation pointer and + advances the active job to `cleanup_old_generations` without deleting the job + namespace in the publish transaction. The new generation is queryable while + cleanup is still pending. The cleanup page then deletes the per-job manifest, + page, phase-summary, iteration-summary, and partial keys under the durable + control namespace, clears the active build lease, and preserves the public + score generation plus current `build_job` status record. Degree, PageRank, + eigenvector, and HITS cleanup-resume coverage now also proves abandoned + attempt-scoped scan, contribution, and hub-raw output remains present during + non-final cleanup pages and is removed by the final job-namespace cleanup + page. Direct cleanup is still refused for an active job unless the cleanup + worker is executing the planned cleanup phase. Failed-job retention remains + future work. +- Planned metric builds now have the first verified publish readiness primitive. + The verifier requires the active build job to be at `publish_generation`, + checks that the durable manifest still matches the active job and current + metric config fingerprint, recomputes prerequisite phase summaries from page + records, requires every pre-publish phase/page to be complete, and records + iterative convergence readiness before allowing publish. This is not yet the + final distributed publish transaction and does not write score pages itself; + it is the metadata gate that future degree/PageRank page executors will call + before flipping the published generation pointer. +- Planned PageRank failure coverage now verifies that a failed rebuild for a + newer edge generation keeps the prior published generation queryable, removes + the abandoned planned-job namespace, deletes any unpublished score generation, + and preserves compact failed-job diagnostics for status. +- Failed planned builds now also append bounded retained failure diagnostics + under the metric control namespace. Status exposes the recent failure records + with sequence, job id, target/building generations, phase, iteration, retry + count, and last error, records matching failed metric events, and prunes both + failure records and event records beyond the fixed recent + retention cap. Fast root coverage now verifies failed planned builds remove + abandoned score generations and job namespaces, retain bounded diagnostics, + and prune old failure and event records. This keeps failure inspection useful + without letting failed job metadata grow without bound. +- `degree` now has the first executable planned-build path. The opt-in planned + runner claims durable pages, completes them with deterministic output + fingerprints, advances through the phase barrier, verifies publish readiness, + and flips the published generation pointer while marking the job complete. + Focused coverage compares planned degree output against the existing local + runner and verifies that an active planned rebuild for a newer edge generation + continues serving direct top-k reads from the prior published generation until + the verified publish completes. The scan page now reports a deterministic + page cursor before completion and executes through a dedicated reverse-edge + cursor scan page, rather than calling the local whole-metric collection + helper. Planned degree coverage now includes local-output parity, prior + generation reads while active, and typed edge-filter parity. This is still a + single-worker, one-page-per-phase executor, but it now runs through a + worker-step loop that claims the next eligible durable page for the active + phase, executes the phase-specific page handler, advances barriers, and + publishes from persisted score state. Its scan page reloads durable page + metadata and honors reverse-edge lower/upper bounds when executing. Planned + degree now writes scan output as job-scoped partials and materializes final + score generation entries in `reduce_ranks`. Coverage verifies that multiple + scan pages can contribute partial counts for the same node and reduce to the + correct final degree. The degree planner now creates deterministic + reverse-edge key-range scan pages and node-range reduce pages, and the worker + loop drains both before publish. Degree scan pages can now persist an opaque + reverse-edge cursor, resume from it on same-worker renewal, and merge resumed + partials into the same page output. Expired degree scan page leases can also + be reclaimed by another worker and safely recompute from the page range start + without double-counting abandoned partial output. Cleanup partitions exist for + degree job namespaces, and planned PageRank plus planned HITS now have local + alternating-worker coverage over partitioned pages. Degree, PageRank, + eigenvector, and HITS now also have threaded concurrent-handle coverage + through the public worker/coordinator boundary, including benign lost-lease + retry behavior for racing page workers. The same planned build boundary is now + exposed through DB/index-manager named calls for build ensure, worker page + step, coordinator step, failure, and drain; DB-level coverage proves a planned + degree generation can publish through index-name/metric-name worker and + coordinator calls without passing metric configs to workers. The + DB/index-manager layer now also has + bounded coordinator and worker sweeps: coordinator sweeps can start queued + background planned builds and advance active build barriers/publish steps, + while worker sweeps claim only active durable page leases by worker id. DB + coverage proves active planned degree, PageRank, eigenvector, and HITS builds + can finish through those sweeps without per-metric drain calls, including + PageRank/eigenvector iterative phase advancement, paired HITS publish, and + cleanup. Reopened DB-handle coverage now proves a PageRank build can be + started, worked, coordinated, published, cleaned up, and read through fresh DB + handles that share only durable job/page state. A bounded planned-maintenance + primitive now composes background-build startup, worker page sweeps, and + coordinator sweeps into one scheduler-facing call; DB coverage proves it can + drain background PageRank, degree, eigenvector, and paired HITS builds without + the local whole-metric runner. Budget exhaustion is now an observable result + flag rather than a scheduler error, and repeated tiny-budget ticks can resume + and finish the same background PageRank build. Pending-work stats now also + expose graph-metric planned-work hints for queued builds, active builds, + capped active/failed page summaries, paused metrics, and truncated page + status, giving the future idle scheduler a cheap way to decide whether the + planned path still has work. DB coverage now also proves paused metrics are + counted as paused but not queued or active work, and explicit planned + maintenance ticks do not start or advance them while the pause is in effect. + Runtime coverage proves the same pause boundary holds for the graph-metric + maintenance runtime: a paused background metric produces an idle runtime tick + with no build started, and an already-started durable planned build is not + claimed, advanced, or published by split worker/coordinator runtime ticks + while paused. Both paths resume and publish through the same runtime once + unpaused. `runUntilIdle` can now exercise planned graph + metric maintenance behind an internal DB-open gate with explicit worker, + round, metric, and page budgets for small bounded background PageRank builds; + budget exhaustion through that path is fail-fast, leaves active planned work + visible in pending stats, and can resume to completion when the budget is + raised. The internal `auto` gate now chooses the planned path for already-active + planned work plus queued degree, PageRank, eigenvector, and compatible HITS + authority/hub pairs under bounded scheduler caps. HITS hub work is counted as + part of the authority-owned pair lifecycle, not as a second independent + metric. Incompatible HITS pairs and explicit operator caps remain outside + planned startup and can fall back to local maintenance. Coverage now proves + queued degree, PageRank, eigenvector, compatible HITS, and multi-metric graph + indexes enter planned maintenance by default, tiny budgets leave resumable + active work, and per-index caps defer extra queued work until capacity returns. + The default idle path uses the same bounded `auto` gate, so scheduler-style + work goes through the planned primitive without adding graph-metric-specific + CI or Make targets. A first internal graph-metric + maintenance runtime now wraps the planned-maintenance primitive with the same + start/stop/notify shape as the other DB maintenance runtimes and can drain a + background PageRank build through repeated bounded `runOnce` ticks. DB-open + coverage now also proves an enabled graph-metric runtime starts automatically, + receives derived-apply notifications, drains a background degree build without + manual runtime ticks, publishes a fresh generation, and reports durable + progress plus owner identity through runtime stats. Separate started + coordinator-role and worker-role runtime loops can also publish a background + degree build through durable job/page state without manual tick calls, and a + started worker-pool runtime can do the same with two configured worker IDs + while a separate coordinator loop owns barriers and publish. The worker-pool + automatic-loop proof now also covers background PageRank, eigenvector, and + paired HITS builds, so the iterative reference metric, reusable single-vector + metric, and paired-vector metric exercise the same split coordinator plus + worker-pool ownership shape. This + proves the first automatic split-owner and multi-worker orchestration boundary + before real remote processes are introduced. The + planned-maintenance primitive and runtime can also cycle an internal set of + worker IDs within one bounded round; coverage proves two runtime worker IDs + complete separate planned scan pages for the same background graph metric, + including reopened-handle runtime paths where each worker-pool, coordinator, + and reader tick comes from a fresh DB handle for partitioned degree, + iterative PageRank, eigenvector, and paired HITS builds. The + runtime now also exposes split coordinator-only and worker-only ticks, and + coverage proves a worker tick can complete a page without advancing the build + phase until a coordinator tick runs. The runtime can now be configured as a + combined coordinator/worker loop, coordinator-only loop, worker-only loop, or + worker-pool loop, so automatic maintenance ticks can be assigned the same + roles as the explicit split entrypoints. Runtime worker/coordinator sweeps now + carry the runtime clock into durable page claiming and exhausted-page checks, + and graph-level coverage proves injected time controls whether a leased page + is still owned by the prior worker or can be reclaimed and completed by a new + worker. Reopened-handle runtime coverage now + proves a background PageRank build can be started, worked, coordinated, + published, cleaned up, and queried through separate fresh DB handles that + instantiate the runtime per role and communicate through durable graph-index + job/page state. The runtime can now also opt into durable runtime-owner + leases, using the same ownership helper as other DB maintenance loops. + Coordinator and combined owners are role-scoped, while worker and worker-pool + owners are scoped by configured worker identity so unrelated workers do not + serialize each other. Worker-pool identity uses an order-independent worker-id + set fingerprint, with unit coverage proving reordered worker-id sets produce + the same runtime owner lease key while different sets do not, and duplicate + worker identities in one pool are rejected by planned-maintenance, runtime + initialization, and command parsing. Manual worker ticks on + lease-owned runtimes are bound to the configured worker identity or worker-pool + set, coordinator-role owners cannot execute worker pages, and worker-role + owners cannot execute coordinator ticks. Coverage proves a second owner for + the same coordinator role, same worker identity, or same worker-pool identity + set is refused while the lease is active, can take over after expiry, and the + former owner observes lease loss without doing graph-metric work, while + independent coordinator and worker owners plus independent worker identities + can hold separate runtime leases and continue the same build together. Split + runtime coverage now also proves distinct lease-owned worker runtimes can + complete separate active pages for one build without serializing on the worker + role, while live duplicate owners for the same worker identity or worker-pool + identity set are fenced before they can claim or complete durable pages. + Coverage also proves a replacement owner for the same worker identity can + take over after the runtime lease expires during an active build, complete a + durable page through the same worker-identity scope, and make the former owner + observe lease loss instead of doing more graph-metric work. The + runtime now also records internal tick telemetry for role, separate runtime + and owner identity hashes, hashed runtime lease scope, configured worker + identity hash/count, runtime lease + ownership/acquisition/loss counters, started/completed ticks, + durable-progress ticks, idle ticks, error ticks, last error name, cumulative + planned-sweep totals, and last planned sweep result, with coverage proving + progress, idle, recovered-error, role, owner accounting, durable lease + takeover, and cumulative coordinator/worker/page/publish counters. That + telemetry is now also surfaced through internal DB stats/runtime-status + snapshots so operations can observe scheduler health and distinguish + coordinator/worker owners without exposing public job resources. Direct + runtime-owner lease cleanup coverage now also verifies that clean runtime + shutdown removes the durable lease record, while stale-owner shutdown after + takeover preserves the replacement owner's lease record. PageRank-style scan + pages now use the same attempt-scoped output adoption pattern as contribution + pages and degree scan pages: partial out-degree/node output is written under + the current page attempt, is invisible to phase aggregation until that attempt + completes, and stale workers cannot adopt or mutate abandoned scan output + after another worker reclaims the lease. Reclaimed PageRank, eigenvector, and + HITS scan-page tests prove abandoned scan-attempt output remains isolated + until the replacement attempt completes and publishes its own scan partials. + Broader remote production multi-worker orchestration remains future work. + +### Phase 3: More Graph Metrics + +Once the framework can reliably materialize PageRank, add more metrics through +the same config, status, storage, and query paths. + +Candidate metrics: + +- `eigenvector`: useful for undirected or strongly connected influence graphs, + but needs careful handling for disconnected and reducible graphs. +- `hits_authority`: useful for citation/search-link authority scoring. +- `hits_hub`: useful for identifying connector or curator nodes. +- `personalized_pagerank`: useful for tenant-, user-, or seed-specific ranking, + but it changes storage cardinality and should not be the second metric. +- `degree`: cheap baseline metric, useful for testing and fallback ordering. + +Current implementation progress: + +- `degree` uses the shared graph metric config, dirty state, generationed score + storage, top-k reads, status, projection, ordering, filtering, and edge + filtering paths. +- `eigenvector` uses the same local generationed materialization and query + framework with fixed-iteration power iteration over the resolved edge scope. + It publishes bounded non-converged output with `converged: false`, matching + the framework contract. +- `hits_authority` and `hits_hub` use the same local generationed + materialization and query framework with fixed-iteration HITS updates over the + resolved edge scope. When a compatible authority/hub pair is configured with + the same scope and iteration settings, local maintenance computes once and + publishes both generations atomically. +- Distributed jobs and personalized PageRank remain future work. + +Do not add a metric until it has: + +- documented graph assumptions +- convergence behavior +- failure behavior for disconnected/sink-heavy graphs +- storage key namespace +- status metadata +- direct top-k tests +- traversal projection/order tests + +For Eigenvector centrality specifically: + +- support fixed-iteration publish with `converged: false` +- expose normalization mode +- document behavior on disconnected components +- consider returning per-component scores or using the largest component only + only if the API names that behavior explicitly + +Recommended order: + +1. `degree` +2. `eigenvector` +3. `hits_authority` and `hits_hub` +4. `personalized_pagerank` + +`degree` is the right second metric because it is cheap, deterministic, and +does not require iterative convergence. It exercises the generic storage, +status, top-k, projection, ordering, and edge-filter paths without adding +algorithmic complexity. + +Eigenvector centrality should come after the framework has a second non-PageRank +metric. It needs explicit documentation for disconnected graphs, reducible +graphs, normalization, and non-convergence. It should reuse the PageRank +iteration/job machinery where possible, but its API must not imply PageRank +damping semantics. + +HITS should be added as a paired metric family. Authority and hub scores are +separate named metrics, but they should be computed by one job when they share +the same edge scope. The publish step should either publish both scores for a +generation or neither. + +Personalized PageRank is intentionally later because it multiplies score +cardinality by seed, tenant, user, or profile. It likely needs an additional +scope key: + +```text +graph_metric_score::::: -> f64 +``` + +Do not add personalized metrics until retention, deletion, and quota controls +exist for high-cardinality metric families. + +### Phase 4: Edge Scope and Metric Families + +The first edge filter supports all edges or typed include lists. Long-term, +metric scope should become richer but still compile to explicit graph-index +state, not arbitrary runtime scanning. + +Add scoped metric families: + +```json +{ + "metrics": { + "pagerank_all": { + "kind": "pagerank", + "edge_filter": { "mode": "all" } + }, + "pagerank_citations": { + "kind": "pagerank", + "edge_filter": { "types": ["cites", "references"] } + } + } +} +``` + +Future edge filters may include: + +- labels +- edge metadata equality/range predicates +- source/target label filters +- tenant or namespace filters + +Constraints: + +- Filter config must be stable and serialized in index metadata. +- Filter changes create a new metric materialization generation. +- Filter predicates should be validated against graph-index edge metadata + definitions where possible. +- Arbitrary predicate execution should wait until graph edge metadata has a + typed expression/predicate contract. + +Implementation milestones: + +- Store resolved edge scope in metric metadata for each generation. +- Validate metric filter config against graph edge metadata. +- Support named metric families with shared kind and different edge scopes. +- Treat edge-filter changes as materialization changes requiring rebuild. +- Add explain/status output that shows the resolved edge scope. + +Metric families should be named explicitly by users. Avoid implicit metric names +such as `pagerank_mentions_cites` generated from filters; stable names are +important for query APIs, status, retention, and migrations. + +Current implementation progress: + +- Published graph metric generations now persist their resolved edge filter as + generation metadata. Status prefers the published generation's stored edge + scope when available and uses the current config only when stored scope is not + present. This keeps status/explainable scope tied to the scores that were + actually published, even if a future config change marks the metric stale or + requires rebuild. +- When a metric's current edge filter no longer matches the published + generation's stored edge filter, status reports the metric as `stale` and + queued for rebuild while still showing the scope of the currently published + scores. +- Published generations also persist a materialization config fingerprint for + algorithm-affecting settings such as metric kind, damping, tolerance, + max-iterations, and edge scope. When the current config fingerprint differs + from the published generation, status marks the metric `stale` even if the + graph edge generation itself has not changed. +- Graph metric edge-scope validation now rejects empty typed scopes, blank edge + type names, duplicate edge types, and `all` scopes that also carry explicit + types. Unknown edge types are rejected when graph edge type metadata is + available. + +### Phase 5: Operational Controls + +Add operational controls only after the default maintenance loop is reliable. + +Useful controls: + +- manual refresh endpoint +- pause/resume metric maintenance +- per-metric max build concurrency +- retention controls for debug/admin users +- rebuild-from-scratch endpoint +- metric deletion with generation cleanup +- progress/event log for failed builds + +Retention should remain latest-only by default. Optional future retention: + +```json +{ + "retention": { + "mode": "count", + "retained_generations": 2 + } +} +``` + +Operational safety rules: + +- A manual refresh should not block writes. +- A rebuild should not delete the prior published generation until publish. +- Metric deletion must remove dirty, published, metadata, score, and job keys. +- Failed builds preserve the last published generation and keep status + queryable. + +Implementation milestones: + +- Add manual refresh and rebuild endpoints. +- Add pause/resume for background maintenance. +- Add metric deletion with full key cleanup. +- Add admin-only retention controls. +- Add per-metric build concurrency and queue visibility. +- Add structured event logs for build failures and publish decisions. + +Operational APIs should be idempotent. Repeating a manual refresh, delete, or +pause request should be safe and should return the current metric state. + +Expose retention only as an admin/debug feature. Latest-only remains the default +because most users need current scores, not historical centrality snapshots. + +Current implementation progress: + +- Local graph indexes expose a metric materialization deletion primitive that + removes dirty, published, metadata, and score keys for a configured metric. + A durable disabled marker prevents background maintenance from immediately + recreating operator-deleted data; refresh, rebuild, or resume explicitly + re-enables the configured metric and rebuilds it from durable graph edges. +- DB and index-manager layers expose local manual refresh and rebuild + primitives for a named graph metric. Manual refresh runs the configured metric + regardless of background refresh mode; rebuild clears materialized state first + and then recomputes from durable graph edges. +- DB and index-manager layers expose local pause/resume primitives for + background graph metric maintenance. Paused metrics remain queryable from the + last published generation, status reports `maintenance_paused`, and manual + refresh can still publish a fresh generation while background maintenance is + paused. +- The public table HTTP layer exposes graph metric operational actions at + `POST /tables/{table}/indexes/{index}/graph-metrics/{metric}:{action}` for + `refresh`, `rebuild`, `delete`, `pause`, and `resume`. Each action is + idempotent at the API contract level and returns the updated graph metric + status. `delete` clears materialized metric state, reports `disabled`, and + suppresses automatic maintenance while leaving the configured metric + available for a later refresh, rebuild, or resume. +- The modular OpenAPI source specs and regenerated Zig public/client contract + modules now describe graph query metric projection, ordering, filtering, + freshness, result status metadata, and the graph metric operational action + route with a typed status response. +- Graph metric status now includes forward-compatible build observability + fields: `phase`, `target_edge_generation`, and `progress`. Local metric + maintenance reports `complete` with `progress: 1.0` for the current published + target and `idle` with `progress: 0.0` when a metric is not ready or has a + pending target generation. These fields are part of the public JSON response, + graph query metric status, and generated OpenAPI client/server contract. +- Graph metric materializations now record durable structured events for local + publish, failed build, delete, pause, and resume decisions. Status exposes both + `last_event` and a bounded newest-first `recent_events` history with monotonic + sequence, kind, timestamp, target edge generation, published generation, and + score count. Durable event keys are pruned to the same retained event window, + keeping local operational history latest-only by default. These fields are + included in public JSON and generated OpenAPI client/server types. This is + the first local event-log primitive; future distributed jobs can extend it + into a paginated history without changing the status shape. +- Failed local metric builds now record a `failed` event, surface `failed` + status for the current target generation, persist consecutive `retry_count` + and `last_error` details, and preserve the prior published generation and + queryable scores. A later successful publish clears the failure details while + retaining the event history. +- Local metric builds now acquire a durable build lease before running. Status + exposes whether a build is queued, the queued generation, the active building + generation, the durable build job id, the active build iteration, the + persisted worker id, persisted phase, and the build lease expiration. + Overlapping local builds fail fast with `GraphMetricBuildAlreadyRunning`, + which gives the future distributed job system a compatible queue/lease status + shape without changing the public status API later. The lease metadata remains + backward-compatible with older local lease records that only stored generation + and expiration, and with the first versioned local lease records that did not + carry iteration or job id. Active status progress now derives from the + persisted build phase and iteration: initial compute reports low progress, + iterative compute advances by iteration/max-iterations, publishing reports + near-complete progress, and complete reports `1.0`. Local PageRank, + eigenvector, HITS, and degree maintenance update that persisted + phase/iteration state while computing and before publishing, providing the + same observable shape that distributed resumable workers will use. Active + leases survive graph-index reopen with their job id, start timestamp, phase, + worker, and iteration intact, while expired leases are ignored by status and + can be reclaimed by a later build. The public OpenAPI graph metric status + schema and hand-written JSON status encoder now include `build_job_id` and + `build_started_at_ms`, and remote response parsing plus deterministic shard + merge preserve matching active build job ids and start timestamps. DB graph + query conversion now deep-copies active build worker ids when cloning graph + metric status out of temporary graph query results, so active lease status + remains valid after the temporary result is released. + +### Phase 6: Planner and Retrieval Composition + +After graph query integration is stable, allow graph metrics to participate in +broader retrieval planning. + +Examples: + +- blend semantic score and PageRank in reranking +- use PageRank as a tie-breaker for graph-expanded retrieval +- use authority scores as a feature in result pruning +- materialize graph metrics into explain/profile output + +Planner constraints: + +- Metric score usage must be explicit in the request or a documented retrieval + profile. +- `fresh` metric requirements should affect sync targets and maintenance waits. +- Query profiles should show metric generation and freshness state. +- Cross-shard score ordering must either use a globally comparable score + generation or fail closed. + +Implementation milestones: + +- Add graph metric score features to query explain/profile output. +- Add explicit rerank expressions that can combine lexical, vector, relational, + and graph metric scores. +- Add freshness-aware planning for requests that require `fresh` metrics. +- Add deterministic cross-shard metric top-k merge behavior. +- Add public tests for metric-assisted retrieval with stale and fresh modes. + +This phase should not make PageRank a hidden default ranking factor. Any metric +use that changes result ordering must be visible in the request, index profile, +or explain output. + +Current implementation progress: + +- Public query profiles now include `graph_metrics` entries for direct graph + metric top-k reads and graph-query metric status when graph queries request + metric status. Each profile entry records the query name, source, graph index, + metric name, effective freshness mode, and the observed graph metric status, + including published generation, target edge generation, state, and progress. +- The distributed merge layer contains a deterministic, fail-closed contract + for a future globally coordinated metric materialization. It groups direct + results by requested query name, validates index, metric, configuration + fingerprint, edge scope, and publication metadata, sorts scores by score + descending with node-key tie-breaking, and applies `top_k` only after merge. + `metric_freshness: "fresh"` is enforced again at fan-in. Focused merge coverage + also + verifies paired HITS authority/hub results preserve the last compatible + published generation when a shard reports a failed rebuild, while `fresh` + fails closed. Hosted cross-range coverage now proves compatible published + shard generations merge through the table-read source and that unpublished or + incompatible shard generations are rejected before ranking. The hosted + table-read gate now also includes a nonuniform eight-shard degree/PageRank/ + eigenvector layout so every shard contributes a published metric score through + the real cross-range reader. Four of those hosted shards are also advanced + into a newer building generation with unpublished high-score targets; + `published` fan-in still merges only the prior target scores and `fresh` fails + closed for all three single-vector-style metrics. Separate hosted active-stale + coverage still exercises direct top-k, traversal + projection/order/filter, and search rerank while `fresh` fails closed. The + hosted compatible HITS authority/hub fixture now also uses an eight-shard + layout with four active/stale shards, covering direct, traversal + projection/order/filter, and authority rerank merge behavior with prior-pair + preservation, `fresh` rejection, remote HITS generation/metadata/edge-filter + mismatch rejection, and missing-status rejection. + This narrows the gap between focused merge tests and promotion-scale shard + evidence, but it does not make independently normalized shard-local scores + globally comparable. +- Until a coordinator builds and publishes one table-wide metric snapshot, + production table reads fail closed before fan-out when a request uses direct + metric top-k, metric-assisted reranking, or graph metric projection, ordering, + or filtering across more than one shard. Traversal without metric scores + remains supported. This gate prevents numerically compatible-looking local + generations from silently producing incorrect global rankings. The internal + merge contract remains tested for the future coordinated implementation. +- Ordinary search requests can now opt into explicit graph metric score + blending with `graph_metric_rerank`. The first implementation validates that + the requested graph metric has a published generation, honors + `metric_freshness: "fresh"` with `MetricStale`, and computes + `final_score = base_weight * existing_score + weight * metric_score`, where + `missing_score` supplies the metric feature value for hits missing from the + published metric generation. Hits are sorted by final score with document id + tie-breaking. This is intentionally visible in the request rather than a + hidden PageRank boost, and it provides the first metric-assisted retrieval + test surface for stale, fresh, and missing-score modes. Profile output now + includes a `source: "graph_metric_rerank"` entry with the observed metric + status, and distributed fan-in fails closed if reranked shard scores do not + report the same nonzero published metric generation, if any shard omits or + reports an unpublished rerank metric status, or if a fresh rerank request + receives a stale/building/failed shard status. Hits reranked by this path now + include `_score_details.graph_metric_rerank` with the base score, + base weight, metric score, metric weight, missing-score fallback flag, final + score, and published metric generation used for that hit. Richer expression + support remains future work. Search rerank coverage now also proves an active + planned rebuild cannot leak building output: published rerank uses the prior + generation and reports building status, while fresh rerank fails closed. The + table-read shard serializer now carries representable single-metric + `graph_metric` reads and `graph_metric_rerank` requests so hosted remote + shards do not silently lose those public read surfaces; multi-metric HITS + traversal fan-in now has focused hosted coverage, while HITS rerank remains a + merge-contract gate until a public multi-metric rerank wire shape is + introduced. + +### Phase 7: Compatibility and Migration + +Long-term graph metric APIs should stay compatible with the PageRank-first +surface: + +- existing `graph_metric` top-k reads continue to work +- `metric_status` remains a map keyed by metric name +- `published` remains the default freshness mode +- `fresh` continues to use `MetricNotReady` and `MetricStale` +- fixed-iteration non-converged results continue to publish unless a metric + explicitly opts out with documented behavior + +If API names change, keep aliases for at least one compatibility window. For +example, if `metric_freshness` later becomes a nested object, continue accepting +the string form: + +```json +{ + "metric_freshness": "fresh" +} +``` + +Implementation milestones: + +- Version graph metric metadata schemas. +- Keep compatibility aliases for renamed API fields. +- Add migration code for metric config shape changes. +- Add validation that refuses ambiguous metric definitions. +- Document deprecation windows for generated clients. + +Migration should never require recomputing scores synchronously during startup. +If a schema migration invalidates a metric generation, mark the metric stale or +not ready, preserve enough metadata for diagnosis, and let maintenance rebuild +asynchronously. + +Current implementation progress: + +- Published graph metric metadata now carries an explicit schema version. + Newly written materializations encode version `3`. Public status responses + expose `metadata_version` while generated clients treat the field as optional + so shard responses remain parseable when a metric has not published yet. +- Graph metric config validation now rejects empty and duplicate metric names + before an index opens. This keeps status maps, direct metric reads, and future + migration code from having to resolve ambiguous metric definitions. + +### Remaining Roadmap: Distributed and Resumable Metric Execution + +The remaining graph-index work is mostly about replacing the local whole-metric +runner with a durable coordinator that can execute the same materialization +lifecycle across pages and workers. The public API should not change materially: +users still configure named metrics on a graph index, read published +generations, request `published` or `fresh` freshness, and inspect status. + +The implementation should continue in small production-hardening slices, but +the center of gravity has moved from proving the planned runner to hardening it. +Degree, PageRank, eigenvector, and HITS now exercise the shared planned +lifecycle. The remaining work should make that lifecycle production-distributed: +real worker ownership, broader restart/failure coverage, attempt-scoped output +adoption where recompute is too expensive, paired-vector hardening for HITS, +and rollout gates that prove query freshness and cleanup behavior before the +planned runner becomes the default. + +Design principle for the rest of the work: user interface stability comes first. +The graph metric lives with the graph index, the user names and configures the +metric there, and all job/page/attempt mechanics remain internal. The only +public effects of distributed execution should be better scalability, richer +status, explicit failures, and the same generation/freshness contract that local +PageRank already exposes. + +The roadmap should now be read as a production-execution plan rather than a +PageRank-only feature plan. Degree is the cheap lifecycle proof, PageRank is the +dynamic-iteration proof, eigenvector is the reusable single-vector proof, and +HITS is the paired-vector proof. Each remaining milestone should preserve the +same external graph-index metric API while moving more work behind durable +coordinator/worker boundaries. + +Roadmap north star: + +```text +named graph-index metric config + -> dirty marker and target edge generation + -> coordinator-owned durable build + -> deterministic manifest pages + -> worker-owned page leases and cursors + -> coordinator-owned phase/iteration barriers + -> verified publish of one complete generation + -> published generation pointer + -> resumable cleanup +``` + +The rest of the work is organized into four release tracks: + +| Track | Purpose | First promotion gate | +| --- | --- | --- | +| Default planned maintenance | Make scheduler-style graph metric work use the bounded planned-maintenance primitive without changing public query behavior. | `runUntilIdle` and background scheduler ticks can finish degree, PageRank, eigenvector, and compatible HITS under latency-safe budgets while unsupported workloads remain outside planned execution. | +| Production worker ownership | Move from in-process role-configured runtimes to real coordinator and worker owners that communicate only through durable graph-index state. | A remote worker can die after claiming a page, another worker can reclaim it after expiry, and duplicate coordinators cannot double-publish. | +| Failure, HITS, and cleanup hardening | Close restart/reclaim/publish/cleanup coverage for PageRank, eigenvector, and paired HITS, including explicit hub phases. | Failed or abandoned rebuilds preserve the prior published generation or compatible HITS pair, and cleanup resumes after restart without unbounded storage growth. | +| Public-read and per-family promotion | Promote the planned runner by metric family only after every score-bearing query path proves the same freshness contract. | Direct top-k, traversal, search rerank, explain/profile, and cross-shard fan-in either read compatible published generations or fail closed with `MetricNotReady`/`MetricStale`. | + +Concrete ordering: + +1. Use the bounded planned-maintenance `auto` gate as the default scheduler + path for degree, PageRank, eigenvector, compatible HITS pairs, and + multi-metric graph indexes. Keep incompatible HITS and explicit operator + caps on the local fallback path, and use telemetry to tune production-safe + idle budgets. +2. Treat the current role-configured graph-metric runtime as the compatibility + harness for the remote-worker contract. Its `combined`, `coordinator`, + `worker`, and `worker_pool` roles should map directly to future process + owners, but the durable job/page records remain the only communication + boundary. +3. Harden the restart/failure matrix while worker ownership is still easy to + test locally. PageRank stays the baseline matrix; eigenvector catches up as + the single-vector iterative case; HITS catches up by applying the same + restart, reclaim, failure, and cleanup matrix to its explicit authority and + hub phases. +4. Harden the explicit HITS hub contribution and hub reduce/norm phases before + enabling remote HITS builds by default. Authority and hub stay separate named + metric scores, but publish, failure, and freshness remain paired for + compatible configs. +5. Keep v1 cleanup aggressive: latest published generation plus bounded failed + diagnostics. Retention, manual retry, pause/resume, and priority scheduling + are future admin/debug controls, not query semantics. +6. Promote per metric family. Degree and PageRank go first, eigenvector follows + after restart/failure parity, and HITS promotes last after paired-vector + partitioning and paired failure coverage. + +#### Canonical Rest-of-Work Plan + +The remaining work is not another public metric API. It is the productionization +of the graph-index-owned metric executor. The current implementation has enough +durable job, page, runtime, scheduler, and status machinery to use the planned +runner as the compatibility harness. The rest of the roadmap should move one +ownership boundary at a time until coordinator and worker processes can safely +operate through durable graph-index state only. + +The user interface should stay fixed through that transition: + +- Users configure named metrics on the graph index. +- Reads choose `published` or `fresh`. +- `MetricNotReady` means no complete generation exists. +- `MetricStale` means a complete generation exists, but not for the current edge + generation requested by `fresh`. +- Fixed-iteration non-converged PageRank, eigenvector, and compatible HITS + publish by default with `converged: false` and final iteration metadata. +- V1 cleanup keeps only the latest published generation plus bounded failure + diagnostics; retention is a future bounded admin/debug option. +- Jobs, pages, attempts, leases, worker identity, retry limits, and backoff + remain internal. Status may summarize them, but users should not manage them. + +Design the rest of the implementation around five internal boundaries: + +| Boundary | Owns | Must not own | +| --- | --- | --- | +| Graph index | Metric config, edge-scope fingerprint, dirty marker, published pointer, public score metadata, query freshness, and status assembly. | Worker scheduling policy or raw job/page controls. | +| Coordinator | Job start/resume, manifest validation, phase barriers, dynamic iteration planning, verified publish, active-build failure, cleanup scheduling, and duplicate-coordinator fencing. | Page execution or worker-local progress. | +| Worker | One leased page at a time: claim, renew, cursor, deterministic output, complete/fail, and stop. | Job creation, config resolution from callers, phase advancement, publish, or failure of the whole build. | +| Runtime/scheduler | Process role, owner identity, worker identity set, tick budgets, wakeups, telemetry, and bounded maintenance loops. | Metric semantics or public freshness behavior. | +| Cleanup runner | Resumable deletion of old score generations, abandoned attempts, completed job namespaces, manifests, pages, summaries, and bounded diagnostics. | Deleting the current published generation or changing query semantics. | + +The production owner model should be explicit before remote workers are enabled: + +- Coordinator ownership is scoped to the graph-metric coordinator role for a DB + or graph-index maintenance domain. Duplicate coordinators are fenced until the + durable lease expires, and a later coordinator can take over after expiry. +- Worker ownership must be scoped by worker identity, not only by the worker + role. Two distinct workers should be able to hold independent runtime-owner + leases and claim different page leases for the same build. A duplicate owner + for the same worker identity should be fenced until expiry; after expiry, a + replacement owner should be able to continue active page work through the same + worker-identity scope while the old owner records lease loss. +- Worker-pool ownership should be keyed by the configured worker identity set in + an order-independent way, or by one stable pool identity. The important + invariant is that unrelated worker owners do not serialize each other, while + duplicate owners for the same worker identity are detectable. Duplicate + worker identities inside one configured pool are rejected at the command and + runtime/scheduler validation boundaries instead of silently running the same + worker twice. Coordinator and single-worker command roles also reject worker + identity lists, and enabled runtime config validation enforces the same + boundary so role argv and in-process owner config stay scoped to the owner + type. +- Process owners must be launched with positive runtime lease TTL and positive + maintenance budgets for rounds, metrics, and pages. A zero TTL or zero budget + is rejected by command/supervisor parsing and by enabled runtime config + validation; it is not a valid idle process state. +- Runtime role gates are independent from durable lease ownership. Disabling + the lease fence can be useful for local compatibility paths, but it must not + let a coordinator execute worker pages, a worker execute coordinator steps, or + a worker role use an unconfigured worker identity. +- Page leases remain separate from runtime-owner leases. Runtime leases fence + process ownership; page leases fence page execution and are reclaimed by + expiry/attempt policy. + +Current runtime coverage now exercises that process boundary through DB-open +configuration, not only through manually constructed runtimes: + +- `OpenOptions.graph_metric_maintenance` can initialize a graph-metric runtime + as `combined`, `coordinator`, `worker`, or `worker_pool`. +- `start_background_loop = false` lets tests and future process harnesses open a + DB handle with the production runtime configuration but drive one explicit + `runOnceDetailed` tick instead of starting an in-process background loop. +- `antfly graph-metric-maintenance` is now the first server-side process + entrypoint for those roles. It opens one DB path with + `OpenOptions.graph_metric_maintenance`, runs explicit bounded ticks as + `combined`, `coordinator`, `worker`, or `worker_pool`, and emits JSON runtime + stats so separate owners can run without passing metric configs to workers. + The command supports bounded polling with `--tick-ms` and + `--max-idle-ticks`, so supervised role processes can stay alive across idle + rounds and still terminate deterministically in tests. It also has a first + `supervise` mode that launches coordinator and worker-pool child role + processes with shared DB path, owner identities, worker identity set, bounded + tick policy, idle policy, and a bounded restart policy; child stdout is + captured so the supervisor can parse per-role durable progress and emit one + JSON summary with child exit/output sizes. Command-level coverage now also + asserts that production-style launched child argv stays scoped to DB path, + role, owner identity, worker identity, lease timing, tick budgets, idle + policy, and page/metric budgets; it does not pass metric names, index names, + target generations, job/page ids, metric configs, or the direct file-backed + writer-lock guard used only by summary-file child launches. Command summary + coverage also pins the stable operations telemetry emitted by role processes: + role, runtime hash, owner hash, worker identity/count, lease-key hash, + acquisition/takeover/lost-lease counters, tick progress, idle/error counts, + and last error. The supervisor/launcher aggregate summary now preserves a + compact version of that child telemetry for each role, and supervised-degree + coverage asserts coordinator and worker-pool owner/worker/lease/tick/error + fields survive aggregation after the build drains to fresh. The spawned + process harness now also validates the same stable role, runtime/owner hash, + worker hash/count, lease-key, tick, and error telemetry on every standalone + coordinator, worker, and worker-pool role process it uses for restart, + lease-fencing, publish, cleanup, and reclaim proofs. Those standalone role + summaries must include durable-progress, idle, and error tick counters, and + completed ticks must be explained by progress, idle, error, lease contention, + or lease loss. They must also report either a runtime-owner lease acquisition + or an explicit lease-acquisition failure, so a role summary cannot silently + omit ownership accounting. The harness rejects any non-null `last_error_name` before + accepting the typed summary. Its final release-gate summary emits required + and observed coverage counts for every remote-owner category, including + separate service multi-page coordinator-takeover and worker-pool-takeover + counts plus explicit service worker-phase, coordinator-phase, and takeover + phase-proof floors, before setting `remote_owner_release_gate: true`, so + release tooling can audit the process gate from one JSON event. The same raw-field guard now + runs over both + standalone role summaries and aggregate supervisor/launcher summaries, + recursively rejecting operational JSON fields such as metric/index names, + target generations, job/page ids, attempt namespaces, storage paths/prefixes, + metric configs, process ids, and local writer details. The same harness now preflights + those standalone and killable role-owner argv vectors through a strict + allow-list, with positive and negative harness checks rejecting metric names, + index names, target generations, job/page ids, metric configs, summary files, + unknown flags, missing values, and the local file-writer guard at that + boundary. +- Fresh coordinator and worker-pool DB handles can publish partitioned degree, + PageRank, eigenvector, and compatible HITS builds through durable graph-index + state only, with no worker-side metric config and no manual runtime + construction. PageRank is the reference proof that the same process-style + boundary survives dynamic iteration, convergence, publish, and cleanup; HITS + proves the same DB-open owner boundary can preserve paired authority/hub + publication. +- This is still an initial process-orchestration proof, not full distributed + failure coverage. The next step is to run supervised child roles under + crash/restart tests that kill and replace active workers and coordinators + while proving lease expiry, stale-attempt fencing, publish idempotence, and + cleanup resume. + +The remaining stages should land in this order: + +| Stage | What changes | Design details | Exit gate | +| --- | --- | --- | --- | +| 1. Scheduler default readiness | Planned maintenance becomes the scheduler path for bounded graph-metric work. | Use the bounded `auto` gate as the single internal entrypoint for queued build startup, worker page sweeps, coordinator sweeps, pending-work stats, and budget exhaustion. Global/per-index caps keep multi-metric indexes fair, and compatible HITS counts as one paired lifecycle. | `runUntilIdle` and background ticks drain degree, PageRank, eigenvector, compatible HITS, multi-metric indexes, and already-active planned work without unbounded latency or partial visibility. Incompatible HITS and explicit caps remain fallback cases. | +| 2. Runtime ownership hardening | The in-process runtime matches the future remote ownership contract. | Add/keep durable owner leases for coordinator, combined, worker, and worker-pool roles. Worker leases must be worker-identity scoped. Runtime stats should expose role, owner hash, worker identity hash/count, lease acquisition/failure/loss, progress, idle, errors, and last sweep result. | Distinct worker identities can coexist; duplicate same-role coordinators and duplicate same-worker owners are fenced; clean runtime shutdown releases the current owner's durable lease without waiting for TTL; stale shutdown after takeover cannot clear the replacement owner's lease; same-worker replacement takeover after expiry is covered during active page work; split coordinator/worker runtimes publish through durable state only. | +| 3. Remote process orchestration | Real coordinator and worker processes use the same durable boundaries. | Coordinators tick only build startup/barriers/publish/failure/cleanup. Workers tick only page claims and execution by index/metric name. No worker receives metric config from a caller. Communication is durable graph-index job/page state plus leases. | Killing a worker abandons only its page/runtime lease; a replacement for the same worker identity takes over after expiry without duplicating page output; duplicate/racing coordinators cannot double-publish or append incompatible phases. | +| 4. Failure and restart matrix | PageRank becomes the reference distributed failure matrix; eigenvector and HITS catch up. | Cover reopen, expired lease, reclaimed output, same-worker cursor resume, failed page retry, exhausted attempts, publish verifier failure, cleanup resume, and prior-generation preservation across prepare, scan, initialize, contribution, reduce, convergence, publish, and cleanup. | Failed or abandoned rebuilds preserve the prior published generation or compatible HITS pair. Every output-writing phase either resumes from cursor, recomputes safely, or adopts attempt-scoped output only after completion. | +| 5. Attempt-output policy | Attempt namespaces are used only where correctness or cost needs them. | Deterministic recompute remains the default. Use attempt-scoped output for contribution-like phases and any reduce-like phase whose partial output can be consumed too early, cannot be atomically replaced cheaply, or is too expensive to recompute at production scale. | Later phases read only adopted output. Abandoned attempts are invisible to readers and removed by cleanup. Reduce-phase adoption is added only when tests prove stale-output or recompute risk. | +| 6. HITS paired-vector hardening | HITS becomes production-ready as the paired-vector proof. | Authority and hub stay separate named metric scores, but compatible configs share one target generation, convergence decision, failure decision, and atomic publish. Hub contribution and hub reduce/norm remain explicit retryable phases, and local lifecycle coverage now proves reclaim, failed-page/exhausted-attempt handling, publish failure, cleanup resume, larger-manifest restart across reopen boundaries, parity, and stale-read behavior preserve the previous compatible pair. | Remote HITS builds are not enabled by default until promotion-scale fan-in, deployment-scale owner evidence, operations evidence, and larger-graph latency all preserve the previous compatible pair. | +| 7. Public read and fan-in gates | Every score-bearing query path proves the same freshness contract. | Direct top-k, traversal projection/order/filter, graph search rerank, explain/profile, and cross-shard fan-in resolve scores only through complete published generations. Active job output, failed attempts, and abandoned generations stay unqueryable. | `published` reads use the latest complete generation during queued/building/failed rebuilds. `fresh` fails with `MetricNotReady` or `MetricStale`. Cross-shard direct metric and graph-search fan-in reject unsolicited or unrequested score/status surfaces, extra unrequested graph metric status names, missing, zero, stale, duplicate/ambiguous graph requests, projected metric requests, graph order metric requests, graph-search node/hit result ownership, or score-node ownership, malformed graph-search traversal/path-shape/node/hit metric payloads or internally inconsistent rerank score details, including mismatched request weights, missing-score semantics, or final-score formula, non-finite traversal distances, path weights, metric/rerank scores, or status numbers, out-of-range progress, mismatched index/metric/status identity, incompatible metadata version or edge filter, invalid state/generation combinations such as `fresh` status pointing at a newer current or target edge generation than the published scores, or incompatible published generations when comparability is required. | +| 8. Cleanup and operations | Cleanup becomes a production invariant, not best-effort housekeeping. | V1 aggressively removes completed job namespaces, abandoned attempts, unpublished scores, manifests, pages, summaries, and old generations when snapshot-safe. Pause/resume/manual retry/priority/retention remain future admin controls. | Cleanup resumes after restart, never hides the current published generation, bounds diagnostics, and prevents unbounded storage growth without requiring a public retention knob. | +| 9. Per-family promotion | Planned/distributed execution becomes default by metric family. | Keep local runners as deterministic CI/debug oracles. Promote degree and PageRank first, eigenvector after single-vector restart/failure parity, and HITS last after paired-vector production coverage. | Each family has local-vs-planned parity, remote-worker coverage, restart/failure coverage, cleanup coverage, public freshness coverage, cross-shard coverage, and operations docs before default promotion. | + +The stages have a strict dependency shape: + +- Scheduler default readiness can promote before real remote processes, but it + must keep conservative gates and resumable budgets. +- Remote process orchestration depends on the runtime ownership model matching + the future process model: coordinator ownership, worker identity ownership, + and page leases must be independently fenced. +- The failure matrix depends on attempt-aware page execution. Progress, + completion, failure, and any output write that becomes visible to a later + phase must validate the durable page attempt, not just worker identity. +- Attempt-scoped output adoption is not mandatory for every phase. The + mandatory invariant is that stale attempts cannot publish, adopt, or write + visible output after the page lease has been reclaimed. +- Public read promotion depends on the failure matrix and cleanup guarantees, + because readers must never need to understand building namespaces, abandoned + attempts, or retained failed-job state. + +The transaction rule for the rest of the implementation is: when a page writes +output that is already public or can be consumed by a later phase, the write +must be guarded by the same durable page-attempt validation that completes or +adopts the page output. If the phase cannot make that write-and-complete path +atomic, it should write to an attempt namespace and adopt only after completion. +This keeps same-worker replacement owners, expired lease recovery, and racing +remote processes from leaking stale page output. + +Concrete next slices: + +1. Promote the DB-open configured runtime path to the canonical integration + test harness. New ownership tests should configure roles through + `OpenOptions.graph_metric_maintenance`, usually with + `start_background_loop = false`, and tick through the runtime attached to the + DB handle. Direct `GraphMetricRuntime.init` tests should remain only for + narrow runtime-unit behavior. +2. Use `antfly graph-metric-maintenance` as the first real process harness for + one coordinator owner and two worker owners against the same DB path. The + command passes only role, owner identity, worker identity set, tick budget, + idle policy, poll interval, and DB path; metric config stays in the graph + index. The first target is degree because it proves remote ownership without + iterative math. +3. Add crash/restart tests around the process harness before broadening the + metric families: worker killed while holding a page, replacement after + runtime lease expiry, stale old worker trying to complete after reclaim, + duplicate coordinator startup, and coordinator restart after workers finish + pages but before publish. +4. Move PageRank onto the same remote harness after degree passes. The PageRank + target is one dynamic later-iteration build where scan, initialize, + contribution, reduce, convergence, publish, and cleanup can all survive at + least one reopened owner boundary. +5. Extend the same harness to eigenvector once PageRank is stable. Eigenvector + should not add new ownership mechanics; it should only prove the + single-vector iterative executor can reuse the PageRank failure matrix. +6. Extend HITS last. HITS remote promotion requires explicit authority and hub + contribution/reduce phase coverage, paired compatible publish, paired + failure preservation, and cleanup restart evidence before default execution + is enabled. +7. Only after remote ownership and public-read coverage pass should the + conservative `auto` gate widen for larger PageRank/eigenvector workloads, + default HITS, incompatible HITS, or multi-metric indexes. + +This sequence intentionally leaves local runners in place until the planned path +has release-quality evidence. Degree proves the non-iterative lifecycle, +PageRank proves dynamic iteration and convergence, eigenvector proves the +single-vector substrate is generic, and HITS proves paired-vector publish and +failure. New graph metrics should be added only after those boundaries are +stable enough that a metric contributes math and metadata, not a new job system. + +#### Remaining Product Contract + +The user-facing model should stay small even as the implementation becomes +distributed: + +- Metrics are configured on the graph index by stable user-provided names. +- `published` reads use the latest complete generation and may report stale, + building, or failed status alongside those scores. +- `fresh` reads require the current edge generation and fail closed with + `MetricNotReady` before the first publish or `MetricStale` after newer graph + writes. +- Fixed-iteration non-converged PageRank, eigenvector, and HITS publish by + default with `converged: false`, completed iteration counts, and final delta + metadata. +- Old generations and intermediate job state are cleaned aggressively in v1; + future retention is a bounded admin/debug option, not query semantics. +- Retry policy, lease timing, worker IDs, page IDs, and attempt namespaces stay + internal, surfaced only through status and diagnostic events. + +#### Roadmap for the Remaining Work + +Current baseline: graph metrics already live on the graph index, use named +metric configs, publish complete generations, report explicit freshness errors, +and can run through durable planned maintenance. Degree, PageRank, eigenvector, +and HITS all exercise the shared planned lifecycle. The new in-process graph +metric runtime gives the maintenance loop a start/stop/notify boundary, can +start automatically from DB open, receives derived-apply notifications, can +cycle internal worker IDs, exposes split coordinator-only and worker-only ticks, +and can run automatic ticks in explicit `combined`, `coordinator`, `worker`, or +`worker_pool` roles. Separate started coordinator and worker runtime loops now +publish a background degree generation without manual tick calls, and a started +coordinator plus worker-pool runtime can publish a partitioned degree build +through two configured worker IDs. Reopened-handle worker-pool coverage also +proves fresh coordinator, worker-pool, and reader handles can advance the same +partitioned degree build through durable state. That is the right staging point +for separating ownership, but it is not the final distributed execution model. + +The remaining roadmap should finish the system in this order. This table is the +authoritative completion plan; the longer sections below expand the same stages +with current implementation status and test coverage. + +| Stage | User/API contract | Internal implementation | Promotion gate | +| --- | --- | --- | --- | +| 0. Keep metrics index-owned | Users configure named metrics inside the graph index. They choose `published` or `fresh`, read scores, and inspect metric status. They never create PageRank jobs, leases, attempts, pages, or cleanup tasks. | Keep job records, manifests, leases, page ids, attempts, retries, and cleanup namespaces internal to the graph index. Status/events may expose summarized progress and errors, but not raw key ranges or operational tuning. | No new public job API is added. OpenAPI/docs describe named metric config, freshness, convergence, failures, cleanup defaults, and status only. | +| 1. Promote planned maintenance safely | Existing reads and writes behave the same. `runUntilIdle` and background schedulers may make bounded graph-metric progress, but unfinished planned work is resumable maintenance, not a user-visible failure. | Route scheduler-style work through one bounded planned-maintenance primitive. Keep the conservative `auto` gate: planned mode is allowed for already-active planned work and safe small cases, while broader larger-graph and multi-metric workloads remain gated until latency is proven. | Degree, PageRank, eigenvector, and paired HITS can drain through planned maintenance with tiny-budget resume coverage; default promotion waits for latency-safe budgets and query/read latency evidence. | +| 2. Turn the runtime into production orchestration | Users still see graph-index metric status, not worker controls. Operational status can show role, owner, phase, page progress, cursor presence, and last error summaries. | Replace same-process runtime ownership with real coordinator and worker processes. Coordinators start/resume jobs, advance barriers, append dynamic iteration pages, publish, fail, and schedule cleanup. Workers only claim pages, renew leases, persist cursors, write page output, complete/fail pages, and stop. | Independent worker processes complete one generation through durable graph-index state only; killing a worker abandons only its lease; another worker reclaims after expiry; duplicate coordinators cannot double-publish or append incompatible phases. | +| 3. Close the restart/failure matrix | `published` keeps returning the latest complete generation during queued, building, or failed rebuilds. `fresh` fails closed with `MetricNotReady` before first publish and `MetricStale` after newer graph writes. | Treat PageRank as the baseline matrix, then bring eigenvector and HITS to the same standard across prepare, scan, initialize, contribution, reduce, convergence, publish, and cleanup. Cover reopen, lease expiry, reclaimed output, failed pages, exhausted attempts, publish failure, cleanup resume, and prior-generation preservation. | Failed or abandoned builds never hide the latest published generation or compatible HITS pair. Every durable-output phase either resumes from cursor, safely recomputes, or adopts attempt-scoped output only after page completion. | +| 4. Harden HITS paired-vector partitioning | Authority and hub remain separate named metric scores, but they publish, fail, and report compatibility as one pair for a target generation. There is no separate HITS job API. | HITS now has explicit retryable phases for authority contribution, authority reduce/norm, hub contribution, hub reduce/norm, paired convergence, paired publish, and paired cleanup. Lifecycle-gated local coverage proves reclaim, exhausted attempts, publish failure, cleanup resume, larger-manifest restart across reopen boundaries, paired failure preservation, paired publish idempotence, and local/planned parity. The remaining work is production remote-worker evidence, promotion-scale fan-in, operations evidence, and latency data before large remote HITS builds are enabled. | Authority and hub pages are independently retryable, but publish and failure stay atomic for the compatible pair. Either side failing preserves the previous pair. | +| 5. Harden every public read gate | Building output is invisible through direct metric top-k, traversal projection/order/filter, search rerank, explain/profile output, and cross-shard fan-in. | Resolve all score-bearing query paths through published generation pointers and compatibility checks. Failed attempt output, abandoned generations, and active job namespaces remain unqueryable. Cross-shard merge validates compatible nonzero published generations before combining scores. Unit-test lifecycle coverage now also runs the fast-root query/profile/fan-in checks for direct metric top-k, traversal status/order/filter, rerank details, failed status preservation, paired HITS failed-status preservation, malformed shard payload rejection, status-generation comparability, and profile generation reporting. Unit-test fan-in coverage combines those fast merge/profile checks with hosted cross-range graph metric fan-in coverage for compatible published shard merges, unpublished and incompatible shard rejection, a nonuniform eight-shard hosted degree/PageRank/eigenvector direct merge layout with four active/stale shards that keep unpublished high-score targets invisible and make `fresh` fail closed, active-stale hosted degree/PageRank/eigenvector traversal projection/order/filter and search rerank published merge plus fresh rejection, compatible HITS authority/hub hosted merge over an eight-shard layout with four active/stale shards and prior-pair preservation across direct, traversal projection/order/filter, and authority rerank surfaces plus fresh rejection, remote HITS generation/metadata/edge-filter mismatch rejection, and missing remote HITS status rejection. | Public e2e coverage proves `published`, `fresh`, `MetricNotReady`, and `MetricStale` behavior for direct reads, graph traversal/search integration, status, and distributed fan-in. Full promotion-scale shard layouts stay a separate release gate. | +| 6. Finish cleanup and operational defaults | V1 keeps aggressive cleanup semantics: latest published generation is retained; completed, failed, abandoned, and unpublished build state is removed when snapshot-safe. Future retention is a bounded admin/debug option, not query behavior. | Make cleanup a resumable phase for old score generations, completed job namespaces, abandoned attempts, manifests, pages, summaries, and bounded failed diagnostics. Keep lease timing, backoff, max attempts, pause/resume, priority, and manual retry as internal/admin concerns until production behavior is stable. | Cleanup resumes after restart, storage does not grow without bound, diagnostics remain bounded, and no retention knob is required for correctness. | +| 7. Promote by metric family | Public metric behavior stays stable while the executor changes behind internal gates. | Keep local runners as deterministic CI/debug oracles. Promote degree and PageRank first, eigenvector after single-vector restart/failure parity, and HITS last after paired partitioning and paired failure coverage. | Each family has local-vs-planned parity, restart, cleanup, freshness, cross-shard, operations, and remote-worker coverage before planned/distributed execution becomes the default. | + +The intended implementation shape is: + +```text +graph index metric config + -> dirty marker and target edge generation + -> coordinator-owned durable build job + -> deterministic manifest pages + -> worker-owned page leases and cursors + -> attempt-scoped intermediate output where needed + -> coordinator-owned phase and iteration barriers + -> verified publish of one complete generation + -> published generation pointer + -> resumable cleanup +``` + +Operationally, the next big design transition is from "one DB process can tick +coordinator and worker work, optionally in explicit runtime roles" to "separate +process owners can safely tick those roles." The durable state model should +remain the communication boundary. Workers should not receive metric configs +from callers, should not mutate public generation pointers, and should not +decide that a build has failed. They should only execute bounded page work for a +metric name and persist enough progress for another worker or coordinator to +continue. + +#### Current Completion Plan + +The remaining work should be planned from the current implementation state, not +from the original PageRank-only design. The graph index already has durable +jobs, manifests, pages, page attempts, planned maintenance, split +worker/coordinator calls, role-configured in-process runtimes, conservative +`auto` scheduling, local-vs-planned parity for the first metric families, and +first coverage for duplicate coordinators, same-worker replacement, reclaimed +attempts, and public stale-read behavior. What remains is production +distribution and promotion. + +The next roadmap should therefore optimize for these outcomes: + +1. Convert in-process runtime roles into real remote owners. + + The current `combined`, `coordinator`, `worker`, and `worker_pool` runtime + roles are the compatibility harness. The production version should preserve + the same responsibilities, but the owners should be independently running + processes that communicate only through durable graph-index state. + + Design requirements: + + - coordinators start or resume jobs, advance barriers, append dynamic + iteration pages, publish, fail active builds, and schedule cleanup + - workers claim page leases, renew, persist cursors, write attempt-fenced + output, complete or fail pages, and stop + - workers resolve work by index name and metric name, never by caller-passed + metric config + - page leases, runtime owner leases, and active build leases remain separate + durability concerns + - duplicate coordinators may observe completed work, but must not duplicate + publish events or append incompatible pages + + Exit gate: a coordinator process and two worker processes can finish one + degree or PageRank generation using only durable state; killing one worker + abandons only its leased page; another worker reclaims after expiry; public + reads continue to see either the old published generation or the verified new + generation. + +2. Promote planned maintenance with bounded fairness. + + The default idle path now uses the bounded `auto` gate for scheduler-style + graph metric work. Keep local runners as oracles and fallback for + incompatible HITS and explicit caps, but do not split the executor into + one-off graph-metric release targets. + + Promotion sequence: + + - already-active planned builds + - queued degree + - queued PageRank + - queued eigenvector + - compatible HITS authority/hub pairs counted as one lifecycle + - multi-metric indexes under global and per-index active-build caps + - explicit iteration caps and incompatible HITS remain local fallback + - broader production latency evidence before removing fallback/oracle usage + + Exit gate: `runUntilIdle` and background scheduler ticks return bounded, + resumable progress for every promoted class; unsupported classes still fall + back to local maintenance or remain explicitly gated without changing query + semantics. + +3. Finish the distributed failure matrix. + + PageRank remains the reference because it exercises scan, initialize, + dynamic contribution/reduce/check iterations, convergence, fixed-iteration + non-converged publish, verified publish, stale reads, and cleanup. + Eigenvector should match the same single-vector matrix. HITS should match it + for paired authority/hub phases. + + Required failure cases: + + - reopen during every phase and at a later dynamic iteration + - expired page lease reclaimed by another worker + - same-worker replacement after runtime lease expiry + - stale prior attempt tries to write progress, output, completion, or failure + - failed page retry and exhausted page attempts + - publish verifier failure after durable score output exists + - crash after publish before cleanup + - cleanup cursor resume after restart + - failed rebuild preserves the prior generation or compatible HITS pair + + Exit gate: every output-writing phase either resumes from a durable cursor, + recomputes safely from the page range, or writes to an attempt namespace that + is adopted only after attempt-validated page completion. + +4. Keep attempt adoption targeted. + + Deterministic recompute should remain the default because it is easier to + reason about and cheaper to operate for small phases. Attempt namespaces are + an executor capability for phases whose partial output can be consumed by a + later phase, whose output cannot be cheaply replaced atomically, or whose + production-scale recompute cost is too high. + + Current policy: + + - contribution-like pages use attempt-scoped output and adoption + - HITS hub raw contribution uses attempt-scoped output and adoption + - scan/initialize/reduce/convergence writes validate the durable page + attempt before writing job-visible output + - future reduce-like phases should move to attempt adoption only when tests + or workload data show stale-output or recompute risk + + Exit gate: later phases read only adopted output or attempt-validated durable + output; abandoned attempts are invisible to readers and cleanup removes them. + +5. Finish HITS as the paired-vector proof. + + HITS should remain two user-visible named metrics, authority and hub, but one + compatible pair for target generation, convergence, publish, failure, and + freshness. It should not introduce a separate HITS job API. + + Remaining HITS work: + + - broaden restart/reclaim/failure/cleanup coverage across explicit authority + contribution, authority reduce/norm, hub contribution, hub reduce/norm, + paired convergence, paired publish, and paired cleanup + - prove remote workers can independently retry authority and hub pages while + the coordinator keeps paired publish/failure atomic + - keep active authority/hub job output invisible to direct top-k and graph + reads until paired publish + - preserve the prior compatible pair after either side fails + + Exit gate: remote HITS builds are enabled only after paired-vector + partitioning, stale-read coverage, failure preservation, cleanup resume, and + local-vs-planned parity all match PageRank-quality evidence. + +6. Gate promotion on public reads, not only executor tests. + + The planned runner is not production-ready until every score-bearing query + path proves the same generation contract. Building output, abandoned + attempts, and failed unpublished generations must stay unqueryable. + + Required public gates: + + - direct graph metric top-k + - traversal projection, ordering, and filtering + - search rerank + - explain/profile/status output + - cross-shard direct metric fan-in + - cross-shard rerank or score-bearing merge + + Exit gate: `published` reads use the latest complete generation during + queued, building, cleanup, and failed rebuilds; `fresh` fails closed with + `MetricNotReady` before first publish and `MetricStale` after newer graph + writes; cross-shard paths prove compatible nonzero published generations or + fail closed. + +7. Make cleanup and diagnostics production invariants. + + V1 should keep aggressive cleanup: retain the latest published generation, + delete completed job namespaces, delete unpublished score generations, + delete abandoned attempts, and retain only bounded failed-job diagnostics. + Retention, manual retry, priority, and extended history can be future + bounded admin/debug features. + + Exit gate: cleanup resumes after restart, never deletes the current + published generation, and prevents completed, failed, abandoned, and + unpublished job state from growing without bound. + +8. Promote by metric family. + + Keep local runners as deterministic CI/debug oracles until each family has + enough planned/distributed release evidence. + + Promotion order: + + 1. degree, because it proves the non-iterative lifecycle + 2. PageRank, because it proves dynamic iteration and convergence + 3. eigenvector, after single-vector restart/failure parity with PageRank + 4. HITS, after paired-vector partitioning and paired failure coverage + + Exit gate: each promoted family has local-vs-planned parity, remote-worker + coverage, restart/failure coverage, cleanup coverage, public freshness + coverage, cross-shard coverage, operations documentation, and generated + client fields for status/freshness/convergence/failure metadata. + +#### Completion Roadmap From Current State + +The current checkpoint has enough durable machinery to treat the remaining work +as product hardening and distributed execution, not as a new metric feature. +The roadmap below is the intended path from the in-process planned runtime to a +production graph-metric subsystem. + +User-facing design stays stable through every milestone: + +- metrics live on the graph index as named metric configs +- users choose `published` or `fresh` freshness at read time +- `MetricNotReady` means no complete generation has ever published +- `MetricStale` means a `fresh` read cannot use the latest complete generation +- fixed-iteration non-converged PageRank/eigenvector/HITS output publishes by + default with `converged: false` +- jobs, pages, attempts, leases, worker ids, retry policy, and cleanup queues + stay internal, with only summarized status and diagnostics exposed + +Implementation design should move one ownership boundary at a time: + +| Milestone | User/API surface | Implementation work | Required proof | +| --- | --- | --- | --- | +| M1. Scheduler default readiness | No public API change. Background maintenance may take longer than local execution for some workloads, but visible reads keep the same published/fresh contract. | Promote the bounded planned-maintenance primitive behind latency-safe scheduler budgets. The `auto` gate starts degree, PageRank, eigenvector, compatible HITS, and multi-metric work within global/per-index caps; explicit caps and incompatible HITS stay on the local fallback path. | `runUntilIdle` and background ticks drain degree, PageRank, eigenvector, compatible HITS, and multi-metric indexes; tiny budgets return resumable `budget_exhausted`; deferred queued work starts when capacity returns. | +| M2. Remote ownership boundary | Users still see graph-index metric status, not job submission or worker controls. Operational status can show role, owner hash, phase, iteration, page counts, cursor presence, attempt, and last error. | Replace same-process runtime roles with independently owned coordinator and worker processes. Coordinators start builds, advance barriers, append iteration pages, publish, fail, and schedule cleanup. Workers claim page leases, renew, persist cursors, write deterministic output, complete/fail pages, and stop. | Killing a worker abandons only its page lease; another worker reclaims after expiry; duplicate coordinators cannot publish twice or append incompatible phase pages; no worker needs caller-supplied metric config. | +| M3. Failure and restart matrix | `published` reads keep returning the prior generation during queued, building, and failed rebuilds. `fresh` fails closed until the current edge generation has published. | Finish restart, lease-expiry, reclaimed-output, exhausted-attempt, publish-failure, cleanup-resume, and prior-generation-preservation tests across PageRank first, then eigenvector, then HITS. | Every output-writing phase either resumes from cursor, recomputes safely, or adopts attempt-scoped output only after page completion. Failed builds preserve the previous generation or compatible HITS pair. | +| M4. Attempt-scoped output policy | No user configuration. Attempt storage is an internal correctness and cost-control mechanism. | Keep deterministic recompute as the default. Use attempt namespaces only where partial output can be consumed by later phases, cannot be atomically replaced cheaply, or is too expensive to recompute at production scale. | Later phases read only adopted output; abandoned attempts are invisible to readers; cleanup removes adopted and abandoned attempt namespaces without growing storage unboundedly. | +| M5. HITS paired-vector hardening | Authority and hub remain separate named metrics, but compatible configs publish, fail, and report freshness as one pair. | HITS now has explicit retryable authority contribution, authority reduce/norm, hub contribution, hub reduce/norm, paired convergence, paired publish, and paired cleanup phases. Direct process coverage now also exhausts a killed hub-reduce page attempt sequence, fails the compatible pair once, and proves a duplicate coordinator cannot fail it again. Lifecycle-gated local coverage now covers active prior-pair visibility, paired publish idempotence, initialize/contribution/reduce/hub/convergence reclaim, cleanup resume after reopen, larger-manifest restart across reopen boundaries, failed-build preservation, publish-failure preservation, local/planned parity, and failed public-read preservation. Finish production remote-worker evidence, promotion-scale fan-in, operations evidence, and latency data before remote HITS is enabled by default. | Authority and hub pages can retry independently, while publish/failure remains atomic for the compatible pair and failed work preserves the previous pair. | +| M6. Public read hardening | Direct top-k, traversal projection/order/filter, search rerank, explain/profile, and cross-shard fan-in all expose the same generation/freshness semantics. | Resolve every score-bearing path through published generation pointers, score metadata, and compatibility checks. Keep active job namespaces, abandoned attempts, and unpublished generations unqueryable. Unit-test lifecycle coverage now includes fast-root query/profile/fan-in checks across direct metric, graph search, and rerank score surfaces, and unit-test fan-in coverage pairs them with hosted cross-range graph metric fan-in coverage. | Public e2e coverage proves `published`, `fresh`, `MetricNotReady`, `MetricStale`, building, failed, and cross-shard incompatible-generation behavior. Promotion-scale shard layouts remain separately qualified. | +| M7. Cleanup and operations | V1 keeps aggressive cleanup. Future retention, pause/resume, manual retry, and priority controls are bounded admin/debug features, not query semantics. | Make cleanup a resumable phase for old score generations, completed jobs, failed/abandoned jobs, attempts, manifests, pages, summaries, and bounded diagnostics. Keep retry, lease, and backoff defaults internal until operational behavior is stable. | Cleanup resumes after restart, never deletes the current published generation, and prevents unbounded growth without requiring a user-facing retention knob. | +| M8. Per-family promotion | Public metric behavior stays stable while the executor changes behind internal gates. | Promote degree and PageRank first, eigenvector after single-vector restart/failure parity, and HITS last after paired partitioning plus paired failure coverage. Keep local runners as CI/debug oracles until each family has release history. | Each promoted family has local-vs-planned parity, remote-worker, restart, cleanup, freshness, cross-shard, and operations coverage. | + +The implementation layers should stay separated: + +- **Graph index:** owns metric config resolution, dirty state, published + generation pointers, metric status, score metadata, and cleanup eligibility. +- **Coordinator:** owns job creation, manifest validation, phase and iteration + barriers, dynamic page planning, verified publish, failure, and cleanup + scheduling. +- **Worker:** owns only one leased page at a time: claim, renew, cursor, + deterministic output, complete/fail, and stop. +- **Runtime/scheduler:** owns when to tick coordinators and workers, budget + limits, role/owner identity, and telemetry. It does not define metric + semantics. +- **Query layer:** owns freshness checks and published-generation resolution. + It must never inspect building output directly. + +The practical promotion order is: + +1. keep the in-process runtime as the compatibility harness +2. make planned maintenance latency-safe for default scheduler use +3. introduce real coordinator and worker process ownership for degree/PageRank +4. finish the PageRank failure matrix as the reference distributed matrix +5. bring eigenvector to the same single-vector matrix +6. finish phase-specific HITS restart, reclaim, failure, and cleanup coverage + for the explicit authority and hub phases +7. harden public read paths and cross-shard fan-in against building, stale, + failed, and incompatible generations +8. promote per family behind internal gates, with local runners retained as + oracles until planned/distributed execution is mature + +#### Rest-of-Work Roadmap + +The remaining work should be delivered as an execution-system roadmap, not as a +set of separate PageRank, degree, eigenvector, and HITS projects. PageRank +remains the reference metric because it exercises dynamic iteration, convergence, +fixed-iteration publish, stale-read behavior, and cleanup. Degree remains the +cheap distributed-runner proof. Eigenvector proves the single-vector iterative +substrate is generic, and HITS proves paired-vector publish and failure. + +The order should be: + +| Step | Workstream | Design decision | Done when | +| --- | --- | --- | --- | +| 1 | Default planned maintenance readiness | Keep the public API unchanged and move scheduler-style background graph metric work through one bounded planned-maintenance primitive. `runUntilIdle` has an explicit planned mode and an internal `auto` gate that selects planned maintenance for queued degree, PageRank, eigenvector, compatible HITS authority/hub pairs, multi-metric graph indexes, and already-active planned builds under global and per-index active-build caps. Compatible HITS counts as one paired lifecycle; incompatible HITS pairs and explicit operator caps stay outside planned startup and fall back to local maintenance. | Budget exhaustion is resumable, pending-work stats expose queued/active/deferred graph metric work, and PageRank/degree/eigenvector/HITS complete through scheduler ticks without partial visibility or unbounded latency. | +| 2 | Production worker orchestration | Workers are lease executors only: they claim pages, persist cursors, write output, complete/fail pages, and stop. Coordinators own job start, phase barriers, dynamic iteration planning, publish, failure, and cleanup. The first internal DB runtime now gives planned graph-metric maintenance a start/stop/notify worker boundary, can cycle multiple internal worker IDs in one bounded round, exposes split coordinator-only and worker-only ticks, can run automatic ticks as `combined`, `coordinator`, `worker`, or `worker_pool`, has started coordinator plus worker-pool coverage that publishes a partitioned degree build through durable state, and now has a real process launcher proof for direct file-backed DB paths. The direct DB launcher serializes storage writes with an explicit local writer guard; production remote orchestration should replace that local storage boundary with the deployment/service boundary while preserving the same graph-index job/page ownership model. | A real worker process can be killed, restarted, or raced without duplicate publish; another worker can reclaim expired pages; status reports owner, attempt, cursor, phase, iteration, progress, and last error from durable records. | +| 3 | Restart and failure matrix | Treat PageRank, eigenvector, and HITS as one matrix over scan, initialize, contribution, reduce, convergence, publish, and cleanup. PageRank is the baseline; eigenvector and HITS must catch up before promotion. | Reopen/retry/reclaim tests cover every phase boundary and at least one later iterative boundary; failed builds preserve the prior published generation or compatible HITS pair. | +| 4 | Attempt-scoped output adoption | Deterministic recompute stays the default. Use attempt-scoped output only where partial output can be consumed early, where replacement is not atomic enough, or where recompute cost is too high. | Later phases read only adopted output; abandoned attempts are invisible and cleanup-owned; reduce-like phases adopt attempts only where there is a demonstrated need. | +| 5 | HITS paired-vector hardening | HITS already has explicit authority contribution, authority reduce/norm, hub contribution, hub reduce/norm, and paired convergence phases. Before enabling remote HITS builds by default, finish restart/reclaim/failure/cleanup coverage for those phases under paired publish semantics. | Authority and hub remain separate named metrics but share one compatible target generation, one convergence decision, one failure decision, and one atomic publish. | +| 6 | Public freshness and fan-in gates | Promotion is blocked until every public read path proves the same generation contract as direct metric top-k. | Direct top-k, traversal projection/order/filter, search rerank, explain/profile output, and cross-shard fan-in use only complete published generations or fail closed with `MetricNotReady`/`MetricStale`. | +| 7 | Cleanup and operations hardening | V1 cleanup remains aggressive. Completed job namespaces, abandoned attempts, unpublished score generations, pages, manifests, and summaries are removed as soon as snapshot safety allows. | Cleanup can resume after restart, failed diagnostics stay bounded, storage does not grow without bound, and retention remains a future bounded admin/debug option. | +| 8 | Per-family promotion | Promote planned execution by metric family behind internal gates, keeping local runners as CI/debug oracles until the planned path has release history. | Degree and PageRank promote first, eigenvector follows after restart/failure parity, and HITS promotes last after paired partitioning and paired failure coverage. | + +The remaining implementation should land as roadmap milestones with evidence +profiles on the existing test harnesses, not as standalone graph-metric Make +targets or independent release gates: + +1. Scheduler-ready planned maintenance. + + Keep the existing public metric API unchanged and make the bounded planned + maintenance primitive the internal scheduler entrypoint. This slice owns + budget semantics, pending-work hints, runtime stats, and `runUntilIdle` + integration. The planned path can be selected explicitly and by the bounded + `auto` gate. Default CI should run the promotion-shaped scheduler coverage + through the existing full-default/unit integration profiles, while larger + latency and deployment evidence stay in named harness profiles. + + Gate: scheduler ticks can drain degree, PageRank, eigenvector, and paired + HITS through the shared primitive; tiny budgets return a resumable + `budget_exhausted` result; queued and active graph-metric work is visible in + DB stats/runtime status; old local whole-metric maintenance remains an + oracle and fallback. + +2. Production coordinator and worker processes. + + Use the existing metric-name worker/coordinator boundary as the contract for + real process owners. A coordinator process starts or resumes builds, advances + phase barriers, appends later-iteration pages, publishes, fails active + builds, and schedules cleanup. Worker processes claim durable pages, renew + leases, persist cursors, write output, complete or fail the page, and stop. + Workers never receive metric configs from callers and never mutate the + published-generation pointer. + + Gate: independently owned coordinator and worker DB handles can complete one + generation using only durable graph-index job/page state; killing a worker + abandons only its page lease; another worker reclaims it after expiry; + duplicate or racing coordinators cannot double-publish or append + incompatible phases. + +3. Distributed failure matrix. + + PageRank is the reference matrix because it exercises dynamic iteration, + convergence, fixed-iteration publish, stale reads, and cleanup. Bring + eigenvector and HITS to the same matrix instead of adding metric-specific + exceptions. Each metric family needs coverage for reopen, expired leases, + reclaimed output, failed pages, exhausted attempts, publish failure, + cleanup-resume, and prior-generation preservation. + + Gate: every phase that writes durable output either resumes from a cursor, + safely recomputes from its page range, or writes through an attempt namespace + that is adopted only on page completion. Failed or abandoned builds preserve + the latest published score generation or the latest compatible HITS pair. + +4. Attempt-scoped adoption policy. + + Keep deterministic overwrite/recompute as the default. Attempt namespaces + are an executor capability for phases whose partial output can be consumed + by later phases, whose output cannot be atomically replaced cheaply, or whose + recompute cost is too high for production. Attempt adoption should remain + internal job machinery, not user configuration. + + Gate: later phases read only adopted job-scoped output; abandoned attempts + are invisible to readers and removed by cleanup; reduce-like phases adopt + attempt output only when tests demonstrate stale partial output or excessive + recompute risk. + +5. HITS paired-vector hardening. + + Keep HITS as the paired-vector proof of the generic runner. Hidden global + hub work has been split into explicit retryable phases: authority + contribution, authority reduce/norm, hub contribution, hub reduce/norm, + paired convergence, paired publish, and paired cleanup. Authority and hub + remain separate named metrics, but a compatible pair has one target + generation, one convergence decision, one failure decision, and one atomic + publish decision. + + Gate: authority and hub output can be rebuilt by remote workers without + hidden global materialization, either side failing preserves the previous + compatible pair, and paired cleanup resumes after restart without hiding + published scores. + +6. Public read and fan-in hardening. + + Treat query behavior as part of the distributed execution design. Direct + top-k, traversal projection, traversal ordering, traversal filtering, search + rerank, explain/profile output, and cross-shard fan-in should all resolve + graph metric scores only through complete published generations. Building + output, failed attempt output, and abandoned generations stay invisible. + + Gate: `published` reads work against the last complete generation even while + a newer build is queued, building, or failed; `fresh` fails with + `MetricNotReady` before first publish and `MetricStale` after newer graph + writes; cross-shard merge refuses incompatible or zero published + generations. + +7. Cleanup, diagnostics, and operational defaults. + + V1 cleanup stays aggressive. Keep the latest published generation, delete + completed job namespaces when snapshot safety allows, remove unpublished + score generations, remove abandoned attempt namespaces, and retain only + bounded diagnostics for failed jobs. Pause, resume, priority scheduling, + manual retry, and retention are future admin/debug controls, not v1 query + semantics. + + Gate: cleanup can resume from durable cursors after restart, completed and + failed builds do not grow storage without bound, failed diagnostics remain + bounded, and no public retention knob is required for correctness. + +8. Metric-family promotion. + + Promote the planned/distributed runner by metric family behind internal + gates. Degree and PageRank should promote first because they prove the + non-iterative and iterative baselines. Eigenvector follows after restart and + failure parity with PageRank. HITS promotes last after paired partitioning is + complete. + + Gate: CI compares local and planned output on deterministic graphs, restart + and cleanup coverage exists for each family, public freshness tests cover + building/stale/failed states, cross-shard score-bearing reads prove + generation compatibility, and generated clients document status, + convergence, freshness, and failure fields. + +The system is not complete until large graph metric rebuilds can run through +remote workers, survive worker death and database reopen, publish exactly one +verified generation, keep previous generations visible on failure, clean their +temporary state, and expose only the stable graph-index metric API to users. + +#### Remaining Architecture Roadmap + +| Lane | Target design | What remains | +| --- | --- | --- | +| Scheduler entrypoint | One bounded graph-metric maintenance primitive starts queued builds, ticks workers, ticks coordinators, reports progress, returns budget exhaustion as a resumable result, contributes queued/active/deferred graph-metric hints to pending-work stats, and is now the default `runUntilIdle` path through the bounded `auto` gate for scheduler-style workloads. Explicit planned mode remains available for tests/operators, and incompatible HITS or explicitly capped cases still fall back to local maintenance. Worker-only sweeps now also report budget exhaustion when they consume their page-step budget and the durable build remains active, so split runtimes can expose resumable work without relying on the combined maintenance loop. Worker-pool rounds spend page budget on actual worker steps rather than idle worker IDs. The auto gate now has coverage for default queued degree, PageRank, eigenvector, compatible HITS, multi-metric graph indexes, already-active planned degree/PageRank/eigenvector/HITS builds, explicit iteration caps, incompatible HITS fallback, and per-index deferred queued work. Compatible HITS reports one eligible queued pair and one active lifecycle before planned execution. Controlled-cap tests prove the scheduler starts work only within global/per-index budgets, defers extra queued work, exhausts tiny budgets as resumable active work, and finishes after budget expansion. | Broaden deployment latency evidence and keep default promotion coverage in existing full-default/unit profiles instead of adding graph-metric-specific Make targets. | +| Remote workers | Workers execute page leases only. They do not pass metric configs, create jobs, advance phases, publish, fail builds, or clean completed jobs outside cleanup pages. Worker-page results now carry an explicit `completed_build` terminal signal so drain loops do not have to overload publish telemetry to detect final cleanup. An internal DB maintenance runtime now repeatedly ticks the same planned worker/coordinator sweeps under bounded budgets, starts automatically from DB open when enabled, wakes from derived-apply notifications, can cycle multiple internal worker IDs across page sweeps, exposes split coordinator/worker tick entrypoints, supports role-configured automatic loops for combined/coordinator/worker/worker-pool ownership, proves separate started coordinator-role and worker-role loops can publish a background degree generation without manual tick calls, proves a started coordinator-role plus worker-pool-role pair can publish a partitioned degree build through two configured worker IDs, keeps job/page semantics in durable index state, passes runtime clock time into page-lease claim/reclaim and exhausted-page decisions, can opt into role-scoped durable runtime-owner leases with acquisition/failure/loss/takeover telemetry, and records internal role/owner/tick/progress/error telemetry plus cumulative planned-sweep totals surfaced through DB stats/runtime-status snapshots for operations. Reopened-handle PageRank coverage now proves lease-owned runtime split ticks can be driven by separate fresh coordinator, worker, and reader DB handles, and reopened-handle degree coverage now proves a lease-owned worker-pool runtime role can do the same with two configured worker IDs. Runtime lease coverage now proves one owner blocks a duplicate same-role owner until lease expiry, a later same-role owner can take over through the durable lease, coordinator/worker owners can hold independent role leases for the same build, distinct lease-owned worker runtimes can complete separate active pages under different worker identities, live duplicate worker owners for the same worker identity or worker-pool identity set are fenced before page execution during active builds, and a replacement owner for the same worker identity can take over after lease expiry and continue active page work while the former owner observes lease loss. Worker-pool runtime identity now has a focused order-independence test proving reordered worker-id sets produce the same runtime owner hash/lease key while different sets do not. Duplicate worker IDs in one command-level worker-pool config are rejected before runtime launch, and coordinator/worker command roles plus enabled runtime config validation reject worker-id lists, matching the runtime/scheduler validation boundary and preventing one configured pool from silently running the same worker identity twice. Page execution now fences progress, completion, and failure by both worker identity and durable page attempt, so a stale process from the prior attempt cannot finish work after same-worker replacement takeover. The process command now has both a sequential `supervise` proof and a bounded `launch` proof that starts independently owned coordinator and worker-pool child processes, captures child summaries, uses a local DB writer guard only for direct file-backed launches, emits command-summary ownership telemetry that pins role, owner, worker, lease, progress, and error counters for operators, carries compact per-child telemetry into the aggregate supervisor/launcher summary, and the spawned-process harness now verifies the same role/owner/worker/lease-key/tick/error telemetry on every standalone coordinator, worker, and worker-pool role process used in restart and reclaim proofs. | Add production remote deployment orchestration and tests where independently owned workers communicate only through durable graph-index job/page state or a service boundary, without relying on direct local file-writer serialization. | +| Coordinator | The coordinator is the only owner of build startup, phase barriers, dynamic iteration planning, publish, active-build failure, and cleanup scheduling. Degree, PageRank, eigenvector, and paired HITS now have duplicate coordinator tick coverage around publish/cleanup proving repeated coordinator ticks do not append duplicate publish events or advance incompatible state. PageRank, eigenvector, and paired HITS also have reopened-coordinator publish-race coverage: one fresh coordinator handle publishes, a second fresh coordinator handle observes cleanup or completed paired state, and status still has one publish event for one visible generation or compatible authority/hub pair. The same no-duplicate-publish proof now runs through the DB/index-manager scheduler boundary and DB graph-metric runtime split-coordinator boundary with lease-owned coordinator roles for PageRank, eigenvector, and paired HITS. Scheduler publish telemetry is now coordinator-owned too: coordinator sweeps count publish when they advance `publish_generation`, while worker sweeps keep cleanup completion out of the publish counter. Those runtime tests now cover both live duplicate-owner fencing while the publishing coordinator still holds the runtime lease, and later post-release idempotence when another coordinator owner observes cleanup/completed state. The spawned-process harness now carries the same publish-boundary invariant through real coordinator role processes for degree, PageRank, eigenvector, and compatible HITS: a second coordinator after publish must not publish, fail, advance phase, or append another publish event. It also carries the failed-publish sibling invariant for PageRank, eigenvector, and compatible HITS: after a real coordinator records a publish-verifier failure, a second coordinator must not publish, fail again, advance phase, or append another failure event. | Harden duplicate/racing coordinator coverage under true remote scheduling and carry the invariant into deployment orchestration. | +| Page output | Cheap deterministic phases can recompute on reclaim; expensive or high-risk phases use attempt-scoped output that is adopted only after page completion. Iterative scan partial writes now validate the durable page attempt before writing job-visible out-degree and node partials, and PageRank/eigenvector/HITS initialize validate the durable page attempt before rewriting iteration-0 rank output. PageRank initialize also rewrites aggregate out-degree through the same validation path. PageRank partial convergence summaries now update cursor, unit progress, `max_delta`, `total_delta`, and `rank_sum` in one attempt-validated write path, so stale reclaimed check pages cannot revive abandoned convergence metadata. Degree reduce validates the durable page attempt in the same write path that materializes public score-generation rows and completes the reduce page, so a stale reclaimed attempt cannot write visible degree scores before completion is rejected. PageRank, eigenvector, and HITS authority/hub reduce now apply the same attempt-validated write path to next-iteration rank output consumed by convergence. HITS hub reduce also validates the durable attempt before writing or reusing hub raw summary state. | Extend attempt-scoped adoption to future reduce-like phases only where recompute cost or atomic replacement risk justifies it. | +| HITS paired vectors | Authority and hub share one compatible target generation, paired convergence metadata, one failure decision, and one atomic publish decision. Hub contribution and hub reduce/norm are explicit retryable phases with attempt-scoped hub raw contribution output. | Broaden phase-specific restart, reclaim, failed-page, publish-failure, cleanup, and remote-worker coverage before enabling remote HITS builds by default. | +| Public reads | Direct top-k, traversal, ordering, filtering, search rerank, status, and cross-shard fan-in read only complete published generations or fail closed. | Add public e2e coverage for building, stale, failed, and cross-shard distributed cases before promotion. | +| Cleanup | Cleanup is a resumable phase that removes old score generations, abandoned building output, attempt namespaces, manifests, pages, summaries, and bounded failed diagnostics without hiding the current published generation. | Broaden cleanup restart/reclaim tests under remote ownership and keep retention knobs out of v1. | + +#### Remaining Delivery Plan + +| Order | Milestone | Design | Ship gate | +| --- | --- | --- | --- | +| 1 | Default planned maintenance | Route background graph metric rebuilds through the bounded planned-maintenance primitive instead of the local whole-metric runner. The explicit `runUntilIdle` planned gate proves the path can publish a small background PageRank build through planned maintenance, tiny planned-idle budgets fail fast while preserving resumable active work, and the default bounded auto gate proves queued degree, PageRank, eigenvector, compatible HITS, multi-metric indexes, and already-active planned PageRank/degree/eigenvector/HITS can use planned maintenance. The auto gate exposes an internal decision summary for active, eligible queued, deferred queued, and ineligible queued work. Default and explicit auto tests assert that compatible HITS reports one eligible queued pair before planned execution, active paired HITS reports one active lifecycle, incompatible HITS and explicit iteration caps remain ineligible before fallback, and per-index caps defer extra queued work until the scheduler has capacity. The remaining work is broader deployment latency evidence and release-profile evidence, not another local execution path. The call must make bounded progress and return `budget_exhausted` instead of treating unfinished work as failure. | `runUntilIdle` and scheduler-style ticks finish PageRank, degree, eigenvector, and paired HITS without unbounded latency or partial visibility. | +| 2 | Remote worker ownership | Introduce production worker processes around the existing metric-name worker/coordinator boundary and the new role-configured runtime. Workers claim pages by index and metric name, persist cursors, complete/fail leases, and exit. Coordinators run as their own owners and never rely on worker-local state. | Killing or racing a worker reclaims only that page lease; active status shows owner, attempt, cursor, and error; coordinator remains the only publish authority. | +| 3 | Distributed failure matrix | Expand restart, expired-lease, reclaimed-output, publish-failure, and cleanup tests across PageRank, eigenvector, and HITS. PageRank remains the reference matrix; eigenvector and HITS must match it before promotion. | Failed or abandoned rebuilds preserve prior published scores or the prior compatible HITS pair across reopen. | +| 4 | HITS paired partitioning | Keep authority contribution, authority reduce/norm, hub contribution, hub reduce/norm, and paired convergence as explicit retryable phases. Finish the restart, reclaim, failure, cleanup, parity, and remote-worker matrix around those phases. | Authority and hub work are independently retryable pages, but publish and failure remain atomic for the compatible pair. | +| 5 | Public freshness and fan-in gates | Treat query freshness as part of the execution design. Promotion requires direct top-k, traversal projection/order/filter, search rerank, explain/profile status, and cross-shard fan-in coverage. | Building output is never visible through public reads; `fresh` fails closed; cross-shard score merges prove compatible nonzero published generations or fail closed. | +| 6 | Per-family promotion | Enable planned execution behind internal gates per metric family: degree and PageRank first, eigenvector next, HITS last after paired partitioning. Keep local runners as CI/debug oracles until planned output has release history. | CI has deterministic local-vs-planned parity, restart/failure/cleanup coverage, and public docs/clients describing status, freshness, convergence, and failures. | + +Roadmap state: + +| Area | Current state | Remaining work | +| --- | --- | --- | +| Job root | Durable per-metric build records exist, and the planned worker step now loads the active job before dispatching page work; planned degree, PageRank, eigenvector, and HITS now have active runners, planned PageRank has local alternating-worker coverage over partitioned pages, and planned HITS has local alternating-worker coverage over partitioned paired-vector pages. The planned build startup path now exposes a public `GraphIndex` ensure call that creates or returns the active durable job/manifest for one target generation, the worker path exposes public metric-name worker-only page and coordinator steps so remote runners do not pass metric configs around, and a public active-build failure call records coordinator-owned failure while requiring the active lease/job pair. The DB/index-manager layer now exposes the same named build ensure, worker page step, coordinator step, failure, and drain boundary by index name and metric name; DB-level coverage proves a planned degree generation can publish through those calls without worker-side config access, and the direct DB/index-manager worker/coordinator split-step boundary now also has injected-time `At` variants for deterministic lease/reclaim tests. The DB/index-manager layer also has bounded planned scheduler sweeps: coordinator sweeps start queued background builds and advance active barriers/publish steps, while worker sweeps claim active durable pages by worker id and now report resumable budget exhaustion when their page-step budget is consumed while the build remains active; DB coverage proves active planned degree, PageRank, eigenvector, and HITS builds can complete through these sweeps, with PageRank/eigenvector exercising iterative phase advancement, HITS preserving paired authority/hub publish, and cleanup completing from active durable jobs. Reopened DB-handle coverage now proves an iterative PageRank build can be started, worked, coordinated, published, cleaned up, and queried through fresh handles that communicate through persisted graph-index job/page state rather than in-memory state; PageRank, eigenvector, and paired HITS also have fresh-DB coordinator race coverage at `publish_generation` proving the DB/index-manager boundary records one publish event when a second coordinator handle ticks after publish. A reusable planned-drain primitive now composes those metric-name ensure, worker-page, and coordinator calls with named worker IDs, giving production scheduling a concrete internal loop without exposing metric configs to workers. A bounded planned-maintenance primitive now wraps background planned-build startup plus worker/coordinator sweeps for scheduler callers, DB coverage proves it can drain background PageRank, degree, eigenvector, and paired HITS generations without the local whole-metric runner, and budget exhaustion is now reported as a resumable result flag rather than an error. The first DB graph-metric runtime wraps those scheduler calls with start/stop/notify lifecycle, split tick entrypoints, role-configured automatic loops for combined/coordinator/worker/worker-pool ownership, and DB stats/runtime-status telemetry that includes role plus hashed runtime/worker ownership; runtime-level PageRank/eigenvector/HITS coverage now proves lease-owned split-coordinator runtime instances fence a live duplicate coordinator owner at publish and cannot duplicate publish after one runtime owner moves the job to cleanup or completed paired state. Cross-family public-boundary coverage now proves degree, PageRank, eigenvector, and HITS page workers do not advance phases or publish when driven through public build ensure plus metric-name worker/coordinator calls, and the same public failure boundary preserves the prior published degree, PageRank, eigenvector, or compatible HITS pair while retaining bounded diagnostics. Degree also has reopened-handle coverage proving independently opened coordinator/worker handles communicate through durable job and page state rather than in-memory state, tolerate duplicate coordinator barrier/publish ticks, and complete scan/reduce pages from concurrent reopened worker handles while the coordinator remains the only publish authority. PageRank, eigenvector, and HITS now have the same first concurrent reopened-worker proof across scan, initialize, contribution, reduce, and convergence pages, plus separate reopened-coordinator publish-race coverage proving a second coordinator owner cannot duplicate the publish event after another owner moved the job to cleanup or completed paired state. | Add true remote scheduling, production worker process orchestration, latency-safe default idle promotion to planned maintenance, broader concurrent/distributed failure coverage, and deepen HITS hub contribution/reduction phase partitioning. | +| Manifest | Durable manifests exist with versioned page range metadata; planned degree partitions reverse-edge scan work, reduce work, and cleanup work into deterministic pages, initial PageRank/eigenvector/HITS manifests partition scan, initialize, contribution, reduce, and convergence pages, non-final iterative convergence dynamically plans the next iteration's contribution/reduce/check pages, and dynamic page appends update manifest page counts idempotently. Planned HITS reduce now has multi-page coverage proving one claimed reduce page writes only authority/hub output for its planned node range, and HITS hub raw contribution state is now materialized as durable job-scoped output with fingerprinted summary metadata before rank writes. HITS reduce page fingerprints include the hub raw summary they depended on. Reusing a HITS hub raw summary validates the raw namespace count, norm, and raw fingerprint; stale summaries or raw values are rejected across reopen, and recompute replaces the iteration's raw namespace before writing replacement raw values. | Broaden paired-vector manifest coverage, including partitioned HITS hub contribution/reduction phases and restart/retry cases across dynamically appended HITS pages. | +| Page leases | Explicit page claim plus next-eligible scheduling, renew, reclaim, complete, fail, idempotent completion, progress, page range primitives, bounded retry exhaustion, and a metric/phase page-executor dispatch exist for planned degree; PageRank scan/out-degree, initialize, contribution, reduce, convergence, and cleanup pages now execute through the same path; eigenvector has a single-vector planned executor; and HITS has a first paired-vector planned executor over the same durable page lifecycle. Status now includes a capped active-page summary for the current build phase, including page id, state, range kind, worker id, attempt, lease expiry, cursor, unit progress, and last error, without exposing raw page range bounds; coverage verifies leased, failed, exhausted-attempt coordinator failure, reopened, capped multi-worker page summaries, and two active scan-page leases owned by independently reopened workers while the coordinator refuses to advance the incomplete barrier. Degree now has reopened-handle ownership coverage for a dead worker's active scan-page lease: status reports the abandoned owner and cursor, early cross-worker claim is refused, expiry allows another worker to reclaim with attempt increment and cursor reset, and the reclaimed page can recompute and finish the generation. Degree, PageRank, eigenvector, and HITS worker-only public steps now also tolerate a racing lost lease as no completed page, allowing another worker tick to retry instead of failing the build. Attempt-aware page execution now validates the durable page attempt before progress, completion, failure, or partial-output writes, so same-worker replacement after lease expiry fences stale prior-attempt work even when the worker id string matches. PageRank has restart coverage after dynamic iteration planning, contribution/reduce/convergence cursor resume after reopen on initial and later iterations, later-iteration failed-page retry across contribution/reduce/convergence phases, later-iteration exhausted contribution-page coordinator failure after graph-index reopen and now through spawned killed-worker processes with duplicate coordinator idempotence, same-worker next-page lease renewal, cleanup prefix cursor resume after reopen, reclaimed scan/initialize/contribution/reduce partial-output recompute coverage, reclaimed convergence-page summary reset before recompute, cleanup resume after reopen, public failed planned-build prior-generation preservation, and concurrent reopened-worker coverage across partitioned scan/initialize/contribution/reduce/convergence pages. Planned eigenvector contribution/reduce pages can now resume from durable cursors after reopen, reclaimed initialize/contribution/reduce pages overwrite stale partial output, later-iteration failed contribution/reduce/convergence pages retry through the generic worker step, later-iteration exhausted contribution-page coordinator failure after graph-index reopen and now through spawned killed-worker processes preserves the prior published generation with duplicate coordinator idempotence, cleanup resumes after reopen with the published scores visible, public failed planned rebuilds preserve the prior published generation, and concurrent reopened-worker coverage now proves the same partitioned scan/initialize/contribution/reduce/convergence ownership path. Planned HITS now has contribution/reduce cursor resume after reopen, multi-page reduce output-range isolation coverage, later-iteration failed-page retry coverage across contribution, reduce, explicit hub contribution, explicit hub reduce, and convergence phases through the generic worker step, later-iteration exhausted hub-reduce coordinator failure after graph-index reopen and spawned killed-worker process coverage that fails the compatible authority/hub pair once while preserving the prior published pair and fencing duplicate coordinator failure, reclaimed initialize/contribution/reduce output overwrite coverage for authority/hub job state, reclaimed convergence-summary reset coverage, cleanup cursor resume after reopen with the published pair visible, public failed planned HITS rebuilds fail the compatible authority/hub pair together while preserving the prior published pair, and concurrent reopened-worker coverage proves paired scan/initialize/contribution/reduce/convergence ownership while preserving coordinator-owned paired publish. | Add broader production failure coverage for expired/reclaimed partial output across all phases and add true remote distributed ownership tests. | +| Barriers | Durable phase summaries, multi-page phase barriers, convergence summaries, executable PageRank/eigenvector/HITS check pages, dynamic iterative advancement, resume-after-reopen coverage for a dynamically planned PageRank later iteration, and later-iteration HITS retry advancement into publish readiness exist. | Broaden failure tests around every iteration boundary and add the same restart matrix for eigenvector and paired HITS. | +| Publish | Local atomic generation publish plus planned-job publish verification exist; planned degree publish advances to a cleanup phase after flipping visibility, planned PageRank can publish converged or fixed-iteration output from durable rank state after dynamic iteration advancement, planned eigenvector publishes a verified single score vector from durable rank state, and planned HITS can atomically publish compatible authority/hub generations together. Planned PageRank, eigenvector, and paired HITS now have coordinator-owned publish-verifier failure coverage after graph-index reopen: a corrupted manifest at `publish_generation` fails the active build, removes abandoned build output, records bounded diagnostics, and keeps the prior published generation or compatible authority/hub pair queryable. Planned HITS active-rebuild coverage also proves newly materialized authority/hub job-namespace ranks stay invisible to direct top-k readers until paired publish, and failed planned HITS rebuilds record failure on both compatible metrics while leaving the previous authority/hub score generations queryable. | Add larger restart and cleanup coverage for paired HITS outputs plus true remote publish-race coverage. | +| Cleanup | Local score-generation cleanup exists; planned degree cleanup now uses separate durable prefix pages for degree partials and final job namespace removal; planned PageRank cleanup now uses separate durable pages for out-degree partials, node membership partials, and final job namespace removal, with restart coverage after a non-final cleanup page and durable cursor resume inside a large cleanup prefix. Planned HITS cleanup now has dedicated hub raw, hub raw summary, and HITS rank namespace cleanup pages before final job namespace cleanup, with cursor-resume-after-reopen coverage while the published authority/hub pair remains queryable. Planned degree scan attempts and planned iterative contribution attempts now write under the job namespace, so final cleanup removes abandoned attempt output after adoption. Planned eigenvector cleanup has cursor-resume-after-reopen coverage for final job namespace cleanup while the published scores remain queryable. Failed planned builds now delete unpublished score generations and the job namespace while preserving compact failed job status, public planned-failure coverage verifies that prior published degree, PageRank, eigenvector, and compatible HITS scores remain visible, and recent failed-build diagnostics are retained with a fixed bound. The public failure boundary now distinguishes an active lease/job pair from retained failed-job diagnostics, so a completed or already-failed build cannot be failed again as if it were active. | Extend attempt-scoped adoption to reduce phases only where deterministic recompute is insufficient, and add production distributed worker ownership. | +| Attempt adoption | Planned degree scan plus planned PageRank, eigenvector, and HITS contribution pages now have attempt-scoped partial output. The executor writes partial output into the current page attempt, same-worker resume accumulates in that attempt, page completion adopts the attempt output into the job-scoped namespace, and later reduce phases read only adopted output. Contribution and HITS hub-raw attempt writes now validate the durable page attempt before writing attempt-namespace partials, and adoption validates the durable page attempt in the same batch that copies attempt output into the job-visible namespace, so reclaimed stale attempts cannot write new partials or adopt abandoned output. PageRank scan still uses deterministic job-scoped page partials, but those writes are now fenced by the same durable page-attempt validation. | Apply the same capability to reduce pages if recompute cost or atomic replacement risk requires it. | +| Reduce-output fencing | Planned PageRank, eigenvector, and HITS authority/hub reduce pages now write next-iteration rank rows through durable-attempt validation paths that also record reduce progress or page completion. HITS hub raw summary creation/reuse is fenced by the same hub-reduce page attempt before rank normalization. Coverage proves reclaimed stale reduce claims cannot write additional rank rows or hub summary state after another worker owns the replacement attempt. | Apply the same write-and-complete fencing pattern to future reduce-like phases, or move those phases to attempt adoption if production-scale recompute/atomicity needs it. | +| Degree executor | Planned degree now runs through the generic metric/phase page-executor dispatch: scan pages write attempt-scoped partials, adopt them into job-scoped partials only on page completion, reduce pages materialize final scores from adopted partials, publish stays coordinator-owned, cleanup removes completed job and abandoned attempt keys through durable prefix pages, output matches the local runner, reopened coordinator/worker handles can start a build through public ensure, finish one generation through the public split-step boundary, and a dead worker's leased scan page can be observed, refused before expiry, reclaimed after expiry, recomputed, and completed through durable state. Degree reduce score writes are now guarded by durable page-attempt validation in the same batch that completes the page, and coverage proves a stale reduce claim cannot write the building score generation after another worker reclaims the page. Name-only public worker/coordinator steps now verify two workers can complete distinct scan and reduce pages while coordinator ticks refuse to advance until each phase barrier is complete. Threaded concurrent-handle coverage now proves independently opened workers can contend on scan/reduce pages, lose page ownership without failing the build, retry, and complete a generation without worker-side phase advancement or publish. Reopened-handle status coverage now proves two active scan-page owners are visible from durable page records before the coordinator advances. | Add remote worker orchestration and production ownership coverage beyond local threaded handles. | +| Rollout | Local PageRank, planned PageRank, degree, planned eigenvector, and HITS paths exist; planned degree, planned PageRank, planned eigenvector, and planned HITS have local-vs-planned parity coverage on deterministic graphs, including partitioned planned PageRank pages and partitioned paired HITS pages drained by alternating workers. Degree, PageRank, eigenvector, and HITS now also have coverage for the reusable planned-drain loop reaching a fresh generation through metric-name worker/coordinator calls and cleanup; the HITS case proves the same loop preserves paired authority/hub publish. Degree, PageRank, eigenvector, and HITS now have first threaded concurrent-handle worker ownership coverage, PageRank/eigenvector/HITS have reopened-coordinator no-duplicate-publish coverage, PageRank has first DB-level reopened-handle scheduler coverage through the bounded sweep interface, background PageRank/degree/eigenvector/HITS have explicit planned-maintenance coverage, default and explicit `runUntilIdle` auto mode now cover queued degree, PageRank, eigenvector, compatible HITS, multi-metric indexes, per-index deferred queued work, incompatible HITS fallback, explicit iteration caps, and active planned degree/PageRank/eigenvector/HITS coverage, and PageRank/eigenvector/HITS now have reopened-handle runtime split coverage through separate coordinator, worker, and reader handles plus automatic coordinator-only and worker-only runtime roles, including split-runtime coordinator publish-race proofs. | Broaden parity/restart coverage, add remote distributed worker coverage, collect deployment latency evidence, then gate and promote the distributed runner through existing profiles. | + +Remaining delivery plan: + +| Milestone | Design | Exit criteria | +| --- | --- | --- | +| R1. Production worker ownership | Replace local drain-style execution with a real worker ownership model. Workers claim only durable page leases; the coordinator owns build startup, phase barriers, dynamic iteration planning, publish, failure, and cleanup. The first split now exists as public `GraphIndex` build ensure, metric-name worker-only page, metric-name coordinator, active-build failure, and planned-drain steps: coordinator startup creates or returns the active durable job/manifest for a target generation, worker-only page execution leaves phases and publish untouched until a coordinator step runs, coordinator/worker callers do not need to pass metric configs, the planned-drain loop alternates named workers through the metric-name worker boundary and coordinator boundary, coordinator failure requires the active lease/job pair and preserves the prior published generation or compatible HITS pair, cross-family public-boundary coverage proves degree, PageRank, eigenvector, and HITS workers cannot advance barriers or publish, public failure coverage spans all four metric families, reopened degree coordinator/worker handles can complete a generation through durable state, DB-level PageRank scheduler coverage now reopens a fresh DB handle for every coordinator, worker, and reader tick while completing the same active durable build, and DB-level PageRank/eigenvector/HITS coverage proves two fresh coordinator handles cannot duplicate a publish event or paired publish event, planned-maintenance coverage proves scheduler callers can drain background PageRank, degree, eigenvector, and paired HITS builds through one bounded primitive, tiny-budget planned-maintenance ticks now report exhaustion and resume to completion without treating unfinished work as an error, pending-work stats report queued and active graph-metric planned work for scheduler callers, role-configured runtime loops can run as combined, coordinator-only, worker-only, or worker-pool owners with role/owner identity visible in internal stats, PageRank/eigenvector/HITS split-runtime coordinator race coverage now proves lease-owned runtime coordinator roles cannot duplicate a publish event or compatible paired publish event, scheduler publish counters are emitted from coordinator sweeps rather than worker cleanup sweeps, duplicate degree coordinator ticks are idempotent across barriers and publish, PageRank/eigenvector/HITS reopened-coordinator publish-race coverage proves two separate coordinator owners still leave one publish event and one published generation or compatible pair, active split-runtime coverage proves distinct worker identities can hold independent runtime leases and complete separate active pages, duplicate worker owners for the same worker identity or worker-pool identity set are fenced before claiming page work, same-worker replacement ownership can take over after runtime lease expiry during active page work, a dead worker's scan-page lease can be reclaimed after expiry without losing published visibility, racing degree workers can lose a page lease without failing the build and retry through another worker tick, exhausted page attempts now make the coordinator record a failed active build, and status reports capped active page owner/attempt/cursor/error summaries from durable page records. Lease timeouts, attempt limits, and backoff stay internal defaults until production behavior is stable. | Remote workers can complete one generation without duplicate publish; killing a worker only reclaims its leased page; status reports active distributed page ownership without exposing raw key bytes. | +| R2. Attempt-scoped output adoption | Keep deterministic overwrite as the default retry path, but add an executor capability for phases whose output is too expensive or risky to recompute. Degree scan and iterative contribution pages now use attempt namespaces: partial output remains invisible until page completion adopts it into the job namespace, and final cleanup removes abandoned attempt keys. Reduce phases should adopt the same capability as needed. | Page retries are safe for large contribution/reduce pages; abandoned attempts are cleaned; adoption does not expose partial output to readers; cleanup covers adopted and abandoned attempt namespaces. | +| R3. Iterative restart matrix | Treat PageRank, eigenvector, and HITS as one restart matrix over prepare, scan, initialize, contribution, reduce, convergence, publish, and cleanup. PageRank, eigenvector, and paired HITS now include later-iteration failed-page retry plus exhausted-attempt coordinator failure after graph-index reopen that preserves the prior published generation or compatible authority/hub pair. All three iterative families also have publish-verifier failure coverage after reopen, with the coordinator recording failed build diagnostics instead of leaking an error or publishing partial output. | Restart/reopen tests exist at every phase and iteration boundary; expired/reclaimed pages either resume from cursor or recompute without stale partial output; failed rebuilds preserve the prior published generation or authority/hub pair. | +| R4. Paired-vector completeness | Finish HITS as the paired-vector proof case. Authority and hub work share one target edge generation, compatible manifests, paired convergence metadata, and one atomic publish decision. Hub contribution and hub reduce/norm are now explicit retryable phases, hub raw contribution output is attempt-scoped, and active-rebuild authority/hub ranks stay invisible to direct top-k until paired publish. Lifecycle-gated local coverage now proves reclaim across authority/hub phases, paired publish failure, cleanup resume, larger-manifest restart after repeated reopen boundaries, failed-build preservation, exhausted hub-reduce pair failure, paired publish idempotence, and local/planned parity. The remaining work is true production remote-worker coverage before remote HITS builds are enabled. | HITS authority/hub output remains atomically visible; hub contribution/reduction work is partitioned into retryable pages; paired publish failure, cleanup resume, and stale-read behavior are covered. | +| R5. Query and freshness gates | Keep the public API stable: named graph metrics, `published` freshness for latest complete output, `fresh` freshness for current edge generation, `MetricNotReady` before first publish, and `MetricStale` for stale published generations. Distributed execution must not leak building output into projections, order-by, direct top-k, search rerank, or cross-shard fan-in. | Public e2e coverage proves direct top-k, traversal projection/order/filter, search rerank, status, and cross-shard fan-in behavior for stale, fresh, failed, and building metrics. | +| R6. Cleanup and retention hardening | Keep v1 cleanup aggressive: retain the latest published generation, remove completed job namespaces immediately when snapshot safety allows, delete failed/abandoned build output, and keep only bounded diagnostics. Future retention is an opt-in debug/admin feature, not part of query semantics. | Completed, failed, reclaimed, and abandoned jobs do not grow storage without bound; cleanup can resume after restart without hiding published output; future retention knobs remain bounded and disabled by default. | +| R7. Promotion | Promote the planned runner by metric family behind internal gates. Local runners remain the oracle until parity, restart, cleanup, and public freshness tests pass. Promotion should be per metric kind, with PageRank and degree before eigenvector/HITS remote defaults. | CI compares local and planned output on deterministic graphs; distributed worker tests run before enabling remote workers; public docs and generated clients describe status, freshness, convergence, and failure fields. | + +Rest-of-work design roadmap: + +The remaining implementation should be organized by production tracks rather +than by algorithm. Degree, PageRank, eigenvector, and HITS are different proofs +of the same graph-metric lifecycle, so each track should land as a narrow, +testable slice that keeps the current local runner as the compatibility oracle. + +| Track | What to build | Dependencies | Exit criteria | +| --- | --- | --- | --- | +| T1. Lifecycle hardening | Close the remaining local planned-runner gaps: expired/reclaimed output tests, cleanup restart tests, failed-build preservation, and reduce-phase attempt adoption only where deterministic recompute is not enough. | Existing planned degree/PageRank/eigenvector/HITS executors and local parity tests. | Building output is invisible until publish; page retry either resumes or recomputes safely; completed and failed jobs clean up without hiding the prior published generation. | +| T2. Coordinator contract | Make the coordinator the only component that creates jobs, advances phase barriers, appends dynamic iteration pages, publishes, marks failure, and schedules cleanup. Worker-only page execution must remain unable to publish or advance phases. Planned build startup is now a public `GraphIndex` ensure call that creates or returns the active durable job/manifest for one target generation while keeping raw lease mutation internal, a public metric-name coordinator step resolves the config internally before ticking barriers or publish, and a public failure step records failed active builds only when the active lease matches the retained job. Public-boundary coverage proves degree, PageRank, eigenvector, and HITS workers cannot advance or publish without coordinator ticks, and public failure coverage preserves the prior published degree, PageRank, eigenvector, or compatible HITS pair. Degree also has direct coverage that reopened coordinators can use that public ensure boundary, duplicate coordinator ticks after barrier advancement become no-ops for the new phase until pages complete, coordinator publish moves the durable job to cleanup once, and a stale duplicate publish attempt fails closed rather than appending another publish event. PageRank, eigenvector, and HITS public-worker coverage now extends that publish idempotence proof across iterative and paired metrics: duplicate coordinator ticks after publish stay in cleanup, completion ticks preserve one publish event, and paired HITS keeps authority/hub publish events singular and compatible. | The worker/coordinator split and durable phase/page summaries. | Duplicate coordinator ticks are idempotent; concurrent coordinators cannot publish twice; status is derived from durable job, manifest, page, phase, iteration, failure, and cleanup records. | +| T3. Worker ownership | Replace local drain-style execution with production page ownership. Workers claim pages, renew leases, persist cursors, complete or fail pages, and stop. They do not create jobs, plan phases, pass metric configs, or publish generations. Worker-only and coordinator steps are now explicit metric-name `GraphIndex` calls and are also available through DB/index-manager calls by index name and metric name; bounded DB/index-manager coordinator and worker sweeps can drive active durable builds across graph indexes without per-metric drain calls; a reusable planned-drain primitive composes those calls over named worker IDs; degree, PageRank, eigenvector, and HITS drain coverage proves the primitive can reach a fresh generation and finish cleanup without worker-side publish, with HITS preserving paired authority/hub publish; degree/eigenvector/HITS split-step coverage starts through public build ensure and reaches publish through durable page and job records; DB-level degree coverage proves the same split boundary can publish through the future scheduler-facing layer, DB sweep coverage proves active degree plus iterative PageRank, eigenvector, and paired HITS work can complete through bounded production-style sweeps, DB-level PageRank coverage now proves those sweeps survive a fresh DB handle per worker/coordinator tick, and planned-maintenance coverage proves background PageRank, degree, eigenvector, and paired HITS can be started and drained through the scheduler-facing primitive; tiny-budget planned-maintenance coverage proves budget exhaustion is resumable and non-fatal, and pending-work stats now expose queued/active graph-metric work before, during, and after those budgeted ticks. The first DB graph-metric maintenance runtime now wraps that primitive in a start/stop/notify loop, focused coverage proves repeated bounded runtime ticks can publish a background PageRank build, multi-worker runtime coverage proves one bounded round can cycle two internal worker IDs across separate planned scan pages for one metric, split runtime coverage proves worker-only ticks complete pages without advancing phases until coordinator-only ticks run, automatic-role coverage proves coordinator-only and worker-only runtime loops preserve that same ownership split without manual split calls, started split-role loop coverage proves separate coordinator-role and worker-role background runtimes can publish a background degree generation without manual ticks, started worker-pool loop coverage proves a coordinator-role runtime plus worker-pool-role runtime can publish a partitioned degree generation through two configured worker IDs, reopened-handle runtime coverage proves background PageRank can publish through separate fresh coordinator, worker, and reader handles, reopened-handle worker-pool coverage proves fresh coordinator, worker-pool, and reader handles can publish a partitioned degree generation through two configured worker IDs, runtime clock plumbing now gives worker/coordinator sweeps deterministic lease-reclaim and exhausted-page time, and runtime stats coverage proves progress, idle, last-result, cumulative coordinator/worker/page/publish counters, recovered-error telemetry, and DB stats/runtime-status propagation. Degree has name-only public-step coverage proving separate workers finish distinct scan/reduce pages and cannot advance barriers without coordinator ticks; degree also has reopened expired-lease coverage proving a dead worker's active page remains visible, cannot be stolen early, and can be reclaimed after expiry with attempt/cursor reset; degree now has injected-time worker-step coverage proving a dead worker lease remains owned before expiry and is reclaimed/reset/completed after the injected time passes expiry; degree threaded concurrent-handle coverage proves racing worker handles can complete scan/reduce pages and retry benign lost-lease contention; degree status coverage now proves multiple active reopened-worker page leases are visible before barrier advancement; PageRank, eigenvector, and HITS threaded concurrent-handle coverage now prove the same public worker-only ownership shape across partitioned scan, initialize, contribution, reduce, and convergence phases; and active leased/failed pages are summarized in status from durable page records, including capped multi-worker active-page coverage. True remote worker scheduling, default idle promotion, and broader production failure tests remain. | Coordinator contract and bounded internal retry defaults. | Remote workers can finish one generation; killing a worker only reclaims its leased page; page ownership, attempt, cursor, phase, iteration, and last error are visible in status without exposing raw key ranges. | +| T4. Algorithm substrate | Keep PageRank as the iterative reference, eigenvector as the single-vector proof, and HITS as the paired-vector proof. Shared code owns planning, leases, barriers, convergence summaries, publish verification, status, and cleanup. Metric-specific code owns edge direction, transition math, normalization, convergence criteria, and score metadata. | Lifecycle hardening plus coordinator/worker contracts. | Adding a new centrality metric does not require a new job system, public query API, or status model. | +| T5. HITS paired-vector hardening | HITS now has explicit durable phases for authority contribute, authority reduce/norm, hub contribute, hub reduce/norm, and paired convergence. The next work is to finish phase-specific restart, reclaim, failure, and parity coverage before enabling large remote HITS builds. | Durable HITS paired publish, explicit hub phases, paired local-vs-planned parity, and active-rebuild stale-read coverage. | Authority and hub publish atomically; either side failing preserves the previous pair; hub and authority work have page-level retry, reclaim, progress, cleanup, and stale-output coverage. | +| T6. Public freshness gates | Prove that distributed execution cannot leak building output through direct top-k, traversal projection/order/filter, search rerank, explain/profile output, or cross-shard fan-in. Direct top-k, traversal projection/order/filter, and search rerank now have active planned-rebuild coverage proving published reads stick to the prior generation while reporting building status, and fresh reads fail closed. Direct top-k, traversal projection/order/filter, and search rerank now also have failed planned-rebuild coverage proving published reads preserve the prior generation while reporting failed status. | Verified publish and public query status plumbing. | `published` reads only complete generations; `fresh` fails closed with `MetricNotReady` or `MetricStale`; failed rebuilds preserve the latest published generation or compatible HITS pair. | +| T7. Operations and cleanup | Keep v1 operations conservative: immediate cleanup when snapshot-safe, deferred internal cleanup otherwise, bounded failed-job diagnostics, and internal retry/lease/backoff defaults. Public failure is an active-build operation, not a retained-diagnostics mutation: once failure clears the active lease, repeated failure attempts fail as not active. Pause, resume, retention, manual retry, and priority scheduling are future admin controls. | Cleanup cursors, failed-build diagnostics, and status. | Completed, failed, reclaimed, and abandoned jobs do not grow storage without bound; cleanup can resume after restart; no user-facing retention knob is required for v1 correctness. | +| T8. Promotion | Promote planned execution by metric family behind internal gates. Degree and PageRank should move first, eigenvector next, and HITS last after paired partitioning. | Parity, restart, cleanup, freshness, and multi-worker coverage for each family. | CI compares local and planned output on deterministic graphs; production distributed ownership tests pass before remote workers are enabled; docs and generated clients describe freshness, convergence, status, and failure fields. | + +Recommended sequence: + +1. Finish lifecycle hardening while execution is still local and deterministic. + Add the missing restart/reclaim tests first because they define the durable + invariants the production worker model must preserve. +2. Tighten the coordinator boundary. The coordinator should be an idempotent + state machine over durable summaries, not a helper that trusts in-memory + worker state. +3. Add production worker ownership with internal lease, attempt, and backoff + defaults. Keep those defaults out of the public graph metric config until + operational behavior is stable. +4. Finish the common iterative substrate. PageRank should remain the reference + for dynamic iterations; eigenvector should reach the same restart matrix for + single-vector normalization; HITS should finish the paired-vector phase + split before large remote builds. +5. Gate every public read path against the same freshness semantics. Query + behavior is part of the distributed design, not a later integration detail. +6. Promote per metric kind. Keep local runners as CI/debug oracles until the + planned path has been stable across releases. + +Long-term shape: + +```text +configured graph index metric + -> dirty marker for edge-scope/config fingerprint + -> durable build job + -> deterministic manifest pages + -> worker-owned page leases + -> phase and iteration barriers + -> verified publish transaction + -> published generation pointer + -> resumable cleanup +``` + +The graph metric should continue to live with the graph index. Users should +configure named metrics on the index, not create a separate job resource for +each algorithm. Jobs, attempts, pages, and cleanup are implementation details +that surface only through status and events. `MetricNotReady` remains the +explicit pre-publish failure, `MetricStale` remains the explicit fresh-read +failure, fixed-iteration non-converged PageRank/eigenvector/HITS output should +publish by default with `converged: false`, and v1 should clean old generations +immediately unless snapshot safety requires the internal deferred cleanup queue. + +Detailed remaining design: + +1. Production coordinator and worker ownership. + + Keep phase ownership separate from page ownership. Workers only claim pages + and persist progress; the coordinator owns active-job creation, phase + barriers, dynamic iteration planning, publish, failure, and cleanup. Page + leases should carry `worker_id`, `attempt`, `lease_expires_at_ms`, + `cursor`, `completed_units`, and `last_error`. Coordinator ticks should be + idempotent and should derive the next phase exclusively from durable page and + phase summaries. A duplicate coordinator tick may observe that a transition + already happened, but it must never publish twice or append incompatible + pages. + + The first production scheduler should use internal defaults for lease + timeout, max attempts, and backoff. These values are operational tuning, not + graph metric API. Process/supervisor entrypoints and enabled runtime config + validation should still reject zero runtime lease TTLs and zero + round/metric/page budgets before starting an owner, so production + misconfiguration fails closed instead of creating an idle role that cannot + claim or renew work. Lower-level planned-maintenance calls may still model a + zero-budget tick as resumable budget exhaustion for scheduler tests and + callers. Status should expose enough to debug ownership and progress, but it + should continue to hide raw key ranges. + +2. Attempt-scoped output adoption. + + Deterministic overwrite remains the default retry strategy for small and + cheap phases. A reclaimed page can delete or overwrite the output range for + its own page and recompute from the range start. This is simpler and should + stay the default for PageRank scan, reduce pages, and other phases whose + page output can be atomically overwritten. Degree scan and iterative + contribution pages use attempt-scoped adoption because their partial output + can otherwise become visible to later phase aggregation before page + completion. + + Keep attempt-scoped adoption as an executor capability for large reduce-like + pages and future remote jobs. The page writes to: + + ```text + metric_attempt///////* + ``` + + Completion validates the attempt output, records its fingerprint, and then + atomically adopts it into the job namespace or records an adopted-attempt + pointer that later phases read. Failed or abandoned attempts are invisible to + readers and are cleanup-owned. This keeps partial remote output from leaking + into published scores while avoiding expensive destructive cleanup before + every retry. + +3. Harden HITS paired-vector partitioning. + + HITS should be the proof that the distributed runner can materialize + compatible vector pairs without a new job system. The planned runner now + explicitly separates authority and hub work instead of letting one reduce + page materialize global hub raw state: + + ```text + initialize_pair + authority_contribute(iteration n) + authority_reduce_and_norm(iteration n + 1) + hub_contribute(iteration n + 1 authority) + hub_reduce_and_norm(iteration n + 1) + check_pair_convergence(iteration n, iteration n + 1) + ``` + + Authority contribution pages scan inbound edge ranges and write + target-node authority partials. Authority reduce pages aggregate by target + node range and write normalized authority ranks. Hub contribution pages then + scan reverse-edge ranges, read the normalized authority rank for each target, + and write attempt-scoped source-node hub raw partials that are adopted only + after the page completes. Hub reduce pages aggregate by node range, repair or + write the hub raw summary, and write normalized hub ranks. The convergence + barrier compares authority and hub vectors together and either plans the next + paired iteration or advances the compatible pair to publish. + + Publicly, authority and hub remain named graph metrics that live with the + graph index. They publish together only when configured as a compatible pair, + fail together on invalid output, and preserve the previous published pair on + failure. There should not be separate HITS query APIs. The remaining work is + to expand restart, lease-expiry, reclaimed-output, exhausted-attempt, publish + failure, cleanup-resume, local-vs-planned parity, and true remote-worker + coverage across the explicit HITS phases. + +4. Restart and failure matrix. + + Treat PageRank, eigenvector, and HITS as one matrix over: + + ```text + prepare -> scan -> initialize -> contribution -> reduce -> convergence + -> publish -> cleanup + ``` + + PageRank is the coverage baseline. Eigenvector needs the same restart, + failed-page retry, reclaimed-output, publish-failure, and cleanup coverage + for its single-vector phases. HITS needs the same matrix for the paired + authority/hub phases, with explicit tests that one side cannot publish + without the other and that stale paired output is overwritten or ignored on + retry. + + The required invariants are: + + - `published` reads never observe building output. + - `fresh` reads return `MetricNotReady` before first publish and + `MetricStale` when the published generation does not match the target edge + generation. + - failed rebuilds preserve the previous published generation or pair. + - expired/reclaimed pages either resume from a durable cursor or recompute + from the range start without accumulating stale partial output. + - cleanup can resume after restart without hiding the published generation. + +5. Query and freshness gates. + + The distributed runner is not complete until public query paths prove the + same freshness semantics as direct top-k metric reads. Gate promotion on + coverage for traversal projection, traversal ordering, filters that depend + on metric score, search rerank, explain/profile output, and cross-shard + fan-in. Cross-shard score-bearing results must prove they refer to compatible + nonzero published generations before merging; otherwise they should fail + closed. + +6. Cleanup and retention. + + V1 cleanup should remain aggressive. Keep the latest published generation, + delete completed job namespaces promptly, delete failed or abandoned building + output, and retain only bounded recent diagnostics. Future retention can be + a bounded debug/admin option, but it should not affect query semantics and + should stay disabled by default. + +7. Promotion sequence. + + Promote by metric family rather than flipping all graph metrics at once: + + 1. keep local runners as the oracle + 2. run planned degree and PageRank by default in tests + 3. enable local multi-worker planned builds + 4. enable production worker ownership for degree/PageRank behind a gate + 5. finish eigenvector restart/failure parity + 6. finish HITS paired restart/failure parity across the explicit phases + 7. enable remote/distributed workers for large graph indexes + 8. demote local runners only after deterministic parity, restart, cleanup, + freshness, and cross-shard tests pass + +Recommended implementation order: + +| Order | Slice | Design intent | +| --- | --- | --- | +| 1 | Planned `degree` reduce | Done: independent scan pages are safe because scan writes per-page partials and `reduce_ranks` materializes final scores. | +| 2 | Add real partition planning | In progress: degree scan, reduce, and cleanup pages are partitioned, initial PageRank/eigenvector/HITS phase pages are partitioned, HITS now has an explicit durable phase shape for authority contribution, authority reduce/norm, hub contribution, hub reduce/norm, paired convergence, publish, and cleanup, and hub raw output is produced through attempt-scoped hub-contribution pages before hub-reduce pages normalize it. Later iterative pages are planned dynamically, and dynamic page appends update manifest page counts idempotently. HITS unit-test local coverage now includes active prior-pair visibility, paired publish idempotence, initialize/contribution/reduce/hub/convergence reclaim, larger-manifest resume across repeated reopen boundaries, cleanup resume after reopen, failed-build preservation, publish-failure preservation, and exhausted hub-reduce pair failure. Remaining work is production distributed coverage. | +| 3 | Add resumable cursors | In progress: degree scan pages resume from same-worker cursors, reclaimed scan pages safely recompute, and degree scan now uses an attempt namespace whose output is adopted only when the page completes; PageRank scan, contribution, reduce, convergence, and cleanup pages persist cursor progress, with PageRank, eigenvector, and HITS contribution partials now also written to attempt-scoped keys and adopted only on page completion. Contribution/reduce/convergence resume is verified across reopen on initial pages and dynamically planned later-iteration pages, cleanup prefix deletion resume is verified across reopen, later-iteration failed pages retry through the coordinator, reclaimed scan/initialize/contribution/reduce pages recompute without stale partial output, and reclaimed convergence pages recompute without stale partial summaries. Planned eigenvector now verifies contribution/reduce cursor resume after reopen, reclaimed contribution/reduce stale-output overwrite, later-iteration failed contribution/reduce/convergence page retry, cleanup cursor resume after reopen, and prior-published preservation after a failed planned rebuild. Planned HITS now verifies contribution/reduce cursor resume after reopen, later-iteration failed contribution/reduce/hub-contribution/hub-reduce/convergence pages retry through the generic worker step and advance to publish readiness, later-iteration exhausted hub-reduce attempts fail the compatible pair while preserving the prior published pair, reclaimed contribution/reduce pages overwrite stale durable authority/hub job output, reclaimed convergence pages reset stale partial summaries before recompute, and cleanup cursor progress resumes after reopen with the published pair still visible. Remaining work is broader attempt-scoped adoption for reduce phases where needed, broader expired/reclaimed partial-output coverage across all phases, and production distributed ownership. | +| 4 | Generalize the page executor | In progress: degree, PageRank, eigenvector, and HITS use the generic metric/phase page-executor dispatch. The combined local worker helper now composes a worker-only page step with a coordinator step, the split worker/coordinator calls are exposed on `GraphIndex`, planned build startup has a public ensure call that keeps raw lease mutation internal, metric-name public worker/coordinator calls resolve configs inside the index, public failure records active-build diagnostics through the coordinator boundary, and a planned-drain primitive now composes metric-name ensure, worker-page, and coordinator calls over named worker IDs. PageRank and cross-family split-step coverage now run through public build ensure plus metric-name worker/coordinator calls, public active-build failure coverage now spans degree, PageRank, eigenvector, and HITS, degree has reopened-handle coverage across public build ensure plus worker/coordinator steps, name-only multi-worker page coverage, threaded concurrent-handle scan/reduce ownership coverage, concurrent active-page status coverage, expired-page reclaim, benign lost-lease retry behavior, and duplicate coordinator tick coverage, DB-level PageRank coverage now drives the bounded scheduler sweeps through a fresh DB handle per worker/coordinator/read tick, background PageRank, degree, eigenvector, and paired HITS can now be drained through the explicit planned-maintenance primitive, budget exhaustion is reported as a resumable result instead of an error, the graph-metric runtime can now run automatic ticks in combined/coordinator/worker/worker-pool roles, and status now summarizes active leased/failed pages from durable page records. Remaining work is true remote production worker orchestration, latency-safe default idle promotion, and broader production failure coverage. | +| 5 | Move PageRank onto durable pages | In progress: scan/out-degree pages write durable out-degree and node intermediates, initialize pages write aggregate out-degree plus iteration-0 rank state, contribution pages write attempt-scoped durable contribution partials and adopt them into the reduce-visible namespace only on page completion, reduce pages write iteration `n + 1` ranks, check pages write convergence summaries, non-final checks dynamically plan later iterations, planned publish materializes public score output, local-vs-planned parity is covered on deterministic graphs, dynamically planned later iterations can resume after reopen, contribution/reduce/convergence pages can resume from a durable cursor after reopen on initial and later iterations, failed later-iteration pages can be retried by another worker before publish readiness, and expired scan/initialize/contribution/reduce pages can be reclaimed without stale partial output; remaining work is broader expired/reclaimed coverage and larger cleanup/failure coverage. | +| 6 | Enable multiple workers | In progress: local alternating-worker coverage now proves planned PageRank pages and paired HITS pages can be claimed by independent workers while the coordinator remains the only phase/publish authority. Degree, PageRank, eigenvector, and HITS planned-drain coverage now proves a reusable metric-name worker/coordinator loop can complete a generation and cleanup with multiple worker IDs; HITS additionally proves paired authority/hub publish through that loop. Degree, PageRank, eigenvector, and HITS also have threaded concurrent-handle coverage proving partitioned pages can complete through independent reopened handles with benign lost-lease retries, PageRank now has DB-level fresh-handle scheduler coverage through the bounded sweep interface, background PageRank/degree/eigenvector/HITS have explicit planned-maintenance drain coverage, and tiny-budget planned-maintenance ticks resume to completion. Remaining work is remote distributed worker orchestration, default idle promotion, and failure coverage across all metric families. | +| 7 | Complete distributed cleanup | In progress: planned degree and PageRank now delete metric-specific partial prefixes before final job namespace removal without deleting published output; planned HITS now deletes hub raw, hub raw summary, and HITS rank intermediates through dedicated cleanup pages before final job namespace removal; planned degree final cleanup now also removes abandoned attempt-scoped scan output, and the shared job namespace cleanup removes abandoned iterative contribution attempts for PageRank, eigenvector, and HITS. Planned PageRank cleanup can resume after graph-index reopen with the published generation still queryable, including durable cursor resume within a large prefix cleanup page. Planned eigenvector and HITS cleanup can also resume after reopen while keeping published scores queryable. Failed planned builds now clean unpublished score output and job namespaces while keeping compact failure status and bounded recent failure diagnostics, and public failure coverage verifies that previous published degree, PageRank, eigenvector, and compatible HITS scores remain queryable. Remaining work is reduce-phase attempt adoption where needed and production distributed worker ownership. | +| 8 | Move eigenvector and HITS | In progress: eigenvector now reuses the single-vector iterative executor with local-vs-planned parity plus attempt-scoped contribution output, contribution/reduce cursor-resume, reclaim, later-iteration failed-page retry, later-iteration exhausted-attempt failure preservation, cleanup resume, public failed-rebuild prior-generation preservation coverage, reusable planned-drain coverage, threaded concurrent-handle partitioned worker coverage, and spawned-process coverage that publishes a fresh eigenvector generation through real coordinator/worker-pool child roles, reclaims killed scan, initialize, contribution, reduce, and convergence page owners through real worker role processes, publishes at the coordinator boundary after setup restart, completes cleanup through a separate worker process, and fails a corrupted publish manifest while preserving the prior generation. HITS now has a first paired-vector planned runner with atomic compatible authority/hub publish parity, attempt-scoped authority and hub contribution output, explicit hub contribution and hub reduce phases in the durable manifest, active-rebuild stale-read coverage for authority/hub top-k, alternating-worker paired-page drain coverage, reusable planned-drain coverage through metric-name worker/coordinator boundaries, threaded concurrent-handle paired-page ownership coverage, later-iteration failed-page retry coverage across authority and explicit hub phases, later-iteration exhausted-attempt paired failure preservation through hub reduce, reclaimed contribution/reduce output coverage, reclaimed convergence-summary reset coverage, cleanup resume coverage, and public paired failure preservation for prior published authority/hub scores. The focused unit-test aggregate is now wired into Zig CI and keeps degree, PageRank, eigenvector, and HITS local-vs-planned parity, partitioned PageRank/HITS worker-drain proofs, eigenvector scan/initialize/contribution/reduce/convergence reclaim, cleanup resume, failed-build preservation, publish-failure preservation, HITS active prior-pair visibility, paired publish idempotence, initialize/contribution/reduce/hub/convergence reclaim, cleanup resume, failed-build preservation, publish-failure preservation, exhausted hub-reduce pair failure, and repeated failed-build cleanup/storage-growth proofs executable without running the entire graph suite. Remaining work is production distributed coverage, promotion-scale fan-in, and latency/default-widening evidence. | +| 9 | Promote behind gates | Compare local and planned outputs, then make the distributed runner the default after restart and parity coverage. | + +Current HITS restart coverage now includes explicit hub contribution and hub +reduce cursor resume across graph-index reopen, in addition to authority +contribution/reduce resume, hub-phase failed-page retry, and hub-reduce +exhausted-attempt preservation of the prior compatible pair. + +Current runtime process-boundary coverage now proves +`OpenOptions.graph_metric_maintenance` can create lease-owned coordinator and +worker-pool runtimes with `start_background_loop = false`; fresh DB-open +handles tick those roles through durable graph-index state and publish degree, +PageRank, eigenvector, and compatible HITS without manual runtime construction +or worker-side metric config. + +This order keeps every slice independently testable. Degree proves the storage, +lease, progress, publish, and cleanup contract without iterative math. PageRank +then proves iteration planning, convergence summaries, and intermediate +storage. Eigenvector and HITS should be last because they are mostly proof that +the executor abstraction is metric-family shaped rather than PageRank-shaped. + +#### Remaining Graph Index Roadmap + +The rest of the graph-index work should be treated as one execution-roadmap, +not as separate PageRank, degree, and eigenvector projects. The durable runner +should become the only way graph metrics materialize large generations, while +the existing local runner remains the compatibility oracle until planned output +matches it. + +Current baseline: + +- The user model is settled: graph metrics live on graph indexes, reads choose + `published` or `fresh`, `MetricNotReady` means no complete generation exists, + and `MetricStale` means a complete generation exists but not for the requested + current edge generation. +- The executor model is settled: coordinators own job creation, barriers, + publish, failure, and cleanup; workers own leased page execution only; runtime + owners and page leases are separate durable fences. +- The implementation already has planned degree, PageRank, eigenvector, and + compatible HITS coverage through durable graph-index state, plus a first + role-configured runtime and `graph-metric-maintenance` command surface. +- What remains is production distribution, not a new public API: remote + deployment orchestration, PageRank-style failure coverage for every promoted + family, public-read fan-in gates, and operations/cleanup hardening. + +Authoritative rest-of-work roadmap: + +| Milestone | Design | Implementation slices | Exit gate | +| --- | --- | --- | --- | +| 1. Process supervisor boundary | Keep the graph metric API unchanged while moving the role-configured runtime into independently managed process owners. Processes receive DB path or service endpoint, runtime owner identity, worker identity, tick budgets, and idle policy only; metric config is resolved from the graph index. | Done for the first process proof: `graph-metric-maintenance supervise` and `launch` run coordinator and worker-pool child roles in bounded rounds, capture JSON summaries, preserve per-role stderr, enforce a narrow argv allow-list, and keep local DB writer-guard behavior confined to direct file-backed proof runs. The integration-test aggregate runs the spawned-process harness with the real `antfly` binary, drains degree, bounded PageRank, eigenvector, and compatible HITS through durable graph-index state, verifies owner/worker/lease telemetry, rejects stale page attempts, and proves duplicate coordinators cannot republish, refail, advance phase, or append duplicate terminal events after publish or publish-verifier failure. | Complete for process smoke: coordinator and worker processes can finish degree, PageRank, eigenvector, and compatible HITS work without worker-side metric config, and the local launch path can drain all four metric families through durable state. Remaining work moves to remote deployment orchestration and rollout readiness. | +| 2. Runtime ownership and lease hardening | Treat durable state, not process memory, as the coordination boundary. Runtime-owner leases fence processes; page leases fence page execution; page attempts fence output adoption. | The process harness now kills a coordinator owner after it has acquired its runtime lease, proves a distinct worker-pool process can continue under an independent runtime lease, proves a duplicate coordinator is fenced before the original lease expires, and proves a replacement coordinator can take over after expiry and advance durable work. It also kills a process that owns a degree scan page lease, proves a replacement worker is fenced before page-lease expiry, proves a real `antfly graph-metric-maintenance --role worker` process can reclaim and complete the page after expiry with an injected clock, and proves the stale old page attempt is rejected after replacement. Same-worker runtime fencing is now covered through real worker role processes: one owner acquires the worker runtime lease and completes a page, a duplicate owner with the same worker id is fenced before expiry, and a replacement owner with the same worker id takes over after expiry, reports runtime `takeover_count`, and completes more durable page work. Replacement coordinator takeover also reports `takeover_count`, giving operations an explicit cross-process replacement signal even when the killed owner cannot observe local `lost_leases`. PageRank now has the same process proof for first-iteration scan/out-degree, initialize, contribution, reduce, and convergence pages; dynamically planned later contribution, reduce, and convergence pages; publish through a real coordinator role process; cleanup through separate real worker role processes; cleanup-page owner death/reclaim with the published generation visible; publish-verifier failure through a real coordinator process while preserving the prior published generation; duplicate coordinator idempotence after that publish-verifier failure; and same-worker-id replacement after page-lease expiry with stale prior-attempt rejection. Remaining hardening is remote orchestration beyond a direct local DB path. | Worker loss abandons only reclaimable page work; stale owners cannot write, adopt, complete, publish, or fail the build after replacement; duplicate coordinators cannot double-publish, double-fail, or append incompatible pages/events. | +| 3. Crash/restart harness | Make failures reproducible before promoting distributed execution. Every phase must be restartable from fresh DB handles and process boundaries. | The current degree harness now covers runtime-owner kill/takeover, scan-page owner kill/reclaim, stale scan-page attempt rejection, publish-boundary restart through a real coordinator process, and cleanup restart through separate real worker processes. PageRank process coverage now covers first-iteration scan/out-degree, initialize, contribution, reduce, and convergence owner kill, dynamically planned later contribution/reduce/convergence owner kill, before-expiry fencing, post-expiry reclaim by a real worker role, stale attempt rejection, same-worker replacement attempt fencing, drain to fresh, publish-boundary restart through a real coordinator process, cleanup restart through separate real worker processes, cleanup-page owner death/reclaim, and publish-verifier failure preserving the prior generation. | Published reads see either the prior complete generation or the newly published complete generation. Active job output, abandoned attempts, and failed generations are never queryable. | +| 4. Distributed degree promotion | Use degree as the first production ownership proof because it exercises scan, reduce, publish, cleanup, leases, attempts, and public freshness without iterative math. | Degree now has a spawned-process restart proof for supervisor completion, coordinator runtime lease loss, worker page lease loss/reclaim, publish-boundary restart, cleanup restart, and stale attempt rejection. Remaining degree work is promotion hardening: keep local/planned parity, broaden public read-surface freshness coverage through the distributed gate, verify active/failed status page summaries from durable records, and decide the internal rollout gate. | Degree can be enabled behind an internal distributed gate with local-vs-planned parity, process crash coverage, cleanup resume coverage, and direct/traversal/search freshness coverage. | +| 5. Distributed PageRank matrix | PageRank is the reference iterative metric. It must prove dynamic iteration planning, convergence, fixed-iteration non-converged publish, publish verification, failure preservation, and cleanup across process boundaries. | The process harness now covers first-iteration and dynamically planned later-iteration page owner death/reclaim/stale-attempt rejection, publish through a real coordinator process, cleanup through real worker processes, cleanup-page owner death/reclaim, same-worker replacement, publish-verifier failure preserving the prior generation, duplicate coordinator idempotence after publish-verifier failure, fixed-iteration non-converged publish metadata through both the process-supervised path and the publish-boundary coordinator process path, a bounded PageRank build drained through independently launched coordinator and worker-pool owners, and public freshness while a spawned coordinator-owned rebuild is active: direct metric top-k, graph traversal projection, and search rerank serve the prior generation with `building` status, while direct `fresh` reads, traversal `fresh` projection/order/filter, and rerank `fresh` fail with `MetricStale`. The real HTTP service-owner path now also covers PageRank publish-verifier failure: a service-targeted coordinator observes a corrupted manifest, fails without publishing, preserves the prior generation, and a duplicate service coordinator cannot fail or publish again. The unit-test aggregate now also enforces PageRank local-vs-planned score parity, partitioned alternating-worker drain behavior, and repeated failed-build cleanup/storage-growth behavior. Remaining work is production remote orchestration and promotion-scale cross-shard public read coverage. | A dirty PageRank rebuild can finish through remote owners, match local output within tolerance, publish fixed-iteration non-converged results with `converged: false`, preserve the prior generation on failed rebuilds, and recover from restart at every phase boundary. | +| 6. Eigenvector parity | Eigenvector proves the single-vector iterative substrate is metric-family shaped rather than PageRank-specific. It must not add a new job system, status model, or query API. | The process harness now has eigenvector process-boundary coverage: the real `antfly graph-metric-maintenance supervise` path starts coordinator and worker-pool child roles, drains a bounded eigenvector build through durable graph-index state, and verifies the target generation becomes fresh; the independently launched coordinator/worker-pool path now drains a separate eigenvector build to fresh through the same graph-index state; separate killed-owner cases cover scan, initialize, contribution, reduce, and convergence page leases, before-expiry fencing, post-expiry reclaim by a real worker role, stale-attempt rejection, and final drain to fresh; publish/cleanup coverage prepares a build to `publish_generation`, publishes through a real coordinator role process, then completes cleanup through a separate worker role process; publish-verifier failure coverage corrupts the manifest at `publish_generation`, fails through a real coordinator process, preserves the prior generation with bounded diagnostics, and proves a second coordinator cannot fail the already-failed build again or append another failure event. The same publish-verifier failure invariant now also crosses the real HTTP service-owner path for eigenvector service coordinators. Fixed-iteration non-converged publish now also crosses the process-supervised path and verifies `converged: false`, `iterations_completed`, and positive finite delta metadata. Active-rebuild public freshness now also crosses a coordinator-owned process boundary: `published` eigenvector direct top-k, traversal projection/status, and search rerank serve the prior generation with `building` status, while `fresh` direct reads, traversal projection/order/filter, and search rerank fail closed with `MetricStale`. Graph-layer parity now also covers disconnected/reducible topology: local and planned eigenvector output match within tolerance, every score is finite, the vector remains L2-normalized, and reducible sink-chain nodes can decay to zero without invalid output. Remaining work is promotion-scale cross-shard fan-in and production rollout evidence. | Eigenvector reaches the same restart/failure/cleanup/public-read matrix as PageRank through the same graph-index metric API and distributed executor. | +| 7. HITS paired-vector production | HITS is the paired-vector proof. Authority and hub are separate named scores, but compatible configs share one target generation, convergence decision, publish decision, failure decision, and cleanup lifecycle. | The process harness now has a first HITS process-boundary proof: a compatible manual authority/hub pair is explicitly started as a planned build, the real supervisor launches coordinator and worker-pool child roles, and both authority and hub publish the same fresh target generation with queryable top-k output. The independently launched coordinator/worker-pool path now also drains a compatible HITS pair to fresh. It also kills process-owned authority contribution, authority reduce, convergence, `hits_hub_contributions`, and `hits_hub_reduce_ranks` page owners, proves replacement workers are fenced before lease expiry, proves real worker role processes reclaim and complete those pages after expiry, rejects stale attempts, and drains the compatible pair to fresh. Paired publish/cleanup now also crosses process boundaries: a real coordinator role publishes the authority/hub pair atomically, authority remains in cleanup while hub is fresh/complete with the same generation and publish event, and separate worker role processes resume cleanup until both metrics are fresh. Publish-verifier failure now also crosses a real coordinator role process: a corrupted HITS manifest fails without publishing, both authority and hub preserve the previous generation and retained publish-failure diagnostics, and a second coordinator cannot fail the already-failed compatible pair again or append another failure event. The same paired publish-verifier failure invariant now also crosses the real HTTP service-owner path: a service-targeted coordinator fails the compatible pair without publishing, and a duplicate service coordinator cannot add another failure or publish event. Failed HITS public reads now preserve that prior compatible pair across direct top-k, traversal projection/status, and search rerank score details, while `fresh` direct, traversal projection, and rerank reads fail closed with `MetricStale`. Active-rebuild public freshness now crosses a coordinator-owned process boundary: published authority/hub top-k reads serve the prior compatible pair with `building` or compatible stale status, traversal projection/status serves prior authority/hub scores, and search rerank over HITS authority uses prior-generation score details, while fresh authority/hub direct reads, traversal projection/order/filter, and search rerank fail closed with `MetricStale`. Exhausted-attempt failure now also crosses process boundaries: repeated killed owners reclaim the same later-iteration hub-reduce page across expired leases until a real coordinator fails the compatible pair with `GraphMetricBuildPageAttemptsExhausted`, preserving the previous pair and prior top-k/direct/traversal/rerank output. Public fan-in now rejects default authority/hub status pairs with mismatched published generations or incompatible stable metadata/edge filters before merging direct metric or graph traversal/search status surfaces. Hosted cross-range fan-in now also proves compatible HITS authority/hub pairs merge for both `published` and `fresh` reads across direct metric, traversal projection/order/filter/status, and HITS authority rerank surfaces, while unpublished or mixed-generation HITS shards fail closed before those score surfaces can mix results. Real HTTP service-owner publish/cleanup and multi-page proofs now also verify paired fixed-iteration metadata for authority and hub: both publish `converged: false`, matching `iterations_completed`, and positive finite deltas when the bounded iteration limit is reached. Remaining work is promotion-scale parity, larger-graph latency, remote deployment orchestration, and promotion-scale cross-shard fan-in coverage under remote owners. Keep hub and authority phase work page-partitioned and attempt-fenced where partial output can be consumed by later phases. | Compatible HITS authority/hub output publishes atomically, either side failing preserves the previous compatible pair, and remote HITS remains disabled by default until paired restart/failure coverage matches the PageRank matrix. | +| 8. Public read and fan-in gates | Distributed execution is not complete until every score-bearing read path uses only complete published generations and fails closed for freshness mismatches. | Current fast-root fan-in coverage now includes direct metric top-k merge, direct missing/unpublished shard rejection, direct failed-rebuild shard-status preservation with `fresh` fail-closed behavior, traversal projection/status generation checks, traversal failed-rebuild shard-status preservation with `fresh` fail-closed behavior, traversal order/filter generation checks, traversal failed-rebuild order/filter status preservation with `fresh` fail-closed behavior, duplicate graph order metric request rejection, duplicate graph-search node/hit ownership rejection, direct/traversal/rerank profile status output, failed-status profile output across direct metric top-k, traversal, and rerank surfaces, direct metric top-k merged profile status after shard fan-in, search rerank merged profile status after shard fan-in, search rerank incompatible-generation checks, rerank score details, rerank score-details arithmetic and missing-score consistency checks, rerank missing/unpublished shard-status rejection, rerank failed-rebuild shard-status preservation with `fresh` fail-closed behavior, HITS authority/hub pair compatibility rejection for direct graph metric and graph traversal/search status fan-in, and direct/traversal/rerank status-generation shape checks that reject `fresh` statuses whose current or target edge generation does not match the published score generation. DB search coverage now also proves direct graph metric top-k and search rerank return `MetricNotReady` for both `published` and `fresh` before first publish, while traversal projection/order/filter DB coverage proves not-ready, stale, building, failed, and fresh-failure behavior. Hosted cross-range coverage now also builds local shard DBs from catalog range metadata, proves compatible fresh published generations merge for `published` and `fresh` direct metric top-k, traversal projection/status/order/filter, and search rerank, proves direct graph metric fan-in and search rerank fail closed when one shard is unpublished, advances one shard to an incompatible published generation, and proves direct metric top-k, traversal projection/status/order/filter, and search rerank reject mixed-generation results. It now also includes paired HITS authority/hub in the hosted path: compatible shard generations merge for both `published` and `fresh` direct metric fan-in, traversal projection/order/filter/status fan-in, and HITS authority rerank, while unpublished or mixed-generation paired HITS shards fail closed before score mixing. It also proves cross-range traversal metric projection/status/order/filter now carries metric payloads through distributed expand/fan-in, reports graph-query metric status in query profile, and `fresh` projection fails closed on an unpublished shard. Focused public API graph metric e2e coverage now covers public HTTP status, not-ready status/direct/projection/rerank behavior before first publish, fresh direct top-k, traversal projection, ordering, filtering, stale `published` direct top-k/projection/search rerank after a `sync_level: "write"` graph update, prior-generation rerank score details, stale rerank profile status, active-build `published` direct/projection/rerank reads with `building` status, failed-build `published` direct/projection/rerank reads with `failed` status, and `fresh` fail-closed behavior across stale, building, and failed public direct/traversal/rerank reads. The broader `public-api-parity-test` covers generated public query bodies with explicit `null` graph metric fields. Those hosted fan-in and public HTTP freshness checks now run through the unit-test aggregate, so hosted fan-in and public HTTP freshness regressions are no longer local-only evidence. Remaining read-gate work is promotion-scale cross-shard generation compatibility for every read surface and, if a standalone explain API is added later, matching graph metric generation/freshness evidence there. Query profile is the current explainable read contract and already carries direct, traversal, and rerank metric status. | `published` reads use the latest complete generation; `fresh` returns `MetricNotReady` before first publish or `MetricStale` when published is behind; cross-shard fan-in rejects missing, zero, stale, or incompatible generations when comparability is required. | +| 9. Cleanup, retention, and operations | V1 cleanup remains aggressive. Retain the latest published generation and bounded diagnostics; remove completed, failed, abandoned, unpublished, and attempt-scoped job state as soon as snapshot safety allows. Retention, pause/resume, manual retry, priority, and lease tuning are future admin controls, not query semantics. | Make cleanup process-resumable for score generations, job namespaces, manifests, pages, phase summaries, iteration summaries, attempt namespaces, failure diagnostics, and runtime-owner records. Keep operations status high level: role, owner hash, worker identity count/hash, phase, iteration, page counts, cursor presence, attempts, progress, last error, and last sweep result. Fast root coverage now includes failed planned-build cleanup of abandoned score/job namespaces plus bounded retained failure diagnostics and metric events, including old-key pruning; runtime-status cache coverage preserves graph metric runtime ownership telemetry such as owner/worker hashes, worker count, lease state, takeover count, lost leases, tick progress, and page counters across sequence-only status refreshes; graph index status now exposes that telemetry as a typed `graph_metric_runtime` OpenAPI summary on aggregate and per-shard graph index status; command summary and real spawned-process harness coverage now preserve the same role/owner/worker/lease/tick/error telemetry in supervisor/launcher output; and direct runtime-owner lease-record assertions prove clean runtime shutdown deletes the current durable owner record while stale shutdown after takeover leaves the replacement record intact. The unit-test aggregate now pins cleanup-page resume for PageRank/eigenvector/HITS, active-job cleanup refusal, failed planned-build abandoned namespace cleanup, bounded retained diagnostics, repeated failed non-iterative/iterative/paired cleanup, and durable runtime-owner lease cleanup. The unit-test aggregate pins active build page status summaries, failed-page cursor/progress/error diagnostics, OpenAPI/runtime-status encoders, internal service maintenance boundary, command/supervisor argv contract, child runtime telemetry parsing, idle-exit behavior, and owner/worker summary fields. | Cleanup resumes after restart, never deletes the current published generation, bounds diagnostics, and prevents graph-metric control/output namespaces from growing without a public retention knob. Deployment-scale cleanup cost and storage-growth qualification remain separate release evidence. | +| 10. Per-family default promotion | Promote planned/distributed execution by metric family, not all graph metrics at once. Local runners stay as CI/debug oracles until distributed parity is boring. | Keep the conservative `auto` gate, collect latency evidence, then promote degree and PageRank first, eigenvector after single-vector parity, and HITS last after paired-vector coverage. The internal auto-gate decision summary now makes the rollout boundary inspectable in tests: active planned work, eligible queued work, and ineligible queued work are counted before default or explicit auto idle paths choose planned execution or local fallback. Queued degree, small PageRank, bounded eigenvector, and active planned PageRank/degree/eigenvector/HITS all assert the planned decision before execution; larger PageRank/eigenvector, multi-metric indexes, default HITS, and incompatible HITS assert fallback before local execution. HITS opt-in is inspectable through the same counters: default HITS and incompatible opt-in pairs remain ineligible, while a compatible opt-in authority/hub pair contributes one eligible queued build before planned execution. PageRank and eigenvector threshold widening are now covered too: raising the internal iteration caps makes previously gated larger single-vector rebuilds eligible for planned execution, proves tiny-budget exhaustion leaves resumable active work, and proves the same jobs complete when the budget is widened. Unit-test default-gate coverage pins those conservative auto-gate, widening, fallback, HITS opt-in, and active-planned-resume decisions in PR and full-default Zig CI. Update generated clients and public docs only around stable user-facing status/freshness fields. | Each promoted family has deterministic parity, process ownership, crash/restart, cleanup, public-read, cross-shard, status, operations, latency-budget, and rollback coverage before distributed execution becomes the default. | + +The next PRs should be cut along these implementation slices: + +1. **Production remote orchestration**: promote the process launcher beyond the + direct local DB proof into deployment hooks for remote coordinator and worker + owners. Workers should still receive only DB path or service endpoint, + role/owner identity, worker identity, budgets, and idle policy; all metric + config must remain graph-index owned. Direct file-backed launch keeps its + explicit local DB writer guard because graph runtime leases are not a + storage-engine multi-writer lock. +2. **Degree distributed gate**: finish promotion hardening for degree, including + local/planned parity, process restart, cleanup, durable page status, public + freshness, and the internal rollout flag. +3. **PageRank distributed gate**: add the remaining PageRank promotion evidence: + local/planned parity under production budgets and cross-shard fan-in + behavior. Fixed-iteration non-converged publish metadata plus direct metric + top-k, graph traversal, and search-rerank active-rebuild freshness are now + covered through the process harness. +4. **Algorithm-family rollout**: bring eigenvector to the PageRank + single-vector matrix, then bring HITS to the stricter paired authority/hub + matrix before enabling either by default. +5. **Read-surface gate**: prove direct reads, traversal, search/rerank, + explain/profile, and cross-shard fan-in all honor the same published/fresh + contract before enabling any distributed metric family by default. +6. **Operations gate**: document and test runtime status, bounded diagnostics, + immediate v1 cleanup, and future bounded retention controls. + +Rest-of-work design plan: + +1. **Finish the production process contract before promoting more defaults.** + The graph metric scheduler already has durable jobs, pages, attempts, + runtime leases, and role-owned process execution. The remaining process work + should prove production orchestration, not introduce another executor: + production worker/coordinator launch, stale worker rejection before + write/adopt/complete, stale coordinator rejection before publish/fail, and + final drain to a fresh published generation. Runtime replacement telemetry is + now covered by `takeover_count` on replacement owners after expired durable + lease takeover. PageRank now covers the first-iteration scan/out-degree, + initialize, contribution, reduce, convergence, publish, cleanup, dynamically + planned later contribution/reduce/convergence process boundaries, + publish-verifier failure, and same-worker replacement. +2. **Promote by metric family, not by executor feature.** + Degree is the promotion canary because it exercises planning, scan/reduce, + publish, cleanup, leases, and stale reads without iterative math. PageRank + follows because it is the reference single-vector iterative metric. + Eigenvector should reuse the PageRank runner after PageRank is stable. HITS + should remain last because paired authority/hub visibility makes failure and + publish semantics stricter than the single-vector metrics. +3. **Keep local execution as the oracle until each family is boring.** + Every promoted metric family should keep deterministic local-vs-planned + parity tests, restart tests, cleanup tests, public freshness tests, and + failure-preservation tests. Local execution can remain a debug/CI oracle even + after planned execution becomes the default for production scheduling. +4. **Treat public reads as a promotion gate, not a follow-up.** + A distributed metric is not done when the worker publishes a score + generation. It is done when direct metric top-k, traversal projection, + traversal order/filter, search rerank, explain/profile output, and + cross-shard fan-in all prove the same generation contract: `published` uses + the latest complete generation, `fresh` fails with `MetricNotReady` or + `MetricStale`, and active/failed/unpublished output is never visible. + Focused public HTTP coverage now proves that contract for not-ready, stale, + active-build, and failed-build states on direct top-k, traversal projection, + and search rerank. Fast-root merge coverage now also proves merged direct + metric top-k and merged rerank profile output carries the merged shard + generation, status, freshness, and failure details. Hosted cross-range + coverage now proves direct metric fan-in fails closed when one local shard has + no published PageRank generation, and hosted cross-range search rerank now + fails closed on that unpublished shard instead of reranking mixed-generation + hits. The same hosted fixture now advances the right shard to a different + published metric generation and proves direct metric top-k, traversal + projection/status/order/filter, and search rerank all reject incompatible + published generations. A compatible hosted two-shard fixture now proves + direct metric top-k, traversal projection/status/order/filter, and search + rerank merge for both `published` and `fresh` reads when both shards report + the same nonzero fresh published generation. + Cross-range traversal metric projection/status/order/filter now carries + metric payloads through distributed expand and fan-in, reports graph-query + metric status in query profile, and `fresh` projection fails closed on an + unpublished shard. The remaining read-gate work is promotion-scale + cross-shard compatibility for every read surface, plus matching generation + and freshness evidence for any future standalone explain API. Query profile + is the current explainable read contract and already reports direct, + traversal, and rerank metric status. +5. **Make cleanup a durable phase for every family.** + V1 should keep the aggressive cleanup default: retain the current published + generation and bounded diagnostics, then delete completed, failed, + abandoned, unpublished, attempt-scoped, and job-scoped state as soon as + snapshot safety allows. Future retention can be opt-in and bounded, but it + should not affect query freshness semantics. +6. **Expose operations state without exposing storage internals.** + Status should summarize role, owner hash, worker identity count or hash, + phase, iteration, page counts, cursor presence, attempts, progress, last + error, lease expiry, and last cleanup sweep. It should not expose raw key + prefixes or make runtime-owner identities part of the stable public API. + +Concrete delivery order: + +| Order | Slice | Primary files | Done when | +| --- | --- | --- | --- | +| 1 | Production remote orchestration | maintenance runtime, process launcher, deployment hooks | Production coordinator and worker processes communicate only through durable graph-index job/page state or the equivalent service boundary, drain degree/PageRank work with bounded restarts, and do not rely on shared local file-writer serialization except for the direct DB launch harness. | +| 2 | Degree distributed gate | DB/runtime config, maintenance runtime, tests | Degree can run through planned maintenance by default behind an internal gate with parity, restart, public freshness, durable page status, and cleanup coverage. | +| 3 | PageRank distributed gate | planner/executor/runtime tests | PageRank distributed execution matches local output within tolerance, preserves failed rebuilds, reports fixed-iteration non-converged publish metadata, and enforces public freshness. | +| 4 | Eigenvector backfill | shared iterative executor plus eigenvector math | Eigenvector reuses PageRank's single-vector lifecycle, convergence summaries, failure behavior, cleanup, and promotion gates. | +| 5 | HITS paired-vector hardening | shared iterative executor plus paired HITS publish | Compatible authority/hub output publishes atomically, fails atomically, and preserves the previous compatible pair under process restart. | +| 6 | Public read/fan-in gate | public API/query/search/e2e tests | Every read surface enforces published/fresh semantics and cross-shard comparability. | +| 7 | Operations and retention gate | status APIs, runtime stats, cleanup tests | Status is useful for operators, diagnostics are bounded, and cleanup prevents namespace growth. | +| 8 | Default promotion | internal gates, CI, docs, generated clients | Each family is promoted only after parity, process ownership, crash/restart, cleanup, public-read, fan-in, and operations coverage pass. | + +Design boundaries for the rest of the work: + +- The graph index owns metric config, target edge generation, build job state, + page manifests, score generations, freshness, and cleanup. +- Coordinators own job creation, phase barriers, iteration decisions, publish, + failure, and cleanup scheduling. +- Workers own page execution only. They receive page metadata from durable + state and must not receive caller-supplied metric config. +- Runtime-owner leases fence long-lived process roles. Page leases and attempts + fence page execution. Publish is fenced by coordinator ownership and verified + durable summaries. +- Attempt-scoped output is required when later phases could consume partial + output. Completed pages adopt output into the job namespace; cleanup removes + abandoned attempts. +- Fixed-iteration non-converged PageRank and eigenvector output should publish + by default when the config asked for a bounded run, but status and metadata + must record `converged: false`, completed iterations, and final deltas. +- Old generations should be cleaned up immediately in v1 once snapshot safety + allows. A future retention option may keep additional published generations + or failed diagnostics, but it must be bounded and opt-in. + +Completed process-proof slice: + +- The top-level `antfly graph-metric-maintenance supervise` path builds under + the full binary. +- Deterministic supervisor tests drive degree through supervisor-built + coordinator and worker-pool child argv. +- The integration-test aggregate builds the real `antfly` executable, seeds a + graph degree DB through the library, runs supervisor mode with spawned child + role processes, and verifies the degree metric is fresh for the target edge + generation after child completion. +- The same real spawned-process harness now parses supervisor and launcher + aggregate summaries and asserts coordinator plus worker-pool child telemetry: + runtime/owner hashes, worker identity count/hash, lease-key presence, lease + ownership, acquisition count, tick progress, and no last error. +- The same spawned-process harness now also seeds a bounded eigenvector metric, + runs it through real supervisor-launched coordinator and worker-pool roles, + verifies the target eigenvector generation is fresh after cleanup, and covers + killed scan, initialize, contribution, reduce, and convergence page owners + being fenced before expiry, reclaimed after expiry by real worker role + processes, and rejected as stale attempts after replacement. It also covers + eigenvector publish through a real coordinator role process, cleanup through a + separate worker role process, and publish-verifier failure preserving the + prior published eigenvector generation. Eigenvector process coverage now also + verifies fixed-iteration non-converged publish metadata after process + execution: `converged: false`, the expected completed iteration count, and a + positive finite delta. +- Eigenvector process coverage now also starts a dirty rebuild through a real + coordinator role process and verifies public read freshness: `published` + direct top-k, traversal, and search rerank serve the prior generation with + `building` status, while `fresh` direct reads, traversal + projection/order/filter, and search rerank fail closed with `MetricStale`. +- Eigenvector graph-layer parity now also covers a disconnected/reducible graph: + local and planned output match within tolerance, scores remain finite and + L2-normalized, and reducible sink-chain nodes can decay to zero without + invalid output. +- The same spawned-process harness now also seeds a compatible HITS + authority/hub pair, explicitly starts the planned authority-side build, + drains it through real supervisor-launched coordinator and worker-pool child + roles, verifies both named metrics publish the same fresh target generation, + and verifies both authority and hub top-k reads return score output. +- HITS process coverage now also kills authority contribution, authority reduce, + convergence, `hits_hub_contributions`, and `hits_hub_reduce_ranks` + page-owner processes, verifies replacement workers are fenced before lease + expiry, verifies real worker role processes reclaim and complete the expired + pages, rejects stale prior attempts, and drains both authority and hub metrics + to the same fresh generation. +- HITS process coverage now prepares a compatible authority/hub build to + `publish_generation`, publishes the pair through a real coordinator role + process, verifies authority stays in cleanup while hub is fresh/complete with + the same generation and publish event, and finishes cleanup through separate + worker role processes until both named metrics are fresh. +- HITS process coverage now corrupts a compatible authority/hub rebuild + manifest at `publish_generation`, fails publish verification through a real + coordinator role process, verifies both metrics preserve the previous + published generation with retained publish-failure diagnostics, and verifies + prior authority/hub top-k output remains visible. +- HITS process coverage now also starts a dirty compatible authority/hub rebuild + through a real coordinator role process and verifies public direct metric + reads: `published` authority and hub top-k queries serve the prior compatible + generation with `building` status, while `fresh` authority/hub reads fail + closed with `MetricStale`. +- HITS process coverage now also exhausts a later-iteration hub-reduce page by + repeatedly killing process owners across expired page leases, then lets a real + coordinator role fail the compatible pair with + `GraphMetricBuildPageAttemptsExhausted` while preserving the previous + authority/hub generation, failure diagnostics, and prior top-k visibility. +- The same process test kills a coordinator after it has acquired a runtime + lease, verifies a worker-pool process can continue under a separate runtime + lease, verifies a duplicate coordinator is fenced before lease expiry, and + verifies a replacement coordinator can take over after expiry and advance + durable work. +- The process test also kills a degree scan-page owner process after it claims + and persists cursor progress, verifies a replacement worker process cannot + reclaim before page-lease expiry, verifies a real worker role process reclaims + and completes the page after expiry through an injected clock, verifies the + stale old attempt is rejected, and then drains the metric to a fresh + generation. +- The process test prepares a degree build to `publish_generation`, closes the + setup DB handle, publishes through a real coordinator role process, verifies + cleanup remains resumable with the new generation visible, completes cleanup + through separate real worker role processes, and verifies the metric returns + to fresh. +- The process test also seeds PageRank metrics and covers killed process-owned + PageRank `scan_edges_and_out_degree`, `initialize_ranks`, + `iterate_contributions`, `reduce_ranks`, and `check_convergence` pages. For + each phase, it persists cursor progress, verifies a replacement worker is + fenced before expiry, verifies a real worker role process reclaims and + completes the page after expiry, verifies the stale old attempt is rejected, + and drains the PageRank build to fresh. +- The process test now prepares a PageRank build to `publish_generation`, + publishes through a real coordinator role process, verifies cleanup remains + visible with the new generation published, completes PageRank cleanup through + separate real worker role processes, and verifies the metric returns to + fresh. +- The process test also kills a PageRank cleanup page owner after the new + generation is published, verifies the killed cleanup page remains fenced + before lease expiry even while another cleanup worker may complete a + different page, verifies a real worker role process reclaims the expired + cleanup page, rejects the stale old attempt, and drains cleanup to fresh. +- The process test now corrupts a PageRank rebuild manifest at + `publish_generation`, runs a real coordinator role process, verifies publish + verification fails without publishing, and verifies the prior published + generation plus retained failure diagnostics remain visible. +- The process test now also kills a PageRank page owner and replaces it after + lease expiry with a new process using the same worker id, proving the + replacement attempt can complete while the stale prior attempt from the same + worker id is rejected. +- The process test now also uses partitioned degree work to prove duplicate + same-worker runtime fencing through real worker role processes: a duplicate + owner with the same worker id is fenced before lease expiry, and a replacement + owner with the same worker id acquires after expiry, reports + `takeover_count`, and completes additional durable page work. +- The process test now also verifies replacement coordinator takeover reports + `takeover_count` after acquiring the expired durable runtime lease left by a + killed coordinator owner. +- The process test also runs PageRank with `max_iterations: 2`, advances through + a non-final convergence barrier into dynamically planned iteration-1 + contribution, reduce, and convergence pages, kills each page owner, proves + before-expiry fencing, proves after-expiry reclaim by a real worker role + process, rejects the stale old attempt, and drains each build to fresh. +- The process test now also verifies fixed-iteration non-converged PageRank + metadata after process execution: `converged: false`, the expected completed + iteration count, and a positive finite final delta are preserved after both + end-to-end process supervision and coordinator-process publish followed by + worker-process cleanup. +- The process test now also verifies direct metric top-k freshness during a + process-owned active PageRank rebuild: after a spawned coordinator starts the + next generation, `published` reads continue to return prior-generation scores + with `building` status and the active building generation, while `fresh` fails + closed with `MetricStale`. +- The same process-owned active PageRank rebuild now also covers graph + traversal projection/order/filter freshness: `published` traversal projection + serves the prior PageRank score with `building` status, and `fresh` + projection, ordering, and filtering fail closed with `MetricStale`. +- The same process-owned active PageRank rebuild now also covers search rerank + freshness: `published` rerank uses prior-generation score details with + `building` status, while `fresh` rerank fails closed with `MetricStale`. + +The next unimplemented slices are: + +1. Replace the local supervisor proof with production remote orchestration + where independently launched coordinator and worker owners communicate only + through durable graph-index state. +2. Finish degree promotion hardening, then put distributed degree behind an + internal feature gate. +3. Finish PageRank promotion hardening: extend the new DB-level + production-budget parity gate into deployment-scale parity and shard fan-in + behavior. +4. Bring eigenvector to the PageRank single-vector matrix, then bring HITS to + the paired authority/hub matrix. +5. Prove direct reads, traversal, search/rerank, explain/profile, and + cross-shard fan-in all honor the same published/fresh contract before + enabling any distributed metric family by default. +6. Document and test runtime status, bounded diagnostics, immediate v1 cleanup, + and future bounded retention controls. + +Plan the rest of the work around dependency order, not around metric names. +The current implementation has enough executor surface that the next changes +should be release gates: + +| Phase | Why it comes next | Work to do | Promotion signal | +| --- | --- | --- | --- | +| A. Freeze the public contract | The UI/API is already the right shape: graph metrics live on graph indexes, users read `published` or `fresh`, and jobs remain internal. Churn here would multiply generated-client and query-path work. | Keep metric config on the graph index, keep freshness on reads, keep `MetricNotReady` and `MetricStale` as the explicit read failures, and document that fixed-iteration non-converged output can publish with `converged: false` metadata. Do not add job handles, worker controls, or per-query metric execution. | Public docs and generated clients expose only stable metric config, status, freshness, convergence, and failure fields. Internal job/lease/page fields stay out of user request bodies. | +| B. Make remote ownership production-shaped | The process harness proves the split role model locally; production still needs the same role contract through deployment-managed owners. | Move from local supervisor proof to deployment hooks for independently launched coordinator and worker processes. Owners receive endpoint or DB location, role, owner id, worker id, tick budget, lease policy, and idle policy. They resolve metric configs only from durable graph-index state. | A coordinator and multiple workers can be started, killed, replaced, and drained without shared memory or caller-supplied metric config. Duplicate owners are fenced by durable runtime leases and stale page attempts. | +| C. Close cross-shard read comparability | Distributed execution is incomplete if shard fan-in can merge incomparable score generations. | Add promotion-scale tests for direct metric top-k, traversal projection/order/filter, search rerank, explain/profile, and shard fan-in across `published`, `fresh`, not-ready, stale, active-build, failed-build, missing-generation, zero-generation, and incompatible-generation cases. | Every score-bearing fan-in either proves all shard scores come from compatible nonzero published generations or fails closed with `MetricNotReady`, `MetricStale`, or an internal unsupported-query error before mixing scores. | +| D. Promote degree first | Degree is the smallest production canary because it exercises scan, reduce, publish, cleanup, page leases, runtime leases, stale attempts, and public freshness without iterative convergence. | Put planned/distributed degree behind an internal gate. Keep local-vs-planned parity, process restart, cleanup resume, active/failed status, page-summary status, and direct/traversal/search freshness tests in CI. | Degree can run by default behind the internal gate with bounded latency, bounded diagnostics, no namespace growth, and a rollback path to the local oracle. | +| E. Promote PageRank as the reference iterative family | PageRank is the single-vector iterative standard that later centrality metrics should inherit. | Keep the DB-level production-budget local/planned parity gate, then finish deployment-scale parity, cross-shard fan-in tests, larger-graph latency evidence, and status/cleanup operations checks. Keep dynamic iteration planning, fixed-iteration non-converged publish metadata, publish verification, failed rebuild preservation, and every phase restart/reclaim test as required checks. | Distributed PageRank matches local output within tolerance, serves prior published generations during rebuilds, fails `fresh` reads while stale, and preserves the previous generation on failed rebuilds. | +| F. Backfill eigenvector through the PageRank substrate | Eigenvector should prove the executor is generic for single-vector centrality, not introduce another lifecycle. | Keep the completed process-boundary, fixed-iteration, active-freshness, disconnected/reducible parity coverage, and DB-level production-budget local/planned parity gate. Add the same cross-shard fan-in, deployment-scale parity, and operations evidence required for PageRank before default promotion. | Eigenvector can be promoted only when its restart, cleanup, failure, freshness, fan-in, and operations matrix is equivalent to PageRank's single-vector matrix. | +| G. Harden HITS last | HITS has stricter semantics because authority and hub are separate named scores sharing one compatible pair lifecycle. | Finish paired local-vs-planned parity, larger-graph latency, remote-owner deployment, public read-surface coverage, fan-in coverage, and pair-level failure/publish/cleanup operations evidence. Preserve attempt-scoped authority/hub output until page completion and publish/fail the pair atomically. | Remote HITS stays disabled until either side failing preserves the previous compatible pair, paired publish is atomic, cleanup is resumable, and paired-vector restart coverage matches the PageRank quality bar. | +| H. Promote defaults one family at a time | Defaulting the executor is a product decision, not just a test result. | Keep local runners as CI/debug oracles. Widen the conservative `auto` gate in this order: degree, PageRank, eigenvector, HITS. Require release history, latency budgets, operations docs, cleanup bounds, and generated-client stability before each widening. | Each family is defaulted only after parity, process ownership, crash/restart, cleanup, public-read, fan-in, status, operations, and latency evidence are routine. | + +This roadmap intentionally keeps the user interface stable while the +implementation changes underneath it: + +- users configure named metrics on the graph index +- users choose `published` or `fresh` at read time +- users inspect status, convergence, progress, and failure diagnostics +- coordinators and workers remain internal maintenance actors +- retention and retry policy stay internal in v1, with only future bounded + admin controls considered after distributed execution is stable + +The implementation dependency chain should stay strict: remote ownership before +default promotion, cross-shard comparability before any score-bearing fan-in +promotion, degree before PageRank defaults, PageRank before eigenvector +defaults, and HITS only after paired-vector failure behavior is routine. + +The remaining roadmap should be delivered as four gates: + +1. **Execution gate**: production coordinators and workers are durable-state + clients. They start, tick, restart, and stop independently; they do not + exchange metric configs or in-memory phase state. Runtime-owner leases fence + process roles, page leases fence page execution, and page attempts fence + output adoption. +2. **Metric-family gate**: each metric family earns promotion separately. + Degree proves the non-iterative path, PageRank proves the reference + single-vector iterative path, eigenvector proves the same substrate with + normalization, and HITS proves paired authority/hub atomic publish. +3. **Read-surface gate**: every user-visible score path reads only published + generations unless `fresh` is requested, in which case stale or absent + generations fail closed with `MetricStale` or `MetricNotReady`. +4. **Operations gate**: status and cleanup are production features, not test + helpers. Status reports progress, failures, ownership counters, and bounded + diagnostics; cleanup removes completed, failed, abandoned, unpublished, and + attempt-scoped state promptly while preserving the current published + generation. + +The steady-state architecture should look like this: + +```text +graph index metric config + -> dirty marker for target edge generation + -> coordinator-owned durable build job + -> deterministic manifest pages + -> worker-owned page leases and cursors + -> attempt-scoped intermediate output where needed + -> coordinator-owned phase and iteration barriers + -> verified publish of one complete generation + -> published generation pointer + -> resumable cleanup and bounded diagnostics +``` + +Roadmap from the current state: + +| Step | Design target | Implementation plan | Exit gate | +| --- | --- | --- | --- | +| R1. Production execution contract | Keep jobs invisible to users. A user configures a named graph-index metric and chooses `published` or `fresh`; coordinators and workers are internal maintenance actors. | Freeze the process role contract around endpoint/DB location, role, owner identity, worker identity, budgets, clock, and idle policy. Move launch/orchestration behind deployment hooks while keeping metric config, target generation, manifests, pages, attempts, and publish decisions inside durable graph-index state. | Independently started coordinator and worker owners can drain degree and PageRank from dirty state to fresh publish without shared memory, caller-supplied metric config, or local test-only process coupling. | +| R2. Degree canary | Degree is the first default distributed family because it exercises scan, reduce, publish, cleanup, ownership, and freshness without iterative convergence. | Promote degree behind an internal distributed gate. Keep local/planned parity tests, active-build freshness tests, failed-build preservation, page-status summaries, cleanup resume, stale attempt rejection, and process replacement coverage. | Degree can be enabled for production maintenance with bounded budgets, durable status, direct/traversal/search freshness behavior, and cleanup that does not grow namespaces. | +| R3. PageRank promotion | PageRank is the reference single-vector iterative metric and the standard for later centrality families. Fixed-iteration non-converged output publishes when requested, with `converged: false` metadata. | Keep the DB-level production-budget parity proof, then finish cross-shard fan-in behavior, public read-surface coverage, deployment-scale parity, and latency evidence. Keep dynamic iteration planning, convergence summaries, publish verification, failed rebuild preservation, cleanup, and process restart coverage as required promotion checks. | PageRank distributed output matches local output within tolerance, serves prior published generations during rebuilds, fails `fresh` reads closed when stale, and preserves the prior generation on failed rebuilds. | +| R4. Eigenvector parity | Eigenvector proves the executor is generic for single-vector iterative metrics. It should reuse PageRank's lifecycle, errors, status, and cleanup rather than adding a metric-specific job system. | Keep the completed process-boundary coverage for page restart/reclaim, publish, cleanup, publish failure, fixed-iteration non-converged publish metadata, and active public freshness across direct top-k, traversal, and search rerank. Keep graph-layer local-vs-planned parity for normalization plus disconnected/reducible graphs. Backfill cross-shard fan-in and any production process evidence still needed for default promotion. | Eigenvector can be promoted only after its restart/failure/cleanup/public-read matrix is equivalent to PageRank's single-vector matrix. | +| R5. HITS paired-vector hardening | HITS remains two named metric scores, authority and hub, but one compatible pair for target generation, convergence, failure, publish, and cleanup. | Keep the completed process-owner coverage for explicit authority and hub contribution, reduce/norm, convergence, publish, cleanup, reclaim, exhausted-attempt, publish-failure, and stale-read behavior. Active and failed public freshness now cover direct top-k, traversal projection/order/filter, and search rerank across coordinator-owned process boundaries, fan-in rejects incompatible default authority/hub status pairs before merging public results, and hosted cross-range tests now prove compatible paired HITS `published`/`fresh` direct metric, traversal projection/order/filter/status, and rerank fan-in plus unpublished/mixed-generation failure. Finish local-vs-planned parity, larger graph latency, remote-worker deployment, and promotion-scale cross-shard fan-in coverage before promoting HITS. Keep authority/hub output attempt-scoped until page completion and publish the pair atomically. | Remote HITS stays disabled by default until either side failing preserves the previous compatible pair, paired publish is atomic, cleanup is resumable, and paired-vector restart coverage matches the PageRank quality bar. | +| R6. Public read and fan-in gate | A metric is not production-ready until every score-bearing read path enforces the same generation contract. | Keep the new hosted paired-HITS `published`/`fresh` cross-range evidence while continuing to cover direct metric top-k, traversal projection/order/filter, search rerank, query profile, any future standalone explain surface, and cross-shard fan-in for `published`, `fresh`, not-ready, stale, active-build, failed-build, incompatible-generation, and incompatible paired-HITS status cases. | `published` always uses the latest complete generation; `fresh` returns `MetricNotReady` before first publish and `MetricStale` when the published generation is behind; fan-in rejects missing, zero, stale, or incompatible generations when comparability is required. | +| R7. Operations and cleanup | V1 cleanup is immediate and aggressive; retention is a future bounded admin option, not part of query semantics. | Make cleanup resumable for score generations, manifests, pages, phase summaries, iteration summaries, attempt namespaces, failure diagnostics, and runtime-owner records. Expose summarized status: phase, iteration, page counts, attempts, cursor presence, progress, last error, owner hash, worker identity count/hash, lease expiry, and takeover counters. | Cleanup never deletes the current published generation, bounds diagnostics, resumes after restart, and prevents graph-metric control/output namespaces from growing without a retention knob. | +| R8. Default promotion | Promote by metric family, not by executor milestone. Local runners remain deterministic CI/debug oracles until distributed parity is routine. | Keep the conservative `auto` gate, collect latency evidence, document operations behavior, update public docs and generated clients only for stable user-facing fields, and widen defaults one family at a time: degree, PageRank, eigenvector, then HITS. | A family becomes default only after parity, process ownership, crash/restart, cleanup, public-read, fan-in, status, operations, and latency-budget evidence all pass. | + +The user-facing interface should stay small for this entire roadmap: + +```json +{ + "metrics": { + "pagerank": { + "kind": "pagerank", + "refresh": "background", + "edge_filter": { "types": ["cites"] }, + "max_iterations": 20, + "tolerance": 0.000001 + } + } +} +``` + +Reads should continue to express freshness as a read policy, not as a job +handle: + +- `published`: return the latest complete generation and include status when + requested, even if a newer rebuild is queued, building, or failed. +- `fresh`: require the current edge generation. Return `MetricNotReady` before + first publish and `MetricStale` when a prior generation exists but is behind. + +The internal interface should stay explicitly different from the public +interface. Coordinators create jobs, append phase pages, make iteration and +convergence decisions, verify publish preconditions, fail builds, and schedule +cleanup. Workers claim pages, renew leases, write cursor progress, write +attempt-scoped output where needed, and complete or fail pages. Workers do not +receive metric configs, create jobs, advance phases, publish generations, or +decide pair-level HITS failure. + +The main non-negotiable invariant for the rest of the work is that workers +execute pages and coordinators publish generations. Workers never receive +caller-supplied metric configs, never create jobs, never advance phases, never +publish, and never decide that a whole build has failed. Any stale process that +lost its runtime lease or page attempt must fail closed before writing, +adopting, completing, or publishing output. + +The planned PageRank runner has moved past the original single-worker +checkpoint. The remaining PageRank work should keep the local runner as an +oracle while hardening the planned lifecycle under restart, retry, cleanup, and +eventual distributed worker execution: + +- `iterate_contributions` reads iteration `n` ranks and aggregate out-degree, + scans the scoped reverse-edge range, and writes deterministic contribution + partials keyed by metric, job id, iteration, target node, and page id. This + executor now exists for first-iteration planned PageRank pages. +- `reduce_ranks` aggregates contribution partials for a target-node range, + applies teleport/base mass and sink handling exactly like the local runner, + writes iteration `n + 1` ranks, and records page-level rank summaries. This + executor now exists for first-iteration planned PageRank pages. +- `check_convergence` compares iteration `n` and `n + 1` ranks for each node + range, writes durable page summaries for `max_delta`, `total_delta`, + `rank_sum`, `score_count`, and `converged`, and leaves iteration advancement + to the coordinator. This executor now exists, and non-final check barriers now + drive dynamic PageRank iteration advancement. +- Dynamic iteration planning creates the next contribution, reduce, and check + pages only after the convergence barrier says another iteration is required. + This now exists for planned PageRank. +- Verified publish copies or adopts the final rank generation into public score + storage only after all PageRank pre-publish phases and the publishable + iteration summary verify. Fixed-iteration planned PageRank publish now + materializes durable rank state into public score output after dynamic + iteration advancement. +- The active planned PageRank runner now drives prepare, scan, initialize, + contribution, reduce, check, publish, and cleanup without test-only manual + page completion, and deterministic local-vs-planned parity coverage exists. +- Planned PageRank can now resume after graph-index reopen once a non-final + check barrier has dynamically planned the next iteration. +- Cleanup removes PageRank job namespaces for out-degree, node membership, + rank, contribution, reduce, phase summary, iteration summary, manifest, page, + and abandoned score records without deleting the current published + generation. Completed planned PageRank jobs now run through the generic + cleanup phase after publish with separate durable pages for out-degree + partials, node membership partials, and final job namespace removal. Failed + planned builds now remove unpublished score generations and job namespaces + while preserving compact failed status and bounded recent failure diagnostics, + and public PageRank failure coverage verifies that the prior published score + generation remains visible after a failed rebuild. + +That PageRank slice is complete when planned PageRank can rebuild from dirty +state under production worker ownership, survive restart at every phase +boundary and iteration boundary, publish the same scores as local PageRank +within tolerance, and preserve the previous published generation on failure. + +The second checkpoint is making the planned runner a real distributed runner. +At that point the metric math should already be correct, so the work is mostly +scheduling and recovery: + +- workers claim only durable pages, never phases +- the coordinator is the sole phase, iteration, publish, and failure authority +- expired leases are reclaimed with bounded attempts +- page progress, completion, and failure validate the durable attempt as well as + the worker id so same-worker replacement owners fence stale prior attempts, + including stale idempotent completion after the replacement attempt completes +- same-worker renewals resume from cursor when possible +- cross-worker recovery can recompute page output from the range start unless + attempt-scoped adoption has been added for that phase +- status reports active pages, attempts, lease owners, cursors, phase progress, + convergence, and last error through stable user-facing fields + +Attempt-scoped output adoption is required for phases whose partial output can +be consumed by a later phase before the page is complete. It also becomes +worthwhile when recomputing a large page is too expensive or when a phase writes +many intermediate keys that are hard to replace atomically. Keep it as an +internal executor capability, not as a public API feature. Degree scan and +iterative contribution pages now implement this capability: same-worker +progress writes to the current attempt namespace, later phases read only +adopted job-scoped partials, page completion adopts the attempt output, and +final cleanup removes abandoned attempt keys with the rest of the job namespace. + +The third checkpoint is promoting cleanup and retention from best-effort +housekeeping into a first-class graph metric phase. V1 should keep aggressive +cleanup as the default: one latest published generation, completed job +namespaces removed immediately when snapshot safety allows, and bounded failed +job diagnostics. A future retention option can be added for debugging, but it +should be opt-in, bounded, and independent from the metric query contract. + +The fourth checkpoint is reusing the iterative runner for non-PageRank metrics. +Eigenvector centrality should be first because it exercises normalization and +non-convergence while still publishing one score vector. HITS should follow +because it requires paired authority/hub output and atomic paired publish. +Neither should add a new query surface; they should be named graph metrics with +metric-specific config, metadata, and convergence behavior. + +The final checkpoint is product promotion: + +- planned degree and planned PageRank become default behind an internal gate +- local-vs-planned parity runs in CI for small deterministic graphs +- multi-worker restart and failure coverage runs before enabling remote workers +- graph traversal/search projection, ordering, filtering, and rerank e2e tests + cover `published`, `fresh`, `MetricNotReady`, and `MetricStale` +- cross-shard top-k and rerank paths verify comparable published generations + or fail closed +- public docs describe graph metric status, freshness, convergence metadata, + non-converged publishes, cleanup defaults, and failure preservation + +After those checkpoints, the graph index metric system is complete enough to +support new centrality algorithms as metric plugins on the same lifecycle +instead of as custom rebuild code. + +#### Remaining Architecture + +The rest of the implementation should be organized around four explicit +contracts: planner, page executor, coordinator, and cleanup runner. + +Planner responsibilities: + +- resolve the current metric config and target edge generation +- create an idempotent manifest for one metric/job/generation +- create stable page ids and page ranges for every planned phase +- estimate `total_units` for status without requiring exact load balance +- refuse to reuse a manifest whose config fingerprint no longer matches +- plan future iteration pages only after convergence says another iteration is + required + +Page executor responsibilities: + +- claim only leased pages assigned to the worker +- read page range and cursor metadata from durable page records +- write deterministic output keys scoped by metric, job, generation, phase, and + iteration +- periodically persist cursor and completed unit progress +- complete pages with output fingerprints and page-level summaries +- make retries safe by overwriting the same output key range or by writing into + an attempt namespace that is atomically adopted on completion + +Coordinator responsibilities: + +- acquire or resume the active job for each dirty metric +- decide the next phase only from durable summaries, not in-memory worker state +- summarize completed pages into phase and iteration summaries +- plan the next phase or next iteration after a successful barrier +- run verified publish in the same transaction as the published pointer flip +- mark failed jobs without hiding the previous published generation +- expose user-facing status from job, manifest, page, phase, and iteration + records + +Cleanup runner responsibilities: + +- delete old published score generations after snapshot safety allows +- delete intermediate job namespaces for completed and failed jobs +- delete abandoned building generations that never published +- preserve bounded diagnostic records for recent failed jobs +- resume cleanup after restart without requiring the original build worker + +The first production version should keep these contracts internal. The public +API should remain metric config, freshness selection, score reads, graph query +projection/order/rerank, and status. Operational controls such as explicit +pause, resume, retry policy, and retention can be added after the distributed +runner is proven. + +#### Implementation Stages + +**Stage A: Planned Degree Reduce** + +`degree` is the non-iterative proof of the distributed execution model. Scan +pages no longer write final score generation entries directly. They write +deterministic partials under the job namespace, keyed by metric, job id, node, +and page id. A `reduce_ranks` page aggregates partials into the building score +generation. + +This keeps retry behavior simple: + +- retrying a scan page overwrites that page's partial keys +- retrying reduce recomputes final scores from completed partials +- verified publish only sees final score generation output +- cleanup can remove the whole completed job namespace after publish + +Stage A exit criteria: + +- planned degree includes `prepare_generation`, `scan_edges_and_out_degree`, + `reduce_ranks`, `publish_generation`, and `cleanup_old_generations` +- one-page planned degree still matches local degree output +- two or more scan pages can contribute to the same node and reduce to the + correct final degree +- scan-page retry does not double-count +- verified publish requires scan and reduce barriers before flipping the + published generation + +**Stage B: Range-Partitioned Manifests** + +Replace one-page manifests with deterministic key-range pages. Degree scan +planning now uses contiguous ranges over encoded reverse-edge keys, degree +reduce planning now uses deterministic node ranges, degree cleanup uses durable +prefix pages, and initial PageRank manifests use the same reverse-edge and node +ranges for scan, initialize, contribution, reduce, and convergence phases. The +remaining partitioning work is dynamic iteration-specific PageRank pages after +each convergence barrier. Exact load balance is less important than stable page +ids and restart-safe replay. + +Manifest/page schema already has the right shape and should be used as the +contract: + +- opaque lower and upper range bounds +- range kind, such as `reverse_edges`, `nodes`, `scores`, `contributions`, or + `job_control` +- planned phase and iteration +- output namespace prefix +- cursor, completed units, and total units + +The planner should be idempotent for the same metric config fingerprint and +target edge generation. A changed config fingerprint should make the old plan +ineligible instead of mutating it in place. + +Stage B exit criteria: + +- reopening the index resumes the same manifest and page ranges +- planning the same metric/target generation twice is idempotent +- scan pages cover the intended reverse-edge keyspace without overlap gaps +- reduce pages cover the intended node/score keyspace without overlap gaps +- status reports page progress without exposing raw key bytes + +**Stage C: Resumable Page Progress** + +Long pages should persist cursors often enough that lease renewal can continue +from the last durable point. Degree scan pages now have the first implementation +of this contract: they persist an opaque reverse-edge key cursor, resume after +that key on same-worker renewal, and merge newly scanned partial counts into the +same page's job-scoped partial output. Cross-worker recovery for degree scans +recomputes from the page range start after lease reclaim and overwrites the same +page partial keys without double-counting. Attempt-scoped output adoption can +come later if recompute cost is too high. + +Cursor rules: + +- same-worker renewal resumes from the persisted cursor +- expired-lease reclaim increments attempt count +- a reclaimed scan or reduce page may recompute from the range start +- completed pages are immutable except for idempotent same-fingerprint + completion +- exhausted attempts mark the page failed and make the phase barrier fail + +Stage C exit criteria: + +- killing a worker during degree scan or reduce either resumes or safely + recomputes +- page progress is durable across graph-index reopen +- failed pages include metric, job id, phase, iteration, page id, worker id, + attempt, and last error +- `published` reads continue to serve the previous generation while a new + generation is building +- `fresh` reads fail with `MetricStale` until the new generation publishes + +**Stage D: Generic Page Executor And Worker Loop** + +Generalize the degree-specific worker-step loop into a registry keyed by metric +kind and phase. Workers should only claim pages and run executors. The +coordinator should remain the only authority for phase summaries, iteration +decisions, publish, failure, and cleanup. + +The first version of this split now exists for planned degree. The degree API +still exposes the same planned runner, but internally it delegates to a generic +worker step that loads the durable job, routes executable phases through a +metric/phase page executor, and keeps `publish_generation` as coordinator work. +`prepare_generation`, `scan_edges_and_out_degree`, `reduce_ranks`, and +`cleanup_old_generations` return a shared page execution result shape so +PageRank can add its own phase executors without adding another worker loop. + +Worker loop: + +```text +claim eligible page +load executor for metric kind and phase +execute page range +persist progress cursor +complete page with fingerprint and summary +repeat +``` + +Executor result: + +```text +PageExecutionResult { + completed_units, + total_units, + output_fingerprint, + optional max_delta, + optional total_delta, + optional rank_sum, + optional score_count, + optional next_cursor +} +``` + +The coordinator interprets summaries by phase. Page executors should not decide +that a job is publishable. + +Stage D exit criteria: + +- degree uses the generic worker loop without hardcoded page ids +- the retry policy is internal and bounded +- duplicate coordinator ticks cannot advance incompatible phase state +- status exposes phase, iteration, completed pages, total pages, active leases, + attempts, cursor, and last error + +**Stage E: Durable PageRank Executor** + +Move PageRank onto the same runner using durable intermediate namespaces. This +is the first iterative metric on the planned executor and should preserve the +existing local PageRank semantics exactly. + +The initial PageRank manifest now creates partitioned durable pages for the +first scan, initialize, contribution, reduce, and convergence phases. The first +scan/out-degree executor writes job-scoped out-degree partials and node +membership keys through the generic page lease path. The initialize executor +reloads durable page range metadata, derives the filtered node set from scan +partials, writes aggregate out-degree state, and writes iteration-0 ranks. The +contribution executor reads iteration ranks plus aggregate out-degree and +writes attempt-scoped durable edge contribution partials that become +reduce-visible only after page completion adopts the attempt output. Scan, +contribution, reduce, and convergence pages can persist cursor progress; +contribution pages now resume after graph-index reopen and merge same-worker +partial output into the current attempt on initial and later iterations. The +reduce executor applies the local PageRank base, sink, and contribution +formula, writes iteration `n + 1` ranks, and now resumes node-range progress +after reopen on initial and later iterations. The convergence executor compares +adjacent rank iterations, +writes durable `max_delta`, `total_delta`, `rank_sum`, and convergence +summaries, and now resumes partial convergence progress after reopen before +completing the phase barrier on initial and later iterations. Non-final checks +now plan later contribution/reduce/check pages; fixed-iteration planned publish +materializes final rank state into public score storage and runs generic job +namespace cleanup. Expired scan, contribution, and reduce pages now have reclaim +coverage proving stale partial out-degree, node membership, contribution, and +rank output is overwritten instead of accumulated. The remaining work is to +broaden parity, cleanup, and failure coverage for full planned PageRank +generations. + +Required phases: + +```text +prepare_generation +scan_edges_and_out_degree +initialize_ranks +iterate_contributions +reduce_ranks +check_convergence +publish_generation +cleanup_old_generations +``` + +Required intermediate namespaces: + +```text +metric_out_degree/// +metric_rank//// +metric_contribution///// +metric_reduce//// +metric_score/// +``` + +Iteration control: + +- `check_convergence` aggregates page summaries into a durable iteration + summary +- if converged, the coordinator advances to publish +- if not converged and iterations remain, the coordinator plans the next + contribution/reduce/check pages +- if `max_iterations` is reached, publish fixed-iteration output with + `converged: false` + +Stage E exit criteria: + +- planned PageRank matches local PageRank within documented tolerance +- restart coverage exists for every phase and iteration boundary +- convergence summaries drive every iteration transition +- invalid score output fails the job and preserves prior published scores +- `MetricNotReady`, `MetricStale`, `published`, and `fresh` behavior is + unchanged + +**Stage F: Coordinator-Owned Publish And Cleanup** + +Workers complete pages; the coordinator publishes. The publish transaction must +verify active job, manifest, summaries, page fingerprints, config fingerprint, +score metadata, and convergence metadata before flipping the published pointer. + +Cleanup should be aggressive in v1: + +- keep the latest published generation +- delete older generations when snapshot safety allows +- delete completed job manifests, pages, phase summaries, iteration summaries, + contributions, reduces, ranks, out-degree state, and abandoned building score + namespaces +- preserve only bounded diagnostic records for recent failed jobs + +Planned PageRank now uses prefix-partitioned cleanup for the completed job path: +one page removes out-degree partials, one page removes node membership partials, +and the final page removes the remaining job namespace and clears the active +lease. Restart coverage now verifies that a build can stop after a non-final +cleanup page, reopen with the published generation still queryable, and finish +the remaining cleanup pages. Large cleanup prefixes now persist a cursor and can +resume after reopen before the cleanup page is marked complete. Failed planned +build cleanup now removes unpublished building score generations, deletes the +job namespace, clears the active lease, and preserves a compact failed job +record plus bounded recent failure diagnostics for status. Larger +attempt-scoped output namespaces still need the same treatment. + +Stage F exit criteria: + +- crash before publish leaves the old generation visible and can resume publish +- crash after publish but before cleanup leaves the new generation visible and + cleanup resumes later +- cleanup never deletes the currently published generation +- failed builds preserve the prior published generation +- failed planned builds delete unpublished score output and active job keys + while retaining compact failure diagnostics +- completed and failed job namespaces do not grow without bound + +**Stage G: Multi-Worker Scheduling** + +Enable production worker ownership only after the planned executor and +in-process split-runtime contract are stable. The current DB graph-metric +runtime already provides the bridge: it can tick the same durable +planned-maintenance primitive as `combined`, `coordinator`, `worker`, or +`worker_pool`, starts from DB open when enabled, wakes from derived-apply +notifications, reports role/owner/worker telemetry, can publish through +separate coordinator and worker loops without manual tick calls, and can also +be initialized from `OpenOptions.graph_metric_maintenance` with +`start_background_loop = false` so process-style tests can drive explicit +runtime ticks from fresh DB handles for degree, PageRank, eigenvector, and +compatible HITS. The new `antfly graph-metric-maintenance` command exposes that +same configured runtime path as an executable process role with bounded polling +and idle-exit controls, and its `supervise` mode drives coordinator and +worker-pool child role processes in bounded rounds, parses their durable +progress summaries, exits after global idle, and applies bounded restart +policy. The full binary command path now builds, and command-level tests drive +degree through supervisor-built coordinator and worker-pool child argv against +one shared DB path. The integration-test aggregate now runs the +real binary through supervisor mode with spawned child roles and verifies that +degree, PageRank, eigenvector, and compatible HITS publish fresh output through +durable DB state only. The same process harness now kills runtime and page +owners across coordinator, worker, and worker-pool roles; proves duplicate +owners are fenced before lease expiry; proves replacements take over after +expiry; rejects stale page attempts after replacement; verifies publish, +failure, cleanup, fixed-iteration, exhausted-attempt, and same-worker fencing +invariants; and emits a final `graph_metric_process_harness_summary` only after +all required owner-boundary coverage categories are observed. The +full-default workflow runs that process harness through the ordinary +`integration-test` aggregate. Release-sized public-read, fan-in, +retained-storage, scheduler, and latency evidence remains rollout readiness +work rather than another graph-only CI target. + +Remote workers should claim independent pages from durable graph-index state by +index and metric name. They should persist cursors, complete or fail leases, +and stop. They should not receive metric configs from callers, create jobs, +advance barriers, fail active builds, publish generations, or clean completed +jobs outside cleanup pages. Coordinators should start and resume jobs, advance +phase and iteration barriers, append dynamic iteration pages, verify publish, +mark failed builds, and schedule cleanup based only on durable summaries. +Retry, lease, backoff, and attempt limits should remain internal defaults in +the first version. + +Stage G exit criteria: + +- independently owned coordinator and worker processes can finish one metric + generation without duplicate publish attempts +- worker loss abandons only the leased page, not the build +- another worker can reclaim an expired page lease and either resume from + cursor or recompute without stale partial output +- duplicate or racing coordinators cannot publish conflicting generations or + append incompatible phase pages +- status and runtime telemetry show role, owner, phase, iteration, page + progress, attempt, cursor presence, and last error without exposing raw key + ranges + +**Stage H: Eigenvector And HITS** + +Eigenvector centrality and HITS should reuse the iterative executor rather than +adding another job system. Eigenvector proves the executor is not PageRank +specific. HITS proves paired vector materialization and atomic paired publish. + +Current state: eigenvector has a planned single-vector runner with local parity, +durable contribution/reduce cursor resume after reopen, reclaimed +contribution/reduce stale-output overwrite coverage, and later-iteration failed +contribution/reduce/convergence page retry through the generic worker step. +Planned eigenvector cleanup can also resume after reopen while published scores +remain visible, and failed planned eigenvector rebuilds preserve the previous +published generation. HITS has a first planned paired-vector runner that +computes authority and hub ranks from durable job state, publishes compatible +authority/hub generations atomically, matches the local runner on deterministic +coverage, resumes authority and hub contribution/reduce cursor progress after +reopen, and retries +failed later-iteration contribution, reduce, hub contribution, hub reduce, and +convergence pages through the generic worker step. Later-iteration exhausted hub +reduce attempts fail the compatible pair while keeping the previous +authority/hub generations visible. Failed planned HITS rebuilds also mark the +compatible pair failed together, and reclaimed contribution/reduce pages +overwrite stale durable authority/hub job output while reclaimed convergence +pages reset stale partial summaries. The planned HITS phase model now also +separates hub contribution and hub reduce/norm from authority reduce/norm: hub +raw output is written through attempt-scoped hub contribution pages and adopted +only after page completion, then hub reduce pages normalize the adopted raw +namespace. The planned cleanup page can also resume after reopen with the +published pair still visible. The remaining work is not a new job system; it is +production hardening of the same runner shape, especially phase-specific HITS +restart/failure coverage under the new hub phases. + +Eigenvector requirements: + +- reuse scan, initialize, iterate, reduce, check, publish, and cleanup +- persist normalization metadata per iteration +- document behavior for disconnected and reducible graphs +- publish fixed-iteration non-converged output with `converged: false` + +HITS requirements: + +- compute authority and hub vectors over the same target generation and edge + scope +- persist paired manifests or one manifest with paired output metadata +- publish authority and hub together when configured as a compatible pair +- fail the pair together if one side has invalid output, while preserving the + previously published pair +- partition paired-vector work fully, including hub reduction work that is still + broader than the authority contribution pages in the first planned runner +- broaden restart, lease-expiry, reclaimed-output, publish-failure, and cleanup + coverage across HITS prepare, scan, initialize, contribution, reduce, + convergence, publish, and cleanup +- keep reclaimed contribution/reduce coverage as the minimum proof for stale + paired-vector output overwrite, and convergence-summary reset coverage as the + minimum proof for stale convergence metadata, before adding broader publish + and cleanup reclaim tests +- keep cleanup cursor-resume coverage as the minimum proof that partially + cleaned HITS job namespaces can finish after restart without hiding the + published pair + +Stage H exit criteria: + +- at least one non-PageRank iterative metric reuses the distributed executor +- HITS paired publish preserves atomic visibility +- local and distributed outputs match within documented tolerance +- paired-vector restart and retry tests preserve the previous published pair, + including failed-page retry and planned-build failure cases +- no new public query/status API is required + +**Stage I: Promotion** + +Rollout should be controlled by internal gates and parity checks: + +1. keep local runners as deterministic CI/debug oracles +2. keep explicit planned maintenance and the conservative `auto` gate for + already-active planned work, queued degree, small queued PageRank, bounded + queued eigenvector, and internally opted-in compatible HITS +3. measure and promote latency-safe idle budgets for broader default + planned-maintenance scheduling +4. run local multi-worker and split-runtime coverage for every promoted metric + family +5. enable real remote workers for degree and PageRank behind internal gates +6. broaden eigenvector restart/failure parity until it matches PageRank's + single-vector matrix +7. finish HITS hub phase restart/failure coverage +8. enable remote/distributed workers for large graph indexes per metric family +9. demote local runners only after planned/distributed parity, restart, + cleanup, freshness, operations, and cross-shard tests pass + +Promotion exit criteria: + +- public e2e tests cover direct metric top-k, graph projection, graph ordering, + graph search rerank, status, and freshness +- failed builds preserve published scores across restart +- dirty markers survive restart and eventually rebuild +- cleanup prevents unbounded growth of failed/intermediate job keys +- cross-shard metric fan-in either proves comparable nonzero generations or + fails closed +- generated clients and public docs describe metric status, freshness, + convergence, fixed-iteration non-converged output, cleanup defaults, and + failure preservation + +#### 1. Verified Publish Gate + +The metadata-only publish verifier now exists as the first half of this slice. +It does not yet need fully distributed score execution. Its job is to make the +final pointer flip depend on the manifest, phase summaries, page records, +iteration summaries, score metadata, and current metric config. + +Before publishing a distributed build, verify: + +- the active build job is at `publish_generation` +- the active build job and manifest agree on `job_id`, target edge generation, + score generation, planned phase/page counts, and config fingerprint +- the manifest's target generation is still eligible to publish for the current + resolved metric config +- every required pre-publish phase has a complete phase summary +- every planned page for those phases is complete and has a stable output + fingerprint +- iterative metrics have a complete convergence summary for the publishable + iteration +- fixed-iteration non-converged metrics record `converged: false`, + `iterations_completed`, and the final delta values +- score metadata has valid counts and no NaN or infinity +- paired publishes, such as HITS authority/hub, have compatible manifests and + publish in one transaction + +The remaining work in this slice is to attach that verifier to the publish +transaction. The transaction should then write the same public metadata as the +local runner: + +- score metadata +- edge-filter metadata +- config fingerprint metadata +- published generation pointer +- publish event +- cleanup eligibility records for old generations and intermediate job keys +- completed job status + +Exit criteria: + +- Query readers either see the old generation or the new generation, never a + mix. +- A crash after score writes but before publish resumes verification and either + publishes or leaves the old generation visible. +- A crash after publish but before cleanup leaves the new generation queryable + and schedules cleanup on restart. +- Cross-shard top-k and rerank fan-in continue to fail closed unless shard + responses report comparable nonzero published generations. + +#### 2. Partition Model + +The current manifest intentionally plans one page per phase. That is enough to +prove durable metadata, but not enough for scale. The next planner version +should create stable key-range pages over graph edge and node namespaces for a +target edge generation. + +Initial partitioning should be simple and deterministic: + +- edge scan pages are contiguous ranges over encoded edge keys +- node/rank pages are contiguous ranges over encoded node ids discovered during + scan +- contribution pages are keyed by target node range and iteration +- reduce pages aggregate one target range at a time +- cleanup pages cover score, contribution, reduce, and job-control key ranges + +The planner does not need perfect load balance in v1. It needs stable +boundaries, restart-safe page ids, and progress counters that can be explained +in status. Later versions can split hot pages or rebalance based on observed +unit counts. + +Exit criteria: + +- Reopening the graph index resumes the same manifest and page boundaries. +- Planning the same metric/target generation twice is idempotent. +- A config fingerprint change makes the old plan ineligible and schedules a new + target generation. +- Status reports phase, iteration, page counts, cursor, completed units, total + units, and lease owner without exposing raw internal key ranges. + +#### 3. Executable Degree Job + +`degree` is the first metric to run through the planned coordinator. It is +non-iterative, cheap to validate against the existing local runner, and +exercises the same publish and cleanup invariants without PageRank's iteration +loop. + +The degree job should execute: + +```text +prepare_generation +scan_edges_and_out_degree +reduce_ranks +publish_generation +cleanup_old_generations +``` + +The first implementation can still run with one local worker, but it should use +the same partial/reduce contract needed for many scan pages. Scan pages write +job-scoped partial degree counts. The reduce page aggregates completed partials +and writes final score entries into the building score generation. Retrying a +scan page overwrites that page's partial keys; retrying reduce recomputes final +scores from the completed partial namespace. + +Exit criteria: + +- Single-worker planned degree output matches local degree output. +- Multiple scan pages can contribute partial counts for the same node without + double-counting. +- Killing the worker during scan or reduce retries the page and produces the + same published generation. +- Publish verification requires both scan and reduce phase summaries. +- `published` reads continue to serve the old generation while the job is + active. +- `fresh` reads fail with `MetricStale` until the new generation publishes. +- Cleanup removes old degree score generations, completed partials, and stale + job keys safely. + +#### 4. Executable PageRank Job + +After degree proves the coordinator, PageRank should move onto the same page +executor path. PageRank needs durable intermediate storage because each +iteration reads the previous rank state and writes contribution/reduce output +before convergence can be checked. + +Required intermediate namespaces: + +```text +metric_score/// +metric_out_degree/// +metric_rank//// +metric_contribution///// +metric_reduce//// +``` + +Required phases: + +```text +prepare_generation +scan_edges_and_out_degree +initialize_ranks +iterate_contributions +reduce_ranks +check_convergence +publish_generation +cleanup_old_generations +``` + +`check_convergence` reduces page-level summaries into the durable iteration +summary. If the summary is converged, the job moves to publish. If it is not +converged and another iteration is allowed, the coordinator plans the next +iteration's contribution/reduce/check pages. If `max_iterations` is reached, it +publishes bounded fixed-iteration output with `converged: false`, matching the +current local behavior. + +Exit criteria: + +- Single-worker planned PageRank output matches local PageRank within the + documented floating-point tolerance. +- Restart during every phase either resumes or safely retries the current page. +- No phase can advance until every required page in that phase is complete. +- Invalid score output fails the job and leaves the prior published generation + visible. +- Non-converged fixed-iteration output publishes with complete convergence + metadata. + +#### 5. Multi-Worker Coordinator + +Once degree and PageRank work with one planned worker, the coordinator can allow +multiple workers to claim pages concurrently. Workers should operate on page +leases only; the coordinator owns phase summaries, iteration decisions, publish, +failure, and cleanup. + +Worker loop: + +```text +claim next eligible page +execute page with deterministic output keys +update cursor and completed units during long work +complete page with output fingerprint +coordinator summarizes phase +coordinator advances phase, plans next iteration, publishes, or fails +``` + +Recovery rules: + +- expired leased pages can be reclaimed after lease timeout +- page attempt increments on explicit failure or expired-lease reclaim +- completed page output is idempotent by fingerprint +- repeated page failures mark the job failed after a bounded internal policy +- failed jobs preserve the previous published generation and expose page-level + failure details + +The first retry policy should stay internal: + +```text +max_page_attempts = 3 +lease_timeout_ms = existing local lease timeout initially +backoff = fixed small delay or coordinator tick interval +``` + +Do not expose retry policy as user configuration in the first distributed +version. It can become an internal tuning option after the behavior is stable. + +Exit criteria: + +- Multiple workers can finish one build without duplicate publish attempts. +- Worker loss only abandons a lease, not the generation. +- Build progress is monotonic within a phase except after lease recovery, where + retry behavior is visible in status. +- Failure status includes phase, page id, attempt, worker id, and last error. + +#### 6. Eigenvector and HITS on the Distributed Runner + +Eigenvector centrality and HITS should not get separate job systems. They should +reuse the PageRank iteration machinery with metric-specific math and metadata. +The current planned HITS slice already uses that shared runner, proves atomic +paired authority/hub publish against the local implementation, verifies the same +metric-name planned-drain loop used by degree, PageRank, and eigenvector can +reach a fresh paired generation through worker/coordinator boundaries, and +verifies failed later-iteration contribution, reduce, hub contribution, hub +reduce, and convergence pages can retry through the generic worker step. +The same planned HITS coverage now resumes explicit hub contribution and hub +reduce cursor progress across graph-index reopen before completing the paired +iteration. +Later-iteration exhausted hub reduce attempts also fail the compatible metric +pair together while preserving the previous published pair, and reclaimed +contribution/reduce pages overwrite stale durable paired-vector output while +reclaimed convergence pages reset stale partial summaries. Planned cleanup can +resume after reopen with the published pair visible. The rest of this roadmap is +production completeness: +restart coverage, lease expiry, reclaimed-output behavior, paired-vector +partitioning, and distributed worker rollout. + +Eigenvector requirements: + +- use the same scan, initialize, iterate, reduce, check, publish, and cleanup + shape as PageRank +- persist normalization metadata per iteration +- document behavior for disconnected and reducible graphs +- publish fixed-iteration non-converged output with `converged: false` + +HITS requirements: + +- compute authority and hub vectors over the same target generation and edge + scope +- persist paired manifests or one manifest with paired output metadata +- publish authority/hub together when configured as a compatible pair +- fail the pair together if one side has invalid output +- partition both authority and hub work into retryable pages before enabling + large remote builds by default +- preserve the previous published authority/hub pair after failed, expired, or + reclaimed pages; planned-build failure is covered, while expiry and reclaimed + output still need broader production coverage beyond contribution/reduce and + convergence summaries + +Exit criteria: + +- At least one non-PageRank iterative metric reuses the distributed executor + without new public query/status APIs. +- HITS paired publish preserves the atomic visibility invariant. +- Local and distributed outputs match within documented tolerance. +- Restart and retry coverage exists for paired-vector dynamic iterations, + publish failure, and cleanup. + +#### 7. Cleanup and Retention + +V1 cleanup should remain aggressive: keep the latest published generation and +delete old generations as soon as snapshot safety allows. Distributed execution +adds intermediate keys that also need cleanup. + +Cleanup should cover: + +- previous score generations +- stale job manifests +- contribution pages +- reduce pages +- out-degree and rank state pages +- per-page lease/status records +- abandoned score generations that never published + +Use the existing resolved default: + +- immediate cleanup when graph metric reads are snapshot-safe +- deferred cleanup queue when snapshot safety cannot be proven +- no user-facing retention knob in the first distributed version + +Future configuration may add retention for debug or audit workflows, but it +should be opt-in and bounded: + +```json +{ + "metrics": [ + { + "name": "pagerank", + "kind": "pagerank", + "retention": { + "published_generations": 2, + "failed_job_manifests": 5 + } + } + ] +} +``` + +Exit criteria: + +- Cleanup never deletes the currently published generation. +- Cleanup can resume after restart. +- Failed or abandoned jobs do not grow storage without bound. +- Debug retention, if later added, is bounded and explicitly configured. + +#### 8. Test Matrix and Rollout + +Testing should be staged with the implementation. Do not wait for the whole +distributed runner before adding coverage. + +Required tests by slice: + +- verified publish rejects incomplete or mismatched manifests +- planner idempotence and manifest restart +- page lease claim, completion, expiry, and retry +- idempotent page overwrite +- restart between every phase +- restart after all score writes but before publish +- restart after publish but before cleanup +- stale reads while a distributed build is active +- fresh reads failing with `MetricStale` until publish +- failed distributed build preserving old top-k results +- non-converged fixed-iteration publish +- local-vs-distributed parity for degree, PageRank, eigenvector, and HITS +- cross-shard direct top-k generation guard +- cross-shard rerank generation guard + +Rollout should use feature gates internally: + +1. Keep the local runner as default. +2. Add verified publish and cleanup validation under local builds. +3. Run planned degree with one local worker and compare output to the local + runner. +4. Run planned PageRank with one local worker and compare output to the local + runner. +5. Enable multiple local workers in tests. +6. Enable remote/distributed workers for large graph indexes. +7. Broaden planned eigenvector and HITS restart/failure coverage, including + paired-vector stale-read and previous-publish preservation tests. +8. Remove or demote the old local runner only after the distributed path has + parity tests for degree, PageRank, eigenvector, and HITS. + +Production-complete criteria for this roadmap: + +- PageRank supports local and distributed materialization. +- Degree, eigenvector, and HITS reuse the same distributed executor model. +- Direct metric top-k, graph projection, graph ordering, graph search rerank, + and status APIs have public e2e coverage. +- Failed builds preserve published scores across restart. +- Dirty markers survive restart and eventually rebuild. +- Cross-shard direct metric top-k has deterministic merge behavior, and + retrieval/rerank metric merges either prove globally comparable generations or + fail closed. +- OpenAPI, generated clients, and public docs describe freshness semantics, + phase progress, convergence metadata, and failure status. + +### Remaining Delivery Roadmap + +The remaining work is a rollout and production-hardening roadmap, not a new +user-facing API. Users should continue to configure named graph metrics on graph +indexes and choose `published` or `fresh` on reads. Coordinators, workers, jobs, +leases, attempts, and cleanup remain internal maintenance machinery. + +The rest of the design should keep three surfaces separate: + +1. **User query surface**: graph metrics stay on graph indexes. Reads name the + metric and choose `metric_freshness: "published"` or + `metric_freshness: "fresh"`. `MetricNotReady` means no complete generation + exists; `MetricStale` means a complete generation exists but does not match + the current edge generation. Direct top-k, traversal projection/order/filter, + search rerank, and query profile should all expose the same generation + contract. +2. **Admin/operations surface**: v1 exposes enough status to answer "what is + building, who owns it, how far did it get, and why did it fail?" without + exposing raw storage prefixes. Pause, resume, priority, manual retry, lease + tuning, and retention are future admin controls. They should not be required + for query correctness or ordinary cleanup. +3. **Executor surface**: coordinators and workers communicate through durable + graph-index state. Coordinators create jobs, advance barriers, publish, fail, + and schedule cleanup. Workers lease and complete pages only. Runtime-owner + leases fence process roles; page leases fence work units; attempt namespaces + fence partial output adoption. + +The near-term PR split should be: + +| PR lane | Purpose | First useful deliverable | +| --- | --- | --- | +| Remote owner orchestration | Move from local process proofs to deployment-owned coordinator and worker roles. | Degree and PageRank can drain through independently managed owners using only durable graph-index state or the production service boundary. | +| Degree default gate | Use degree as the first low-risk distributed default. | Internal gate can route degree through planned maintenance with parity, cleanup, status, restart, and public freshness coverage. | +| PageRank promotion gate | Make PageRank the reference single-vector iterative metric. | Production-budget parity, larger graph evidence, fixed-iteration metadata, failed rebuild preservation, and cross-shard read compatibility all pass together. | +| Eigenvector parity | Prove the single-vector executor is generic. | Eigenvector uses the same lifecycle, errors, status, cleanup, freshness, and fan-in gates as PageRank. | +| HITS paired-vector gate | Prove paired metrics can share one lifecycle safely. | Authority and hub publish/fail atomically and preserve the previous compatible pair under restart, failure, and cleanup. | +| Operations contract | Make distributed metric work explainable and bounded. | Status exposes summarized role/owner/progress/failure/cleanup facts; cleanup and diagnostics remain bounded without a public retention knob. | +| Public fan-in contract | Prevent mixed-generation scoring from becoming a distributed correctness bug. | Every score-bearing read path either proves compatible nonzero published generations across shards or fails closed. | + +The implementation should move in this order: + +| Gate | Design target | Work remaining | Exit signal | +| --- | --- | --- | --- | +| 1. Production process ownership | Coordinators and workers are independently managed durable-state clients. | Move from local supervisor/process proof to deployment-managed owners. Owners receive endpoint or DB location, role, owner id, worker id, tick budgets, lease policy, and idle policy only. Metric config, target generation, manifests, page ranges, attempts, publish decisions, failure, and cleanup stay inside graph-index state. Command argv coverage now locks that boundary down for launched coordinator and worker-pool children: no metric/index names, target generations, job/page ids, metric configs, or direct file-backed writer-lock guard are present in production-style child argv. The spawned-process harness applies the same preflight to standalone coordinator/worker/worker-pool role processes before ordinary restart/reclaim runs and killable long-lived lease-loss owners are spawned. Command-summary coverage pins the stable operations telemetry emitted by those owners, including role, runtime/owner/worker hashes, lease-key hash, worker count, acquisition/takeover/lost-lease counters, tick progress, idle/error counters, and last error; aggregate supervisor/launcher summaries now preserve compact per-child telemetry for those same fields; standalone process-role coverage now asserts the same role/owner/worker/lease-key/tick/error telemetry for every coordinator, worker, and worker-pool process used in restart and reclaim proofs. | A coordinator and multiple workers can start, stop, crash, restart, and drain degree plus PageRank without shared memory, caller-supplied metric config, or local test-only writer serialization. Duplicate owners are fenced by runtime leases; clean shutdown releases the current owner lease immediately; stale page attempts cannot write, adopt, complete, publish, or fail. | +| 2. Degree canary | Degree is the first production-distributed metric because it proves scan, reduce, publish, cleanup, leases, attempts, and freshness without iterative convergence. | Promote planned/distributed degree behind an internal gate. Keep local-vs-planned parity, process replacement, stale attempt rejection, cleanup resume, active/failed status, page-status summaries, and public freshness checks in CI. | Degree can run through distributed maintenance with bounded latency, bounded diagnostics, no control-namespace growth, and rollback to the local oracle. | +| 3. PageRank promotion | PageRank is the reference single-vector iterative family. | Keep the DB-level production-budget parity proof, then finish larger-graph evidence, cross-shard fan-in checks, and public read-surface coverage. Keep dynamic iteration planning, convergence summaries, fixed-iteration non-converged publish metadata, publish verification, failed rebuild preservation, cleanup, and per-phase restart/reclaim tests as required promotion checks. | Distributed PageRank matches local output within tolerance, serves prior published generations during rebuilds, fails `fresh` reads while stale, preserves the prior generation on failed rebuilds, and recovers from restart at every phase boundary. | +| 4. Eigenvector parity | Eigenvector should reuse the PageRank single-vector substrate rather than adding a metric-specific lifecycle. | Keep the existing process-boundary, fixed-iteration, active-freshness, disconnected/reducible parity, publish-failure, cleanup coverage, and DB-level production-budget parity proof. Add the same cross-shard fan-in, deployment-scale parity, and operations evidence required for PageRank before default promotion. | Eigenvector is promoted only when its restart, failure, cleanup, freshness, fan-in, status, and operations matrix is equivalent to PageRank's single-vector matrix. | +| 5. HITS paired-vector hardening | HITS authority and hub are separate named scores but one compatible pair for target generation, convergence, failure, publish, and cleanup. | Finish paired local-vs-planned parity, larger-graph latency evidence, deployment-scale owner evidence, public read-surface coverage, and promotion-scale cross-shard fan-in coverage. Keep the default authority/hub pair compatibility guard in direct metric and graph traversal/search status fan-in plus the hosted paired-HITS `published`/`fresh` cross-range direct, traversal projection/order/filter/status, rerank evidence, and service-owner replacement proof. Keep authority/hub output attempt-scoped until page completion and publish/fail the pair atomically. | HITS stays disabled by default until either side failing preserves the previous compatible pair, paired publish is atomic, cleanup is resumable, and paired-vector restart coverage matches the PageRank quality bar. | +| 6. Public read and fan-in gate | Distributed execution is not complete until every score-bearing read path enforces the same generation contract. | Keep hosted paired-HITS `published`/`fresh` cross-range success/failure coverage while covering direct metric top-k, graph projection, graph ordering, graph filtering, graph search rerank, query profile, any future standalone explain surface, and shard fan-in for `published`, `fresh`, not-ready, stale, active-build, failed-build, missing-generation, zero-generation, incompatible-generation, and incompatible paired-HITS status cases. | `published` reads use the latest complete generation; `fresh` returns `MetricNotReady` before first publish and `MetricStale` when published is behind; fan-in rejects missing, zero, stale, or incompatible generations when comparability is required. | +| 7. Operations and cleanup | V1 cleanup is aggressive and bounded. Retention remains a future admin/debug option, not query semantics. | Make cleanup resumable for score generations, manifests, pages, phase summaries, iteration summaries, attempt namespaces, failure diagnostics, and runtime-owner records. Status should summarize phase, iteration, page counts, attempts, cursor presence, progress, last error, owner hash, worker identity count/hash, lease expiry, and takeover counters; active-page status coverage now requires capped building payloads to include leased page state, worker identity, lease expiry, attempt, cursor/error fields, progress units, and finite aggregate progress; cache-preservation coverage now verifies graph metric runtime ownership telemetry survives status refreshes that only carry synthetic or sequence-only table state, graph index status now exposes a typed `graph_metric_runtime` OpenAPI summary with role, owner/worker hashes, lease state, takeover/lost-lease counters, tick progress, and page counters, command-summary coverage proves standalone role processes emit the same stable ownership/progress/error fields for operators, and aggregate supervisor/launcher coverage carries that telemetry up to the orchestration summary. | Cleanup never deletes the current published generation, resumes after restart, bounds diagnostics, and prevents completed, failed, abandoned, unpublished, and attempt-scoped state from growing without a retention knob. | +| 8. Default promotion | Promote by metric family, not by executor milestone. Local runners remain deterministic CI/debug oracles until distributed parity is routine. | Widen the conservative `auto` gate one family at a time: degree, PageRank, eigenvector, then HITS. The PageRank and eigenvector auto thresholds now have explicit widen/rollback-shaped coverage: larger single-vector cases remain local under conservative caps, become planned when the caps are raised, and resume after budget exhaustion. Require release history, latency budgets, operations docs, cleanup bounds, generated-client stability, and rollback paths before each widening. | A family becomes default only after parity, process ownership, crash/restart, cleanup, public-read, fan-in, status, operations, and latency-budget evidence all pass. | + +The steady-state architecture for every metric family should remain: + +```text +graph index metric config + -> dirty marker for target edge generation + -> coordinator-owned durable build job + -> deterministic manifest pages + -> worker-owned page leases and cursors + -> attempt-scoped intermediate output where later phases could consume partials + -> coordinator-owned phase and iteration barriers + -> verified publish of one complete generation + -> published generation pointer + -> resumable cleanup plus bounded diagnostics +``` + +The strict dependency chain is remote ownership before default promotion, +cross-shard comparability before score-bearing fan-in promotion, degree before +PageRank defaults, PageRank before eigenvector defaults, and HITS only after +paired-vector failure behavior is routine. + +Status/API maturity should follow the same rule. Internal runtime telemetry can +be richer than the public contract, but generated clients should only gain +fields once they are stable summaries: metric state, freshness, published and +target generations, current phase/iteration, page progress, bounded failure +diagnostics, cleanup state, role, owner hash, worker count/hash, lease state, +takeover/lost-lease counters, tick progress, and page counters. Raw storage +keys, exact owner identifiers, attempt namespaces, and local-process details +should remain implementation details. The first stable runtime summary is now +`graph_metric_runtime` on graph index status; aggregate status merges shard +counters while per-shard status preserves each shard's runtime summary. If +independent shard runtimes report different roles, aggregate status keeps the +counter and identity-hash summary but omits `role` rather than implying a single +owner role. The shared plus joined Zig OpenAPI client contracts parse that +summary as generated typed status. The public generated Go, Python, and +TypeScript SDK surfaces also carry the typed runtime summary so operations code +can inspect the stable telemetry without depending on internal graph-index +records. + +#### Rest-of-Roadmap Execution Plan + +This subsection is the canonical roadmap from the current PR state. Older +roadmap notes below remain useful as implementation history, but this is the +plan to execute next. + +The remaining work is not a new PageRank API. It is the path from the current +durable graph-index metric executor to a production distributed executor that +can be enabled by default one metric family at a time. Users should keep seeing +one graph-index metric model: + +- metrics are configured on the graph index by stable names +- reads choose `published` or `fresh` +- `MetricNotReady` means no complete generation exists +- `MetricStale` means a complete generation exists but not for the requested + current edge generation +- fixed-iteration non-converged PageRank, eigenvector, and compatible HITS + publish by default with `converged: false` +- jobs, leases, pages, attempts, retries, owners, and cleanup policy remain + internal, with only stable summaries exposed through status + +The implementation should now move through these release bands: + +| Band | Goal | Implementation design | Exit gate | +| --- | --- | --- | --- | +| 1. Owner restart/failure closeout | Make the service-backed owner path as strong as the direct DB harness. | Use the internal graph-metric-maintenance service boundary as the deployment-shaped path. Degree and bounded PageRank now have direct service-route and real HTTP service-targeted process coverage for abandoned coordinator and worker-pool leases, duplicate owners before TTL, takeover after TTL, stale release after takeover, and continued durable progress after replacement. Eigenvector now has the same service-owner replacement proof for its single-vector path. HITS now also has a service-owner replacement proof through `hits_authority`, followed by compatible authority/hub freshness verification. The spawned-process harness also accepts strict service-targeted owner argv in its role preflight while rejecting mixed or incomplete DB/service targets. Keep extending this same evidence to larger deployment and promotion-scale cases while keeping the direct DB writer guard as local harness-only behavior. | Degree, bounded PageRank, eigenvector, and the HITS compatible pair can drain through service-targeted owners; killed or abandoned owners lose only their runtime/page leases; replacements continue after expiry; duplicate coordinators and duplicate worker owners are fenced. | +| 2. Degree distributed canary | Promote the cheapest metric first to prove the mechanics. | Put degree behind the first internal distributed-maintenance gate. Require local/distributed parity, multi-worker page partitioning, stale-attempt rejection, publish idempotence, active/failed status, cleanup resume, public `published`/`fresh` checks, and bounded control-record growth. The first internal degree canary decision now admits exactly one queued or active degree build, blocks active or queued non-degree metric work, rejects failed/truncated page summaries, and applies a configurable control-record cap so rollout can fall back before namespace growth becomes unbounded. Queued degree work now contributes a planned build control-record estimate before the build starts, so an oversized queued build is rejected before it creates job/page state. Planned maintenance results now also report `rounds_executed`, and degree canary coverage proves a one-round budget reports resumable exhaustion before a later bounded budget drains to fresh. The DB idle maintenance mode now has an explicit internal `degree_canary` setting: eligible degree-only work runs through planned maintenance, queued mixed/non-degree work rolls back to the local graph-metric oracle, and already-active planned work outside guardrails fails fast instead of silently reporting idle. Active degree rebuilds now also have direct top-k, traversal projection, and search rerank freshness coverage in both canary-mode DB maintenance and service-route ownership: `published` reads keep serving the prior generation with building status, while `fresh` reads fail closed with `MetricStale`. | Degree can be enabled internally without duplicate publish, stale score visibility, unbounded latency, unbounded namespace growth, or loss of rollback to the local oracle. | +| 3. PageRank production gate | Make PageRank the reference iterative metric. | Run PageRank through the same owner harness with production budgets, larger graph evidence, dynamic iteration restart coverage, fixed-iteration non-converged metadata, publish verification, failed rebuild preservation, cleanup resume, and fan-in checks. | Distributed PageRank matches the local oracle within tolerance, preserves prior scores on failed rebuilds, serves prior `published` scores while rebuilding, fails `fresh` with `MetricStale`, and recovers from every phase boundary. | +| 4. Eigenvector single-vector parity | Prove the single-vector executor is generic. | Reuse the PageRank scan/initialize/contribution/reduce/convergence/publish/cleanup substrate. Eigenvector should add metric math, metadata, tolerance rules, and disconnected/reducible graph behavior, not a new job model. | Eigenvector passes the same parity, restart, failure, cleanup, fan-in, public freshness, fixed-iteration metadata, and operations matrix as PageRank. | +| 5. HITS paired-vector hardening | Prove paired metrics can share one lifecycle safely. | Keep authority and hub as separate named scores with one compatible target generation, convergence decision, publish decision, failure decision, and cleanup owner. Authority and hub pages may retry independently, but publish and failure stay atomic for the compatible pair. Service-owner replacement is now covered for the compatible pair through the authority metric path, direct process coverage exhausts hub-reduce attempts while preserving the previous compatible pair, and unit-test lifecycle coverage now covers local paired reclaim, publish failure, cleanup resume, failed-build preservation, publish idempotence, failed public-read preservation, and local/planned parity. | HITS remains off by default until promotion-scale fan-in under deployment-shaped owners, larger deployment-scale owner evidence, and larger-graph latency evidence all preserve the previous compatible pair. | +| 6. Public read and fan-in closeout | Treat every score-bearing read as distributed correctness. | Route direct top-k, traversal projection/order/filter, search rerank, query profile, explain/status, hosted fan-in, and cross-shard merge through published generation pointers and compatibility checks. Building, failed, abandoned, and attempt-scoped output stays invisible outside status. | `published` reads use the latest complete generation; `fresh` fails closed with `MetricNotReady` or `MetricStale`; fan-in rejects missing, zero, stale, malformed, incompatible, or incomparable generations before merging scores. | +| 7. Cleanup and operations | Make distributed metric maintenance safe to leave enabled. | Keep v1 cleanup latest-only and aggressive. Make cleanup resumable for old score generations, manifests, pages, phase/iteration summaries, attempts, failures, and runtime-owner records. Keep retention, manual retry, pause/resume, priority, and lease tuning as future bounded admin controls. | Cleanup never removes the current published generation, resumes after restart, bounds diagnostics, and prevents completed, failed, abandoned, unpublished, or attempt-scoped state from growing without bound. | +| 8. Default widening | Promote by metric family, not by executor feature. | Keep local runners as deterministic CI/debug oracles. Widen the conservative `auto` gate in order: degree, PageRank, eigenvector, then HITS. Require generated-client stability, docs, latency evidence, operations evidence, and rollback controls for each widening. | A family becomes distributed-by-default only after parity, remote ownership, crash/restart, cleanup, public-read, fan-in, status, operations, latency-budget, docs, generated-client, and rollback checks are all green for that family. | + +The internal boundaries should stay fixed while those bands land: + +| Boundary | Owns | Must not own | +| --- | --- | --- | +| Graph index | Metric config, edge-scope fingerprint, dirty marker, published pointer, score metadata, public freshness, and status assembly. | Worker scheduling, raw job/page controls, or lease tuning exposed to users. | +| Coordinator | Job start/resume, manifest validation, phase barriers, dynamic iteration planning, verified publish, failed-build recording, and cleanup scheduling. | Page execution or worker-local cursor progress. | +| Worker | Claiming one page lease, renewing it, persisting cursor progress, writing attempt-fenced output, completing/failing the page, and stopping. | Job creation, config resolution from callers, phase advancement, publish, or whole-build failure. | +| Runtime/scheduler | Role, owner identity, worker identity set, budgets, idle policy, lease policy, wakeups, and telemetry. | Metric semantics or public query freshness behavior. | +| Cleanup runner | Resumable deletion of old generations, abandoned attempts, completed job namespaces, manifests, pages, summaries, and bounded diagnostics. | Deleting the current published generation or changing query behavior. | + +The next concrete PR sequence should be: + +1. **Service-owner restart/failure PR**: extend the new degree and bounded + PageRank direct service-route lease-loss/takeover proofs to real killed-owner + process coverage for coordinator and worker-pool abandonment, duplicate-owner + fencing before TTL, takeover after TTL, stale release fencing, and continued + durable progress after replacement. +2. **Degree canary PR**: add the internal rollout gate, latency/storage + guardrails, distributed public freshness coverage, cleanup-resume coverage, + and rollback to local execution. +3. **PageRank promotion PR**: keep the DB-level production-budget parity gate, + then finish larger-graph runs, fixed-iteration non-converged metadata checks, publish-verifier + failure, failed rebuild preservation, cleanup resume, and cross-shard + generation compatibility. +4. **Eigenvector parity PR**: reuse the PageRank single-vector executor and add + the missing restart/failure/fan-in/operations evidence plus documented + disconnected and reducible graph behavior. +5. **HITS paired-vector PR**: finish paired authority/hub remote-owner, + restart, reclaim, failed-page, publish-failure, cleanup, fan-in, and latency + evidence before enabling default HITS execution. +6. **Operations rollout PR**: document status fields, scheduler defaults, + cleanup defaults, release qualification metrics, generated-client behavior, + and rollback/default-widening controls. + +The dependency that should not be skipped is owner correctness before default +promotion. A metric family must not become distributed-by-default while +duplicate coordinators can race publish/fail, duplicate workers can write stale +attempts, cleanup can grow without bound, or fan-in can merge incomparable +generations. + +The remaining graph-index work should be tracked as a release train from +implementation proof to production default. The current PR should be treated as +the durable-executor baseline: graph metrics are index-owned, planned work is +durable and resumable, status exposes the stable runtime summary, and all metric +families have at least one shared lifecycle proof. The rest of the work should +avoid adding new user concepts unless a gate below proves the existing +graph-index metric surface is insufficient. + +Target end state: + +- Graph metric configs remain part of the graph index definition. +- The default maintenance path can run locally or through remote owners without + changing query semantics. +- Degree, PageRank, eigenvector, and HITS all use the same durable + coordinator/worker/job/page model. +- Every score-bearing read path enforces the same published/fresh generation + contract. +- Operations can answer ownership, progress, freshness, failure, and cleanup + questions from bounded status records. + +The rest of the roadmap should be implemented in these shippable milestones: + +| Milestone | Scope | User-visible result | Required implementation | Release gate | +| --- | --- | --- | --- | --- | +| A. Freeze the v1 public contract | Lock the naming, freshness, status, and error vocabulary before widening execution. | Users keep one graph-index metric API for degree, PageRank, eigenvector, and HITS. `MetricNotReady` and `MetricStale` stay the explicit read failures. | Audit OpenAPI, SDKs, docs, and public e2e tests so direct metric reads, traversal, search rerank, query profile, and status use the same fields and error names. | No remaining read path has metric-specific freshness behavior or ad hoc not-ready errors. | +| B. Production owner harness | Turn the current runtime/process proofs into the canonical distributed harness. | Operators can run coordinator and worker owners as independently managed processes or service roles. | Promote `antfly graph-metric-maintenance`/service-role configuration to the test harness for one coordinator plus multiple workers. Remove local test-only writer serialization from the success path. Keep role, owner, worker identity, budget, idle, and lease policy as the only runtime inputs. | Degree and PageRank drain through independent owners using only durable graph-index state or the service boundary. Killing and replacing a worker reclaims only its page after expiry. | +| C. Degree production canary | Use degree as the first default candidate because it is cheap and non-iterative. | Degree metrics become the canary for distributed maintenance with rollback to the local oracle. | Keep local-vs-planned parity, multi-worker page partitioning, restart, stale-attempt rejection, cleanup resume, status summaries, and public freshness coverage in CI. The internal degree canary decision now provides the first rollout guard: exactly one queued or active degree build is eligible, non-degree queued/active work blocks the degree-only rollout, failed/truncated page summaries block rollout, and a configurable control-record cap guards namespace growth. Queued degree builds now estimate their planned job/page control records before rollout, so work that would exceed the cap falls back before planned state is created. Planned-maintenance results now expose `rounds_executed`, and canary tests prove tight round budgets are reported as resumable rather than hidden unbounded work. DB idle maintenance can now select `degree_canary`, which uses planned maintenance only when that guard passes, falls back to local maintenance before starting blocked queued work, and fails fast when active planned work is already outside guardrails. Direct top-k, traversal projection, and search rerank freshness now cross both the DB canary path and the in-process service-route owner path for active degree rebuilds: prior `published` scores remain visible with building status and `fresh` fails closed. Unit-test coverage now includes those degree canary guardrails so those guardrails are release-gated. Remaining work is production-scale latency data. | Degree can be enabled internally by default without unbounded latency, duplicate publish, stale score visibility, or namespace growth. | +| D. PageRank distributed promotion | Make PageRank the reference iterative metric. | PageRank remains the primary user-facing centrality metric, but its default executor can become distributed when gates pass. | Keep the DB-level production-budget local/planned parity proof, then finish larger-graph parity, dynamic iteration restart coverage, fixed-iteration non-converged metadata, publish-verifier failure, cleanup resume, public read coverage, and cross-shard fan-in checks. | Distributed PageRank matches the local oracle within tolerance, preserves the previous generation on failed rebuilds, fails `fresh` while stale, and recovers from every phase boundary. | +| E. Eigenvector parity | Prove the single-vector executor is generic. | Eigenvector looks like another named graph metric, not a separate subsystem. | Reuse the PageRank scan/initialize/contribute/reduce/check/publish/cleanup substrate. Add the same restart, failure, cleanup, status, fan-in, and operations evidence as PageRank. Document reducible/disconnected graph behavior and fixed-iteration non-converged output. | Eigenvector has no metric-specific job model and passes the same single-vector promotion matrix as PageRank. | +| F. HITS paired-vector hardening | Prove paired metrics can be promoted safely. | Authority and hub scores remain separate named reads but share compatible lifecycle and failure semantics. | Keep authority/hub output attempt-scoped until page completion. Preserve the completed active/failed process-boundary public freshness coverage, hosted paired-HITS `published`/`fresh` cross-range fan-in coverage, the hosted active paired-HITS fan-in proof for direct top-k, traversal projection/order/filter/status, authority rerank, service-owner replacement proof, and default authority/hub pair compatibility rejection while finishing promotion-scale cross-shard fan-in, larger-graph latency, deployment-scale remote-owner evidence, and paired parity evidence. Require atomic compatible-pair publish and paired failure preservation. | HITS stays off by default until either side failing preserves the previous compatible pair and paired-vector restart coverage matches the PageRank bar. | +| G. Public fan-in and retrieval gate | Prevent distributed scoring from mixing incomparable generations. | Cross-shard top-k, traversal/search scoring, rerank, and profile either compare compatible published generations or fail closed. | Add/keep tests for missing metric status, zero generation, stale generation, incompatible generation, incompatible paired-HITS status, active build, failed build, and mixed shard runtime roles; hosted paired-HITS `published`/`fresh` success/failure coverage now exists for local shard DB fan-in. Explain/profile should report the generation/freshness basis used for scoring. | Every score-bearing fan-in path rejects missing, zero, stale, or incompatible generations when global comparability is required. | +| H. Operations, cleanup, and rollout | Make the executor safe to leave on. | Admins get bounded status and cleanup behavior without needing retention or lease tuning for correctness. | Finish resumable cleanup for published-generation supersets, manifests, pages, phase/iteration summaries, attempts, failures, and runtime-owner records. Add event/status docs, latency budgets, scheduler defaults, and rollback controls. | Cleanup never removes the current published generation, resumes after restart, bounds diagnostics, and gives operators enough telemetry to debug stuck work. | + +Milestones should merge in order, but their tests should overlap. For example, +PageRank promotion should reuse the owner harness created for degree, while +eigenvector and HITS should start gaining public-read and cleanup coverage +before PageRank becomes default. The dependency that should not be skipped is +owner correctness before default promotion: a metric family must not become +distributed-by-default until duplicate coordinators, duplicate workers, stale +attempts, failed rebuilds, cleanup, and fan-in are already proven for that +family. + +Design rules for the remaining implementation: + +- Prefer adding metric families by plugging algorithm-specific phases into the + shared executor rather than adding metric-specific job records. +- Keep local runners as deterministic CI/debug oracles until distributed output + parity and operations evidence are routine. +- Keep worker callers name-based. A worker should never receive the resolved + metric config from a caller; it should load durable graph-index state by + index and metric name. +- Treat attempt-scoped output as mandatory when a later phase could consume + partial output. Deterministic recompute is acceptable only when stale output + cannot become visible and recompute cost is bounded. +- Keep v1 cleanup aggressive. Retention, manual retry, pause/resume, priority, + and lease tuning can become bounded admin controls later, but they should not + be needed for correct queries. +- Treat failed planned builds as terminal for automatic planned scheduler + starts at the same target generation. Status can still expose failed work as + needing attention, but duplicate/background coordinators must not immediately + recreate the same failed generation and append another failure. A later dirty + generation or an explicit future retry control can start new work. DB-level + scheduler coverage now pins this policy: failed planned PageRank is not + counted as auto-eligible planned work, a duplicate coordinator sweep does not + start another build or append another failure event, and a later graph write + makes the next generation eligible again. Runtime-owner coverage now pins the + operations shape too: a runtime tick over only terminal failed planned work + reports no durable progress through both runtime-local and stable DB runtime + stats, increments idle rather than error/failure counters, leaves one failure + event, and starts work again only after a newer dirty generation appears. +- Promote defaults by metric family, not by executor feature. Degree can ship + before PageRank; PageRank before eigenvector; HITS last because paired-vector + failure is the highest-risk contract. + +The minimum dashboard/status vocabulary for the rest of the rollout is: + +| Status area | Stable summary fields | +| --- | --- | +| Freshness | metric state, published generation, target edge generation, freshness result, dirty state | +| Build progress | phase, iteration, expected/completed/failed pages, units completed/total, convergence metadata | +| Ownership | runtime role, owner hash, worker count/hash, lease held, last acquired time, takeover count, lost-lease count | +| Failure | bounded recent failures with phase, iteration, page attempt, retry count, and last error | +| Cleanup | cleanup phase, remaining namespace/page counts when available, last cleanup error, diagnostics pruned count | +| Fan-in | per-shard generation/freshness compatibility and aggregate role omission when shard roles differ | + +Default-promotion checklist for each metric family: + +- Local-vs-distributed parity over typed and all-edge scopes. +- Restart coverage across every phase boundary. +- Expired page lease reclaim and stale-attempt rejection. +- Failed rebuild preserves prior published generation. +- Cleanup resumes after restart and bounds abandoned state. +- Public `published` and `fresh` reads behave consistently across direct, + traversal, search rerank, profile, and shard fan-in paths. +- OpenAPI and generated Go, Python, TypeScript, and Zig client surfaces are + stable. +- Scheduler/runtime budgets have latency evidence and a rollback path. + +#### Detailed Roadmap Design From Here + +The remaining work should be planned as a sequence of production-boundary +changes rather than more algorithm-specific experiments. The current codebase +already proves the durable graph-index executor shape locally: metric configs +belong to graph indexes, coordinators and workers communicate through persisted +job/page state, runtime and page leases fence owners, attempt namespaces fence +partial output, and public reads use only published generations. The unfinished +roadmap is to make that boundary production-shaped, prove it at promotion scale, +and then widen defaults one metric family at a time. + +The production architecture should have these internal components: + +| Component | Responsibility | Boundary rule | +| --- | --- | --- | +| Graph index metric config | Own metric names, algorithm parameters, edge filters, freshness semantics, and stable status. | Never copy resolved metric configs into worker process arguments or external job payloads. | +| Dirty-generation detector | Detect the target edge generation that needs a rebuild. | Creates or resumes durable planned work; it does not synchronously compute metric scores. | +| Coordinator owner | Ensure jobs, advance phase/iteration barriers, append dynamic pages, verify publish, fail builds, and schedule cleanup. | Exactly one active coordinator per role lease may mutate build lifecycle state; duplicate coordinators must be idempotent observers or fenced. | +| Worker owner | Claim page leases, renew leases, persist cursors, write attempt-scoped output, and complete or fail pages. | Workers do not start jobs, decide convergence, publish, fail whole builds, or receive caller-supplied metric configs. | +| Runtime lease registry | Fence independently managed coordinator and worker processes. | Runtime lease loss stops further page/build mutation; clean shutdown deletes only the current owner record. | +| Page/attempt store | Fence individual work units and partial output adoption. | Stale owners from older attempts cannot write, complete, adopt, publish, or fail after replacement. | +| Publish verifier | Validate manifest/config fingerprints, page summaries, iteration metadata, and score output before moving the generation pointer. | Failed verification preserves the previous published generation and records bounded diagnostics. | +| Cleanup runner | Delete old generations, manifests, pages, summaries, attempts, failures, and runtime records when safe. | Cleanup never removes the current published generation and is resumable after restart. | +| Public read gate | Enforce `published` and `fresh` semantics across direct, traversal, rerank, profile, and fan-in paths. | Building, failed, abandoned, or attempt-scoped output is never queryable as score data. | + +The roadmap should be executed in four release bands: + +| Release band | Goal | Primary PRs | Must prove before moving on | +| --- | --- | --- | --- | +| 1. Production owner boundary | Replace local-process proof assumptions with deployment-shaped coordinator and worker owners. | Remote owner harness, service-boundary adapter, runtime lease ownership, command/status telemetry. | Degree and PageRank drain from dirty state to fresh publish through independently managed owners without shared memory, worker-side metric config, or local test-only writer serialization. | +| 2. Canary and reference promotion | Promote the lowest-risk family first, then the reference iterative family. | Degree canary, PageRank production promotion, latency/storage budget collection, rollback wiring. | Degree and PageRank pass parity, restart, stale-attempt rejection, failed rebuild preservation, cleanup, public read, status, and fan-in gates under production budgets. | +| 3. Generalization and paired metrics | Prove the executor is not PageRank-specific and that paired metric families remain compatible. | Eigenvector parity, HITS paired-vector hardening, promotion-scale paired fan-in, larger-graph runs. | Eigenvector matches the PageRank single-vector matrix; HITS authority/hub publish and fail atomically while preserving the previous compatible pair. | +| 4. Default widening and operations | Make distributed maintenance safe to leave enabled. | Per-family default gate widening, ops docs, dashboard/status docs, cleanup and diagnostics bounds, generated-client stability. | Each family has latency evidence, rollback controls, bounded cleanup, stable status, and release history before becoming distributed-by-default. | + +Current checkpoint for that roadmap: + +| Area | Already true | Remaining design work | +| --- | --- | --- | +| Local owner proof | `graph-metric-maintenance` can launch independently owned coordinator and worker-pool roles against a shared local DB path, drain degree, PageRank, eigenvector, and compatible HITS, and prove lease/page/publish idempotence through spawned processes. | Keep this path as the compatibility harness and local oracle, but stop treating shared local DB access plus the writer guard as production orchestration. | +| Runtime boundary | The runtime owner loop now calls a `MaintenanceBoundary` for combined, coordinator, worker, and worker-pool ticks instead of directly embedding every `IndexManager` call. The boundary is now a small vtable with a reusable role-dispatch tick helper, so the same runtime tick contract can target direct DB state or another implementation. | Keep extending the service-backed boundary implementation until production supervisors use it by default and direct DB access is only the local oracle/harness. | +| Service API | `/internal/v1/groups/{group}/tables/{table}/graph-metric-maintenance` accepts the same role, runtime owner, worker, lease, tick budget, page budget, and clock-shaped inputs as the direct DB path, the API client has a matching method, and `antfly graph-metric-maintenance` can target either a local DB path or that service endpoint with group/table identity. The service path now acquires or renews the durable runtime-owner lease before running a tick, reports runtime ownership stats in the maintenance response, fences duplicate owners before TTL expiry, and reports takeover after expiry. The service-targeted command path now runs through the same `MaintenanceBoundary` role dispatch as the runtime, and worker-pool service ticks preserve the worker set as one owner-scoped request instead of expanding to local per-worker DB calls. The `supervise` and `launch` entrypoints now also accept the same service target and launch coordinator/worker-pool children through the service boundary without the local DB writer guard. The same endpoint now accepts `tick`, `status`, and `release` actions: status reports the current owner hash and expiration without exposing raw keys, release deletes only the current owner's lease, stale-owner release after takeover is fenced, and service-mode commands send a final release on clean shutdown. Focused in-process service-route proofs now drain background degree and bounded PageRank builds to fresh through service-targeted coordinator and worker-pool owners without opening the DB through the command path. Degree service-route coverage now also leaves a coordinator-started rebuild active and proves direct top-k, traversal projection, and search rerank use the prior published generation with building status while `fresh` fails closed. Degree and bounded PageRank now also have service-route owner restart/failure coverage: abandoned coordinator and worker-pool owners fence duplicate replacements before TTL, allow takeover after TTL, reject stale release after takeover, and continue to a fresh generation through replacement owners. The service command test hook now writes its ready marker and optional hold before clean release, so the spawned-process harness can kill a service-targeted owner after a real lease-bearing tick instead of after shutdown cleanup. Degree and bounded PageRank now have real HTTP service-targeted process proofs: the harness exposes each seeded graph DB through `ApiHttpServer`, kills service-targeted coordinator and worker-pool owner processes after ready, observes duplicate-owner fencing before TTL, observes replacement takeover after TTL, and drains the generation to fresh through replacement service-targeted processes. Degree, PageRank, eigenvector, and paired HITS now also have larger service-targeted multi-page proofs: real HTTP service coordinators start 130-source builds, a two-worker service worker-pool drains multiple degree scan/reduce pages, multiple PageRank/eigenvector scan/initialize/contribution/reduce/convergence pages, and paired HITS scan/initialize/authority-contribution/authority-reduce/hub-contribution/hub-reduce/convergence pages through bounded ticks, the coordinator publishes, and cleanup drains to fresh without the command opening the local DB. The degree, PageRank, eigenvector, and paired HITS multi-page service proofs now also kill the coordinator after start and kill the scan worker-pool after bounded page work, prove duplicate owners are fenced before TTL, prove replacements take over after TTL, and then continue the same 130-source builds to publish and cleanup. HITS now also has service-owner restart/replacement coverage through `hits_authority`, with a final compatible authority/hub freshness check proving the pair drains together. Degree, PageRank, eigenvector, and HITS now also have service-targeted publish-to-cleanup proofs: a real HTTP service coordinator publishes a prepared build, a second coordinator cannot duplicate publish, and separate worker-pool service processes resume cleanup to a fresh generation, including degree/PageRank/eigenvector/HITS multi-page cleanup. PageRank, eigenvector, and paired HITS now also have service-targeted publish-verifier failure proofs: a real HTTP service coordinator fails a corrupted manifest without publishing, preserves the prior generation or compatible pair, and a duplicate service coordinator cannot publish or fail again. The HITS service publish/cleanup and larger multi-page service proofs now also verify paired fixed-iteration metadata for authority and hub after service-owned publish and cleanup. Separate real HTTP service-targeted degree, PageRank, eigenvector, and paired-HITS public-read proofs now publish initial generations through service owners, start later rebuilds through the same service boundary, and verify direct top-k, traversal projection, and search rerank use the prior published generation while `fresh` fails closed. Degree, PageRank, and eigenvector report `building`; paired HITS preserves the compatible prior pair while authority/hub statuses may be `building` or `stale` depending on which side owns the active lifecycle state. | Owner-boundary closeout is covered for degree and bounded PageRank, HITS has compatible-pair service-owner replacement evidence, degree/PageRank/eigenvector/HITS have multi-page two-worker service-boundary evidence, degree/PageRank/eigenvector/paired HITS have first multi-page service killed-owner timing evidence, PageRank/eigenvector/paired HITS publish-verifier failure now crosses the service process boundary, HITS has paired fixed-iteration metadata evidence across service-owned publish/cleanup and multi-page service runs, degree/PageRank/eigenvector/HITS cleanup restart now crosses the service process boundary, and the degree/PageRank/eigenvector/HITS active public-read contracts now cross the real HTTP service owner path; remaining work is broader promotion-scale remote-owner evidence, public fan-in coverage, broader cleanup operations, promotion-scale failure timing, and per-family default gates. | +| Public reads | Direct metric, traversal, rerank, profile, hosted fan-in, not-ready, stale, active-build, and failed-build behavior are covered in focused tests for many surfaces. Hosted cross-range fan-in now also covers active degree, PageRank, eigenvector, and paired HITS rebuild shards with compatible prior published generations: direct metric top-k, traversal projection/order/filter/status, and search rerank merge the prior generation for `published`, while `fresh` fails closed. Degree/PageRank/eigenvector report `building`; paired HITS may report the authority side as `building` and the compatible hub side as `stale` while still preserving the prior pair. Degree, PageRank, eigenvector, and paired HITS also have direct/traversal/rerank active public-read proof through the real HTTP service owner path. Paired HITS now also has a direct DB failed-planned-rebuild proof: authority and hub `published` direct reads, traversal projection/order/filter/status, and authority rerank return the prior compatible pair with failed status, omit attempt output from the failed generation, and `fresh` fails with `MetricStale`. | Repeat the public-read matrix under production remote owners and promotion-scale shard layouts before widening defaults. | +| Cleanup and operations | Latest-only cleanup, bounded diagnostics, runtime-owner lease telemetry, command summaries, and graph-index runtime status are now visible enough for local/process proofs. Direct process proofs cover degree, PageRank, eigenvector, and HITS publish/cleanup restart, PageRank/eigenvector/HITS exhausted page-attempt failure, and all four families now have service-targeted publish/cleanup restart proofs that publish through the real HTTP service owner path and resume cleanup through replacement worker-pool service processes. Degree, PageRank, eigenvector, and paired HITS now also have graph-level repeated-failure storage-growth proofs: each failed planned build removes abandoned score/job namespaces immediately while recent failure/event diagnostics remain capped. | Promote those summaries into the operational contract for remote owners without exposing raw storage keys, page ids, attempt namespaces, or process-local writer details; broaden cleanup qualification across larger retained namespaces and promotion-scale storage growth before default widening. | + +The implementation sequence from this checkpoint should be: + +| Step | Slice | Design | Done when | +| --- | --- | --- | --- | +| 0 | Service-targeted owner command | Done for the first boundary slice: `antfly graph-metric-maintenance` accepts either `--db-path` or `--base-uri --group-id --table-name`, sends only role, runtime owner identity, worker identity or worker set, lease TTL, tick/page budgets, background-start policy, and optional test clock inputs to the internal group maintenance endpoint, rejects mixed/incomplete targets, and keeps metric names, index names, target generations, job ids, page ids, manifest paths, score prefixes, and metric configs out of the request body. The command accepts the service response envelope and merges server runtime ownership telemetry into its summary. | A coordinator, worker, worker-pool, or combined owner can run bounded ticks through the internal group maintenance endpoint and aggregate durable progress, idle, error, sweep-result, and runtime-owner lease summaries without opening the local group DB directly. | +| 1 | Service-backed `MaintenanceBoundary` | Done for the boundary contract slice: the internal service boundary now has the same request-shaped coordinator/worker operations and service-side runtime owner lease acquire/renew/takeover fencing for bounded ticks. The runtime boundary itself is implementation-neutral, and the service-targeted command path uses that boundary dispatch for combined, coordinator, worker, and worker-pool ticks. The supervisor and launcher can target either a local DB path or service endpoint, and service-targeted launched children omit the local DB writer guard. The endpoint now has explicit status/release actions, clean service command shutdown releases only the current owner lease, and stale release after takeover preserves the replacement owner. Treat the internal API as a deployment boundary, not as a public job API. | The runtime can execute the same owner loop through a direct DB boundary or service boundary, and tests prove the request/response contract preserves durable progress, idle, error, conflict, lease-acquisition, lease-loss, takeover, clean-release, and status semantics. | +| 2 | Remote owner harness | Replace local launch assumptions with deployment-shaped coordinator and worker processes using the service target by default in integration coverage. Degree and bounded PageRank now have in-process service-route drain proofs: service-targeted coordinator and worker-pool owners alternate bounded ticks through the internal group route, release their runtime-owner leases on shutdown, and publish fresh generations without the command path opening the local DB. They also prove abandoned service-route coordinator and worker-pool leases are fenced before TTL, taken over after TTL, and stale releases cannot clear the replacement owner before the build drains to fresh. The spawned-process preflight now treats service targets as first-class owner argv and rejects mixed or partial targets. Degree, PageRank, eigenvector, and paired HITS now extend that service proof to larger 130-source builds where a two-worker service worker-pool completes multiple partitioned pages before service coordinator publish and cleanup; PageRank and eigenvector cover scan, initialize, contribution, reduce, and convergence phases, while HITS covers both authority and hub contribution/reduce phases before compatible pair publish. Degree, PageRank, eigenvector, and paired HITS also combine those 130-source service proofs with killed coordinator and killed worker-pool timing: duplicates are fenced before TTL, replacements take over after TTL, and the same builds continue through remaining pages, publish, and cleanup. Keep the direct DB writer guard only for the local proof harness. | Killing service-targeted owners, expiring leases, replacing owners, and retrying stale attempts produce the same durable results as the direct DB harness; multi-page degree, PageRank, eigenvector, and paired HITS service builds prove the worker-pool request shape can carry real partitioned non-iterative, single-vector iterative, and paired-vector iterative work, not only one-page fixtures. Degree, PageRank, eigenvector, and paired HITS now have first multi-page service killed-owner timing proofs; the same timing matrix still needs to be broadened at promotion scale. | +| 3 | Degree canary | Use degree as the first distributed-by-default candidate because it exercises planning, page leases, publish, cleanup, status, and freshness without iterative math. The internal degree canary decision now separates degree-only rollout eligibility from the broader auto gate: one queued/active degree build can run planned, active/queued non-degree work forces fallback, failed/truncated page summaries force fallback, and a control-record cap bounds namespace growth. Queued degree builds now add an estimated planned control-record count to that cap check before startup. Planned sweep results now carry `rounds_executed`, giving the canary a direct bounded-round signal for latency-budget qualification. The `degree_canary` idle mode wires that decision into `runUntilIdle`, so canary-enabled DBs exercise planned degree maintenance while retaining rollback to the local oracle before blocked queued work starts and failing fast when already-active planned work is outside guardrails. Direct top-k, traversal projection, and search rerank freshness now have active-rebuild coverage for prior-generation `published` reads and `fresh` failure in canary-mode DB maintenance, in-process service-route ownership, the real HTTP service owner path, and focused hosted cross-range fan-in. | Degree passes local/distributed parity, owner restart, stale-attempt rejection, cleanup resume, active/failed status, public read freshness, latency budget, storage-growth budget, and rollback-to-local checks under the service harness. | +| 4 | PageRank production gate | Promote PageRank only after the reference iterative matrix passes with remote owners. Dynamic iteration planning, convergence metadata, fixed-iteration non-converged publish, publish verification, failed rebuild preservation, cleanup, and fan-in are all required. The real HTTP service owner harness now covers the active public-read part of that matrix: after an initial service-owned publish and a service-started rebuild, direct top-k, traversal projection, and search rerank serve the prior published generation while `fresh` fails closed. Hosted cross-range fan-in now carries the same active PageRank invariant across compatible shard generations for direct metric top-k, traversal projection/order/filter/status, and search rerank. | PageRank matches the local oracle within tolerance, preserves prior scores on failure, serves prior `published` scores while rebuilding, fails `fresh` with `MetricStale` when appropriate, and rejects incompatible shard fan-in at promotion scale. | +| 5 | Single-vector and paired-vector backfill | Bring eigenvector to the PageRank matrix through the same single-vector executor, then bring HITS through the stricter paired authority/hub lifecycle. Eigenvector now has real HTTP service-owner active public-read coverage for direct top-k, traversal projection, and search rerank, service-owner replacement coverage, hosted cross-range active fan-in coverage, two-worker service-boundary multi-page phase coverage, service-targeted publish/cleanup restart coverage, unit-test scan/initialize/contribution/reduce/convergence reclaim coverage, cleanup resume after reopen, failed planned-build preservation, coordinator publish-failure preservation after reopen, failed planned-rebuild public-read preservation, and DB-level production-budget parity through a 130-source one-page-budget planned drain matching the local oracle. Paired HITS now has real HTTP service-owner active public-read coverage, service-owner replacement proof through the authority path, two-worker service-boundary multi-page authority/hub phase coverage, first service killed-owner timing for coordinator and scan worker-pool replacement, scan-page reclaim coverage that rejects stale partial writes, failed planned rebuild public-read coverage that preserves the prior authority/hub pair for direct top-k, traversal projection/order/filter/status, and authority rerank while `fresh` fails with `MetricStale`, paired fixed-iteration metadata verification after service-owned publish/cleanup and multi-page service publish, focused hosted active fan-in coverage for authority/hub direct top-k, traversal projection/order/filter/status, authority rerank while `fresh` fails closed, service-targeted publish/cleanup restart coverage, and DB-level production-budget parity through a 130-source one-page-budget planned drain matching the local authority/hub oracle for the compatible pair. | Eigenvector adds no new job system or status model and still needs deployment-scale parity, promotion-scale fan-in, operations evidence, and latency data before promotion. HITS authority/hub publish and fail atomically, preserve the previous compatible pair, and stay disabled by default until promotion-scale paired fan-in, broader deployment-scale owner evidence, and larger-graph latency evidence are complete. | +| 6 | Operations and default widening | Widen the conservative `auto` gate one family at a time only after operational evidence is routine. | Status, generated clients, docs, dashboards, cleanup bounds, diagnostics bounds, rollback controls, latency budgets, and release qualification all pass before a family becomes distributed-by-default. | + +The first release band should turn the current process harness into the +canonical production harness: + +1. Define a narrow owner launch contract: endpoint or DB location, role, owner + id, worker id or worker-id set, tick/page budgets, idle policy, lease TTL, + and optional test clock inputs. +2. Keep metric name, index name, target generation, page id, manifest details, + metric config, score prefixes, and publish decisions inside durable + graph-index state or the service API that fronts it. +3. Introduce a production service-boundary adapter that has the same semantics + as the direct DB harness: coordinator sweeps, worker page sweeps, lease + acquire/release, status fetch, and cleanup ticks. The runtime now routes + owner ticks through a `MaintenanceBoundary` instead of calling the + `IndexManager` inline; the current boundary is direct/embedded, and the + internal group service boundary now exposes the same budget-shaped + coordinator, worker, worker-pool, and combined operations through + `/internal/v1/groups/{group}/tables/{table}/graph-metric-maintenance` plus + the matching API client method. The `graph-metric-maintenance` command can + now call that service endpoint directly for bounded owner ticks through the + same boundary dispatch helper used by the runtime, and the endpoint now + acquires/renews durable runtime owner leases before executing + service-targeted work. Worker-pool service ticks remain owner-scoped service + requests with the worker set intact instead of being expanded into local + direct-DB worker calls. The supervisor and launcher entrypoints now accept + service targets too, and service-targeted launched children do not take the + local DB writer guard. The service endpoint now also supports status and + owner-scoped release actions, and clean service-mode command shutdown sends + a final release while stale-owner release after takeover is fenced. The + first service-route drain proofs now publish degree and bounded PageRank + through alternating service-targeted coordinator and worker-pool owners + without the command path opening the local group DB directly. Degree and + bounded PageRank now also prove abandoned service owner leases are fenced + before TTL, replacements can take over after TTL, stale releases after + takeover are refused, and replacement owners can continue to fresh output. + Service-mode owner processes now write their test-ready marker and hold + before clean shutdown release, which makes "kill after ready" a true + abandoned-lease scenario for the real service-targeted process harness. + Degree and bounded PageRank now have that live HTTP proof: the + spawned-process harness serves each seeded graph DB through `ApiHttpServer`, + kills service-targeted coordinator and worker-pool owners after they acquire + runtime leases, verifies duplicate replacements are fenced before TTL, + verifies replacements take over after TTL, and drains to a fresh generation + through service-targeted replacement processes. Eigenvector now has the same + service-owner restart/replacement proof for its single-vector path, and HITS + has service-owner restart/replacement coverage through `hits_authority`, + followed by a compatible authority/hub freshness check. The remaining + production work is promotion-scale remote-owner evidence, public read/fan-in + coverage, cleanup operations, and per-family default gates. +4. Preserve the direct local DB launch path only as a proof harness, with its + explicit file-backed writer guard documented as local-only storage + serialization rather than a graph metric coordination primitive. +5. Add crash/restart tests that start real owners, kill a coordinator or worker, + advance the clock past the lease, start replacements, and prove the same + generation either publishes once or fails once while preserving prior scores. + +The second release band should make degree and PageRank promotion decisions +explicit: + +| Metric family | Why this order | Promotion evidence | +| --- | --- | --- | +| Degree | Exercises the distributed mechanics without iterative convergence. | Local/planned parity, page partitioning, process replacement, stale-attempt rejection, publish idempotence, cleanup resume, active/failed status, direct/traversal/rerank freshness, two-worker service-boundary multi-page scan/reduce evidence, latency budget, storage-growth bound, rollback to local. | +| PageRank | Reference single-vector iterative metric and primary user-facing centrality score. | DB-level production-budget local/planned parity now drains a 130-source multi-page PageRank build through one-page planned maintenance rounds and matches the local oracle. Unit-test lifecycle coverage now also includes later-iteration exhausted-attempt preservation for the prior published generation plus failed planned-rebuild public-read preservation for direct top-k, traversal projection/order/filter/status, and search rerank while `fresh` fails with `MetricStale`. Hosted fan-in now includes a nonuniform eight-shard PageRank direct merge layout with four active/stale shards that keep unpublished target scores invisible while `fresh` fails closed. Remaining promotion evidence is deployment-scale cleanup, broader hosted fan-in surfaces, and latency evidence. | +| Eigenvector | Proves PageRank's single-vector substrate is generic. | Same as PageRank plus documented disconnected/reducible graph behavior and fixed-iteration non-converged output. Active direct/traversal/rerank freshness now has real HTTP service-owner coverage, service-owner replacement coverage, two-worker service-boundary multi-page scan/initialize/contribution/reduce/convergence evidence, focused hosted cross-range fan-in coverage including a nonuniform eight-shard direct merge with active/stale shards, unit-test scan/initialize/contribution/reduce/convergence reclaim coverage, cleanup resume after reopen, failed planned-build preservation, coordinator publish-failure preservation after reopen, later-iteration exhausted-attempt preservation, direct DB failed planned-rebuild public-read preservation for direct top-k, traversal projection/order/filter/status, and search rerank while `fresh` fails with `MetricStale`, and DB-level production-budget local/planned parity through one-page planned maintenance rounds; deployment-scale parity, broader promotion-scale fan-in, and latency evidence remain required. | +| HITS | Highest-risk contract because two named scores share one compatible lifecycle. | Authority/hub paired parity, paired restart/reclaim/exhausted-attempt coverage, atomic compatible publish, paired failure preservation, paired cleanup, fixed-iteration metadata, public freshness, promotion-scale fan-in, larger-graph latency. Real HTTP service-owner active public-read coverage, service-owner replacement through `hits_authority`, two-worker service-boundary multi-page authority/hub contribution/reduce evidence, unit-test and direct-process killed-worker exhausted hub-reduce attempt coverage with duplicate coordinator idempotence, scan-page reclaim coverage that rejects stale partial writes, direct DB failed planned rebuild coverage for prior compatible authority/hub direct top-k, traversal projection/order/filter/status, and authority rerank with `fresh` `MetricStale`, paired service-owned fixed-iteration metadata, focused hosted fan-in, and DB-level production-budget local/planned parity now cover published/fresh, active, failed, and exhausted-attempt paired-HITS reads plus one-page-budget paired drains for direct top-k, traversal, and authority rerank. | + +The public/API work should stay deliberately small. The graph metric user model +does not need a new job-submission API for this roadmap. The only stable public +surface that should grow is status: fields that summarize freshness, phase, +iteration, page progress, convergence, owner hash, worker count/hash, lease +state, takeover/lost-lease counters, bounded failures, and cleanup progress. +Exact owner ids, raw keys, page ids, attempt namespaces, process pids, and local +writer-guard details should remain internal. + +The operations roadmap should be tracked with the same severity as algorithm +coverage: + +- Define scheduler defaults for tick budget, page budget, idle budget, lease + TTL, and retry attempts, plus rollback controls for every default widening. +- Add release-qualification runs that record latency, storage growth, page + counts, cleanup duration, retry counts, and fan-in behavior for representative + graph sizes. +- Document how to inspect stuck metrics: status, recent events, role ownership, + active pages, failed pages, lease expiry, and cleanup state. +- Keep v1 retention latest-only. Historical generation retention, manual retry, + pause/resume, priority, and lease tuning can be later admin controls, but none + should be required for query correctness. +- Treat generated client changes as part of the release gate; status fields + should not appear in OpenAPI/SDKs until their semantics are stable. + +The work is complete only when a clean install can enable distributed graph +metric maintenance, restart any owner at any point, continue to serve correct +published scores, fail `fresh` reads when appropriate, reject incomparable shard +fan-in, clean abandoned state, and give operators enough bounded telemetry to +understand what happened. + +### Remaining Implementation Backlog + +The rest of the work should be cut as a sequence of narrow release gates. Each +gate should leave the public graph metric API unchanged and move one internal +boundary closer to production distributed execution. + +| Order | Backlog item | Implementation work | Verification required | +| --- | --- | --- | --- | +| 1 | Remote owner harness | Promote the process/runtime path from local proof to the canonical coordinator/worker harness. Owners should accept only DB path or service endpoint, role, owner id, worker identity, tick budget, idle policy, and lease policy. Metric config, target generation, page manifests, attempts, publish, failure, and cleanup stay in graph-index state. Launched child argv tests now enforce that narrow owner/budget interface and keep metric/index/job/page details out of process arguments. Command-summary tests now pin the stable role/owner/worker/lease/progress/error telemetry emitted by standalone coordinator and worker-pool processes, and aggregate supervisor/launcher summaries now preserve the same compact per-child telemetry. Standalone role summaries are now treated as a stable operational contract too: they must expose durable-progress, idle, and error tick counters; explain completed ticks as progress, idle, error, lease contention, or lease loss; prove either runtime-owner lease acquisition or explicit acquisition failure; and report `last_error_name: null` for successful owner ticks. Standalone role summaries and aggregate supervisor/launcher summaries must both omit raw metric/index names, target generations, job/page ids, attempt namespaces, storage paths/prefixes, metric configs, process ids, and local writer details. The runtime owner loop now depends on a `MaintenanceBoundary` for combined/coordinator/worker/worker-pool ticks, the internal group write API now exposes the same narrow graph metric maintenance request over HTTP with a matching client method, the command can call that service target directly through the same boundary dispatch path, service-targeted ticks now acquire/renew/take over durable runtime-owner leases before mutating graph metric state, supervisors can launch service-targeted coordinator/worker-pool children without local writer serialization, clean service shutdown releases only the current owner lease while stale releases are fenced, and degree plus bounded PageRank now drain to fresh through an in-process internal service route using service-targeted owners. Degree and bounded PageRank service-route restart/failure coverage now proves abandoned coordinator and worker-pool owners are fenced before TTL, replaced after TTL, stale releases cannot clear replacement owners, and replacement owners continue to fresh output. The spawned-process harness now preflights service-targeted owner argv as a strict endpoint/group/table target and rejects mixed or incomplete local/service targets, real HTTP killed-owner process coverage now proves the same restart/fencing path for degree and bounded PageRank, eigenvector now has service-owner replacement proof, HITS now has service-owner replacement proof with compatible pair freshness verification, real HTTP service-targeted PageRank/eigenvector/paired-HITS active-read coverage now proves the service owner path preserves prior published direct/traversal/rerank results while a rebuild is active, PageRank/eigenvector/paired-HITS publish-verifier failure preserves prior output through service-targeted coordinators, degree/PageRank/eigenvector/HITS have service-targeted publish/cleanup restart proof across coordinator publish, duplicate coordinator idempotence, and worker-pool cleanup resume, degree/PageRank/eigenvector/paired-HITS now have 130-source two-worker service multi-page proofs for non-iterative, single-vector iterative, and paired-vector iterative phase families, and direct-process page-reclaim proofs now read the durable page record to require replacement-worker completion under a newer attempt before stale-owner completion is rejected. The process harness now enforces and emits a final `graph_metric_process_harness_summary` JSON event with `remote_owner_release_gate: true` only after launch, service-owner restart, service publish/cleanup, service publish-failure, service multi-page worker-pool, service active-read, direct publish/cleanup, direct publish-failure, direct active-read, page-reclaim, fixed-iteration, exhausted-attempt, and same-worker fencing coverage all reach their required category counts. That event now also gates explicit service multi-page phase floors: 27 worker phase proofs, 31 coordinator phase proofs, and 8 takeover phase proofs across degree, PageRank, eigenvector, and paired HITS. The remaining remote adapter work is promotion-scale deployment evidence and rollout hardening rather than another owner-boundary proof. | One coordinator and multiple workers drain degree, PageRank, eigenvector, and paired HITS through durable state only. Killing a worker abandons only its page/runtime lease; replacement after expiry completes the page; stale workers cannot complete reclaimed pages; duplicate coordinators cannot publish or fail twice; and active single-vector or paired-HITS service rebuilds do not expose attempt output through public reads. | +| 2 | Degree canary rollout | Put degree behind the first internal distributed-maintenance gate. Degree should exercise scan, reduce, publish, cleanup, page leases, runtime leases, status, and freshness without iterative convergence risk. The first gate is now implemented as an internal decision that admits only one queued/active degree build, blocks non-degree queued/active work, rejects failed/truncated page summaries, and applies a configurable control-record cap before planned rollout proceeds. Queued degree builds are now estimated against that cap before job/page records are created. Planned maintenance reports `rounds_executed`, so canary qualification can assert tight budgets exhaust resumably and expanded budgets finish within their configured round cap. That gate is now selectable through the internal `degree_canary` DB idle maintenance mode, which runs planned degree work when eligible, falls back to local maintenance when queued work is blocked before planned execution starts, and fails fast when already-active planned work is outside guardrails. Direct top-k, traversal projection, and search rerank `published`/`fresh` coverage now runs through the canary path, service-route owner path, real HTTP service owner path, and focused hosted cross-range fan-in while a planned rebuild is active. Repeated failed degree builds now also prove abandoned score/job namespaces are cleaned immediately and recent diagnostics stay bounded. The unit-test aggregate now includes the canary guardrails. | Local-vs-distributed parity, process restart, stale-attempt rejection, cleanup resume, active/failed status summaries, public `published`/`fresh` behavior, scheduler latency budget, storage-growth bounds, and rollback to the local oracle all pass in CI or release qualification. | +| 3 | PageRank production promotion | Promote PageRank after the single-vector iterative matrix is complete. Keep dynamic iteration planning, convergence summaries, fixed-iteration non-converged publish metadata, publish verification, prior-generation preservation, and cleanup as required checks. Process coverage now includes killed-worker exhausted-attempt failure for a later contribution page plus a duplicate coordinator tick that cannot fail the build twice, and service-targeted process coverage includes the active public-read contract for direct top-k, traversal projection, and search rerank. Hosted cross-range fan-in now includes active PageRank shards in both the focused two-shard public-read matrix and a nonuniform eight-shard direct merge layout; both keep the prior published generation mergeable while `fresh` fails closed. Graph-level repeated failed-build coverage now proves abandoned PageRank score/job namespaces are cleaned while diagnostics stay bounded, and unit-test lifecycle coverage now includes later-iteration exhausted-attempt preservation, failed planned-rebuild public-read preservation for direct top-k/traversal/rerank, and DB-level production-budget parity: a 130-source PageRank build exhausts and resumes through one-page planned maintenance rounds, then matches the local oracle status and top-k output. Promotion still needs deployment-scale cleanup, broader hosted fan-in surfaces, and latency evidence. | Distributed PageRank matches the local oracle within tolerance, preserves the previous generation on failed rebuilds, serves prior published scores while building, fails `fresh` with `MetricStale` when stale, recovers from every phase boundary, and passes promotion-scale cross-shard fan-in checks. | +| 4 | Eigenvector single-vector parity | Reuse the PageRank substrate for eigenvector rather than adding metric-specific job state. Eigenvector should contribute only metric math, metadata, and tolerance rules. Service-targeted process coverage now includes the active public-read contract for direct top-k, traversal projection, and search rerank, service-owner replacement coverage, two-worker service-boundary multi-page scan/initialize/contribution/reduce/convergence evidence, hosted cross-range fan-in for active eigenvector direct metric top-k/traversal/rerank while `fresh` fails closed plus nonuniform eight-shard active/stale direct merge coverage, publish/cleanup restart and publish-verifier failure through the service owner boundary, killed-worker exhausted-attempt failure for a later contribution page plus duplicate coordinator idempotence through the process harness, unit-test later-iteration exhausted-attempt preservation, unit-test failed planned-rebuild public-read preservation for direct top-k/traversal/rerank, graph-level repeated failed-build cleanup/storage-growth proof, unit-test scan/initialize/contribution/reduce/convergence reclaim coverage, cleanup resume after reopen, failed planned-build preservation, coordinator publish-failure preservation after reopen, and DB-level production-budget parity through a 130-source one-page-budget planned drain matching the local oracle. | The eigenvector matrix still needs deployment-scale parity, broader promotion-scale fan-in, and latency evidence before promotion. | +| 5 | HITS paired-vector promotion | Keep authority and hub as separate named metric scores with one compatible lifecycle. Authority and hub must share target generation, convergence decision, publish decision, failure decision, and cleanup ownership. Focused hosted fan-in now covers an eight-shard active/stale compatible authority/hub layout whose direct, traversal projection/order/filter, and authority-rerank merges preserve the prior pair and reject `fresh`, while real HTTP service-owner active reads, service-owner replacement coverage, two-worker service-boundary multi-page authority/hub phase coverage, first service killed-owner timing for coordinator and scan worker-pool replacement, killed-worker exhausted hub-reduce attempt coverage through the direct process harness and unit-test lifecycle coverage, unit-test active prior-pair visibility, paired publish idempotence, initialize/contribution/reduce/hub/convergence reclaim, cleanup resume after reopen, failed-build preservation, publish-failure preservation, scan-page reclaim coverage that rejects stale partial writes, failed planned rebuild public-read coverage for prior compatible authority/hub output, paired fixed-iteration metadata after service-owned publish and cleanup, service-targeted publish/cleanup restart, service-targeted publish-verifier failure preservation, graph-level repeated failed-build cleanup/storage-growth proof, and DB-level production-budget parity through a 130-source one-page-budget paired drain cover the paired-HITS `published`/`fresh` contract for direct top-k, traversal projection/order/filter/status, authority rerank, compatible pair freshness after replacement, compatible pair cleanup after publish, failed/exhausted-attempt prior-pair preservation, bounded failed-build diagnostics, and local-oracle parity under bounded scheduler budgets. | HITS remains disabled by default until promotion-scale fan-in, broader deployment-scale owner evidence, and larger-graph latency evidence all preserve the previous compatible pair. | +| 6 | Public read-surface closeout | Treat every score-bearing read path as part of distributed correctness. Building, failed, abandoned, and attempt-scoped output must remain invisible outside status. Unit-test lifecycle coverage now runs the fast-root query/profile/fan-in invariants for direct metric top-k, graph traversal/search metric status, order/filter generation checks, rerank score details, failed status preservation, paired HITS failed-status preservation, malformed shard payloads, and profile generation reporting. Unit-test fan-in coverage combines that fast-root artifact with hosted cross-range graph metric fan-in for compatible published generation merge, unpublished/incompatible generation rejection, nonuniform eight-shard hosted degree/PageRank/eigenvector direct merge coverage with four active/stale shards and invisible unpublished targets, active-stale hosted degree/PageRank/eigenvector traversal projection/order/filter and search rerank published merge plus fresh rejection, compatible HITS authority/hub hosted direct, traversal projection/order/filter, and authority-rerank merge coverage over an eight-shard layout with four active/stale shards plus fresh rejection, remote HITS generation/metadata/edge-filter mismatch rejection, missing remote HITS status rejection, and serializer coverage for representable single-metric shard reads/rerank. Unit-test public API graph metric coverage combines the public graph-query e2e with the public graph metric action route plus generated OpenAPI/client contracts for graph metric status, runtime ownership summaries, active build pages, and graph metric action responses. | Direct top-k, traversal projection/order/filter/status, search rerank, query profile, hosted fan-in, public action/status routes, generated clients, and any future standalone explain surface either read compatible complete published generations or fail closed with `MetricNotReady`/`MetricStale`. Mixed shard generations, missing statuses, zero generations, incompatible edge filters, incompatible HITS pairs, and malformed score/status payloads are rejected before merging. Full promotion-scale shard layouts remain the separate release gate. | +| 7 | Operations and cleanup release gate | Make the executor safe to leave enabled. V1 keeps latest-only retention, bounded diagnostics, immediate cleanup when snapshot-safe, and deferred internal cleanup only when needed for readers. Degree, PageRank, eigenvector, and HITS now have service-boundary cleanup restart evidence after publish plus graph-level repeated-failed-build cleanup/storage-growth evidence across non-iterative, iterative, and paired-vector families. The local promotion-budgeted release summary now also exposes configured/observed operations-floor evidence for active page probes, active leased/detailed status pages, terminal no-work status, untruncated work/status pagination, and finite progress. The release-qualification active-page probe now persists cursor progress after reclaim and rejects status that omits the reclaimed page cursor or completed/total progress units. Unit-test operations coverage now also runs the capped active-page status test that requires every reported active page to carry worker, lease, attempt, cursor/error, and progress-unit details. Release qualification still needs larger retained namespaces and promotion-scale storage-growth evidence under deployment-shaped owner load. | Cleanup resumes after restart for score generations, manifests, pages, phase/iteration summaries, attempts, failures, and runtime-owner records. Direct runtime-owner lease-record tests now prove clean shutdown deletes the current owner record and stale shutdown preserves a replacement owner. Status exposes stable summaries for freshness, phase/iteration progress, owner hashes, worker counts, lease state, takeover/lost-lease counters, page counters, bounded failures, and cleanup progress without raw storage keys or attempt namespaces. | +| 8 | Default widening | Widen the conservative `auto` gate one family at a time: degree, PageRank, eigenvector, then HITS. Keep local runners as CI/debug oracles until distributed parity and operations evidence are routine. | A metric family becomes distributed-by-default only after parity, remote ownership, crash/restart, cleanup, public-read, fan-in, status, operations, latency-budget, generated-client, docs, and rollback checks are all green for that family. | + +The critical dependency is owner correctness before default promotion. A family +must not become distributed-by-default while duplicate coordinators can race +publish/fail, duplicate workers can write stale attempts, cleanup can grow +without bound, or fan-in can merge incomparable generations. Degree is the +canary because it proves the mechanics cheaply; PageRank proves iterative +single-vector execution; eigenvector proves the substrate is generic; HITS +proves paired-vector failure and publish semantics. + +Recommended follow-up release cuts from the current checkpoint: + +1. **Promotion-scale remote owner evidence**: keep the existing service-targeted + owner boundary as the contract, then widen larger coordinator/worker-pool + deployments against realistic page counts and failure timing. This PR should + not add a new public job API. Degree, PageRank, eigenvector, and paired HITS + now have first two-worker service-boundary multi-page proofs for + non-iterative, single-vector iterative, and paired-vector iterative phase + families. Degree, PageRank, eigenvector, and paired HITS now also have first + multi-page service killed-owner timing proofs: killed coordinators and killed + worker-pools fence duplicate owners before TTL, allow replacement takeover + after TTL, and continue the same 130-source builds through publish and + cleanup. The process-harness summary now makes those killed-owner timing + proofs auditable as explicit required/observed service multi-page + coordinator-takeover and worker-pool-takeover counters, plus a separate + required/observed service cleanup-takeover counter for killed cleanup owners + after publish. It also breaks `remote_owner_release_gate` into + `service_remote_owner_release_gate`, `direct_remote_owner_release_gate`, and + `failure_reclaim_release_gate` so promotion tooling can tell whether missing + evidence is service-boundary, direct-boundary, or failure/reclaim coverage. + Degree, PageRank, eigenvector, and paired HITS service-targeted cleanup now + also have killed-owner timing after publish: a cleanup worker-pool can die + mid-cleanup, duplicate cleanup ownership is fenced before TTL, and a + replacement owner takes over after TTL and finishes to fresh scores, + fixed-iteration metadata, or a fresh compatible HITS pair. The + future rollout evidence should reopen + the DB handle between every progressing maintenance tick to prove durable + resume for degree, PageRank, eigenvector, and paired HITS. The remaining work + is to extend killed-owner evidence to promotion-scale deployments and keep + runtime summaries bounded, no-error, and useful. The process harness now + proves reclaimed pages are completed by the replacement worker under a newer + attempt before the stale owner is rejected, so stale workers cannot silently + complete reclaimed direct-process pages. The process-harness summary now + makes those direct abandoned-attempt checks auditable with separate + required/observed reclaimed-attempt-completion and stale-attempt-rejection + counters in addition to the generic direct page-reclaim phase count. +2. **PageRank production gate**: finish the iterative single-vector promotion + matrix under remote owners. DB-level production-budget local/planned parity + now covers a 130-source one-page-budget planned drain against the local + oracle. Existing graph-level and process-level coverage also covers + later-iteration restart/retry, fixed-iteration non-converged publish + metadata, publish-verifier failure preserving prior output, + exhausted-attempt failure under killed process owners with duplicate + coordinator idempotence, failed planned-rebuild public-read preservation for + direct top-k/traversal/rerank, cleanup resume, cleanup killed-owner + replacement, and active public-read freshness through local, planned, + service-owned, and hosted fan-in paths. The remaining work is promotion-scale remote-owner + deployment cleanup, larger cross-shard fan-in, and latency data. + PageRank promotes only when `published` and `fresh` reads behave the same + under local, planned, service-owned, and hosted fan-in execution. +3. **Eigenvector parity gate**: reuse the PageRank single-vector executor + instead of adding metric-specific job machinery. DB-level + production-budget parity now covers a 130-source one-page-budget planned + drain against the local oracle. Unit-test lifecycle coverage now covers scan, + initialize, contribution, reduce, and convergence reclaim; cleanup resume + after reopen; failed planned-build preservation; coordinator publish-failure + preservation after reopen; and failed planned-rebuild public-read + preservation for direct top-k/traversal/rerank while `fresh` fails closed. + The remaining work includes deployment-scale parity, operations evidence, + promotion-scale fan-in, and latency data. +4. **HITS paired-vector gate**: treat HITS as the last default-promotion family. + Authority and hub stay separate named metrics, but compatible configs share + one target generation, convergence decision, publish decision, failure + decision, and cleanup owner. DB-level production-budget parity now covers a + 130-source one-page-budget planned drain against the local compatible-pair + oracle. Scan-page reclaim now rejects stale partial writes before replacement + completion. Failed planned rebuilds now preserve the prior compatible pair + for authority/hub direct `published` reads, traversal projection/order/filter + and status, and authority rerank; omit failed-generation attempt output; and + make `fresh` reads fail with `MetricStale`. Direct process coverage now also + exhausts a killed hub-reduce page attempt sequence, fails the compatible pair + once, preserves the previous pair, and fences duplicate coordinator failure. + This cut must still finish promotion-scale fan-in, broader deployment-scale + owner evidence, operations evidence, and larger-graph latency + before any default HITS execution is enabled. +5. **Cleanup and storage-growth gate**: broaden the current latest-only cleanup + proof from graph-level repeated failed builds and service-boundary cleanup + restart into deployment-scale qualification. Degree, PageRank, eigenvector, + and paired HITS now add service-boundary killed-owner cleanup proofs after + publish. Degree, PageRank, eigenvector, and HITS now prove repeated failed + planned builds remove abandoned job namespaces and unpublished score + generations while bounding recent diagnostics. Unit-test coverage now makes that evidence explicit in CI: cleanup pages resume after + reopen, active jobs refuse direct cleanup, failed and repeated failed builds + remove abandoned namespaces while bounding diagnostics, and runtime owner + lease cleanup is fenced. Future rollout evidence should add aggregate + storage-footprint evidence for degree, PageRank, eigenvector, and + paired HITS: successful planned cleanup leaves zero durable job namespace + records, zero attempt records, and exactly one retained metric-control + record; repeated failed builds preserve the prior score-record count, remove + abandoned failed job and attempt namespaces, verify retained metric-record + counts against published score records plus fixed per-metric metadata and + bounded failure diagnostics, emit retained + score/metric/control/job/attempt/failure/event record counts, and keep + retained control/failure/event record counts bounded under the configured + ceiling. The harness now also records cleanup tick count and cleanup elapsed + time from the planned-maintenance/status boundary; zero cleanup ticks is + valid when a tiny workload finishes cleanup in the publish round, while + larger iterative and paired runs expose separate cleanup cost. Promotion + release qualification now requires a configured cleanup latency ceiling and + fails the observed deployment-shaped gate when the max cleanup phase exceeds + that ceiling. Promotion qualification also exposes failure-churn as its own + configured/observed floor: the promotion profile must request the repeated + failed-build floor and bounded diagnostics, every completed family must + observe that retry volume, paired HITS must report compatible paired + diagnostics, and failed cleanup must leave zero job and attempt namespaces + with only bounded failure/event records retained. Rollout qualification should + also close and reopen the planned DB between + nonterminal maintenance ticks to prove durable scheduler state resumes after + handle/owner restart boundaries. The remaining gate is larger retained + namespaces, deployment-shaped killed-owner churn with abandoned in-flight + attempts, and promotion-scale storage growth under longer-running owner + churn. Larger retained namespaces can be a future admin/debug option, but v1 + correctness should not depend on a retention knob. +6. **Latency and default-widening gate**: widen `auto` one family at a time only + after scheduler tick latency, public read latency, cleanup cost, and storage + growth are measured. The order stays degree, PageRank, eigenvector, then + HITS. Multi-metric indexes, larger single-vector workloads, default HITS, + and incompatible HITS pairs remain gated until fairness and fan-in coverage + are routine. The unit-test aggregate now makes the current + conservative default boundary explicit in CI: safe degree/PageRank/eigenvector + cases and compatible opt-in HITS can use planned maintenance, larger or + incompatible cases fall back before local execution, and threshold widening + remains an intentional internal decision rather than an accidental default. + Rollout qualification should include latency budgets for local oracle + publish, planned publish, cleanup, published reads, fail-closed fresh reads, + and synthetic fan-in merge/fail-closed paths. Promotion evidence should + include conservative public-read, fresh-failure, and fan-in latency ceilings + so active/failed direct, + traversal, rerank, paired HITS, and fan-in read paths become pass/fail + evidence instead of log-only measurements. It also has disabled-by-default retained score/metric/control/failure/event + storage, page-claim, cleanup-tick, executed-round, failure-retry, worker-step, and + coordinator-step budget flags so promotion runs can turn observed storage and + scheduler footprints into pass/fail gates after real baselines exist, without + baking arbitrary timing, storage, page-count, retry-count, or role-step + thresholds into PR CI. +7. **Operations and client surface gate**: stabilize the status fields exposed + through OpenAPI, generated clients, docs, and dashboards. Operators should + see freshness, phase/iteration progress, owner hashes, worker counts, lease + state, takeover/lost-lease counters, page counters, bounded failures, cleanup + progress, and last error summaries. They should not see raw storage keys, + page ids, attempt namespaces, or process-local writer details as public API. + Unit-test operations coverage now makes capped active-page status + details, failed-page cursor/progress/error diagnostics, the + runtime-summary OpenAPI/client shape, graph index encoders, + internal service maintenance route, and command/supervisor telemetry contract + explicit in PR and full-default Zig CI. Unit-test public API graph metric coverage now also + pins the public action route and generated client-facing graph metric status + types so client drift is caught with the public graph metric read surface. + +The remaining promotion work should be tracked as rollout evidence rather than +PR unit coverage or graph-specific build targets. The local process harness is +the canonical rollout-summary producer: it accepts `--profile smoke` and +`--profile promotion`, emits one `graph_metric_process_harness_summary` row, and +reports top-level rollout, public-read, remote-owner, service, direct, and +failure/reclaim gates. Hosted or deployment-sized qualification should consume +that summary shape instead of adding standalone graph-metric Make targets. + +- The rollout qualification path + runs degree, PageRank, eigenvector, and paired HITS through the graph-index + metric API, drains planned maintenance with configurable graph size, worker + count, tick budget, metrics-per-round budget, page budget, iteration cap, + deterministic graph fanout, and maintenance mode, compares planned top-k + output against the local oracle, and checks that an active rebuild keeps + `published` reads on the prior generation while `fresh` fails closed. During + that active rebuild it also verifies that status reports the prior published + generation, target building generation, active job id, finite progress in the + `[0, 1]` range, and a bounded page-status payload. During that check the + harness deliberately claims one durable build page with a stale probe worker + id, verifies another worker cannot steal the live lease early, reclaims that + same page after its stored lease expiry, and then requires active status to + expose the reclaimed page's worker/attempt/lease summary while pending-work + stats count an active page. The active rebuild can be preceded by multiple + independent write/derive mutation cycles with + `--active-mutation-writes`, and the harness requires the active target + generation to advance by exactly that count. It emits both the resulting + active target generation and generation delta so release runs can prove dirty + graph generations coalesce behind one unpublished rebuild while `published` + remains pinned to the prior complete generation. Active rebuilds also exercise + graph traversal projection, metric ordering, metric filtering, and graph + search rerank: `published` reads must use the prior complete generation and + report the active state, while `fresh` traversal/rerank reads must fail with + `MetricStale`. The same active and failed published-read windows now encode + public query profile output and require explainable graph metric entries for + the direct metric source, graph traversal source, and graph metric rerank + source, all reporting the prior published generation and the observed + building or failed status. The HITS run also issues paired authority/hub + direct metric reads plus paired traversal projection and graph search rerank + reads during active and failed rebuilds: `published` must return + prior-generation results and paired traversal projection/profile output must + include both authority and hub metric scores, while `fresh` reads must fail + closed. The harness also injects + repeated failed rebuilds after the active-build read check and verifies that + direct, traversal, and graph search rerank `published` reads still serve the + prior generation; direct, traversal, and rerank `fresh` reads still fail + closed; the prior generation is preserved in failed status; and bounded failure + diagnostics stay under a configurable retained-record ceiling across multiple + failed target generations. The harness now also records aggregate metric + storage footprints and verifies successful cleanup leaves no durable job + namespace records, exactly one retained metric-control record remains, exactly + one retained score generation remains for the current graph shape, repeated + failed generations do not increase retained score records, abandoned failed + job namespaces are removed, retained score/metric/control/failure/event + record counts are emitted, retained metric records match the published score + records plus fixed per-metric metadata overhead, and retained failure/event + records match the configured bounded retention window exactly: + `min(failure_repeats, max_failure_diagnostics)` failures and + `min(failure_repeats + 1, max_failure_diagnostics)` events per metric, doubled + for compatible HITS pairs. Failed retained metric records must equal the fresh + retained metric-record count plus the bounded retained failure records. + Retained control records must stay within the combined diagnostic ceiling. It + also verifies that terminal fresh and terminal failed statuses do not expose + active build pages, truncated page payloads, job ids, worker ids, or cursors. + Cleanup tick and elapsed-time evidence must agree: runs with no cleanup ticks + must report zero cleanup latency, while runs that execute cleanup ticks must + report nonzero cleanup elapsed time. + For paired HITS, the harness verifies that authority and hub retained event + and failure diagnostics stay in sync and that paired retained diagnostic + storage remains bounded. With `--reopen-between-ticks`, it also closes and reopens the + planned DB after each progressing nonterminal tick, forcing PageRank, + eigenvector, degree, and paired HITS to resume from durable graph-index state + rather than in-memory scheduler state. The reopen count is checked as a + correctness gate: enabled reopen runs must report one reopen for every + nonterminal maintenance tick, while disabled reopen runs must report none. The + smoke and promotion profiles enable reopen by default; focused control runs + can pass `--no-reopen-between-ticks` to validate the zero-reopen branch. With + `--maintenance-mode split`, the harness drives + coordinator-before, worker-pool, and coordinator-after sweeps directly instead + of the combined idle loop, giving release runs a cheap deployment-shaped + preflight for the same durable coordinator/worker boundary. + Split release runs must now prove that every worker-pool sweep is bracketed by + exactly two coordinator sweeps and that worker-pool sweeps match executed + scheduler rounds, while combined release runs must prove one combined sweep per + tick with no split-role sweeps. + The harness now also samples the DB pending-work graph metric stats: before + draining it requires exactly one canonical queued build and no active pages, + during the active rebuild it requires the scheduler to report active work + rather than queued or failed work, after fresh publish/cleanup it requires no + queued/active/failed page work, and after repeated failed rebuild cleanup it + requires the failed terminal metric to leave no scheduler-visible work. This + turns the operational pending-work surface into release evidence rather than + only ad hoc status output. Before the first planned publish, the harness now + also issues direct graph metric top-k reads, graph traversal reads, and graph + search rerank reads. Direct top-k and rerank reads with both `published` and + `fresh` freshness must return `MetricNotReady`. Traversal projection with + `published` must return null metric scores plus `not_ready` metric status, + while traversal fresh projection and traversal published ordering/filtering + must fail closed with `MetricNotReady`. HITS runs the same pre-publish + not-ready checks for the hub side, so a compatible pair cannot accidentally + expose one side before either side has a complete generation. The same + pre-publish probe checks graph metric status: primary metrics, and the HITS + hub side, must report `not_ready`, no published or building generation, a + queued target generation equal to the current edge generation, finite zero + progress, and no active page payload. + Each family run also builds a + synthetic direct metric fan-in probe from the published top-k + results: compatible shard payloads must merge to the same top-k ordering, an + active/building shard with the same prior published generation must still + merge for `published`, a mixed active shard set with both `building` and + `failed` status must still merge the prior published score generation for + `published` while reporting the higher-severity failed status, the same active + fan-in must fail closed for `fresh`, and a deliberately incompatible + published generation must fail closed through the normal query merge path. + Deliberately incompatible metric metadata + versions and edge filters must also fail closed before score mixing, proving + release fan-in compares metric compatibility as well as generation. Shards + that report mismatched index, metric, or status-name identity must fail closed + before score mixing. A shard that omits the requested metric result must also + fail closed before score mixing. A shard that duplicates a requested metric, + returns an unrequested metric, carries a non-finite score, reports a + non-finite status number, reports out-of-range progress, or reports an + invalid published state such as `not_ready` or `disabled` must also fail + closed before score mixing. The + synthetic shard splitter is itself a correctness gate: shard ranges must + exactly cover the published score set in order and stay balanced to within one + score, so increasing `--synthetic-fan-in-shards` exercises a real merge layout + instead of silently creating a malformed fixture. The active/stale fan-in + breadth is controlled separately by `--synthetic-fan-in-active-shards`, so + promotion runs can prove more than the minimum two active shard statuses while + keeping the same merge layout. + HITS runs an additional paired authority/hub synthetic fan-in probe where each + shard must provide both requested metric results; compatible pairs merge + together, active pairs preserve the prior published generation for + `published`, mixed active pairs with failed and building shard statuses still + preserve the prior compatible pair for `published`, `fresh` rejects active + pairs, a missing authority/hub metric result rejects, + duplicate/unrequested/non-finite score payloads reject, + non-finite status numbers and out-of-range progress reject, mismatched + index/metric/status-name identity rejects, invalid published states reject, + and incompatible authority/hub generation, metadata-version, and edge-filter + pairs reject before merge output is published. +- The rollout runner should emit structured JSONL for each run: graph size, edge count, + metric family, maintenance mode, target generation, tick count, budget + exhaustion, generated graph topology evidence, configured round/metric/page + budgets, worker and coordinator step counts, combined/coordinator/worker-pool + sweep counts, pre-drain metrics-scanned and queued-build counts, pre-publish + not-ready direct-read, rerank, traversal projection, traversal fail-closed, + and status counts, page counts, + phase advances, publish/failure counts, local/planned publish latency, active page probe + claim/reclaim flags, active status page count, leased-page count, + detailed-page count, cursor-bearing page count, progress-bearing page count, + active status truncation count, truncation flag, progress, cleanup tick count, cleanup elapsed time, DB reopen + count, configured worker identity count, split worker identities that made + tick progress, split worker identities that made page claim/completion + progress, split worker min/max page-progress counts, configured active + mutation write count, active target generation, active generation delta, + HITS active paired published-read latency, published-result count, + published-score count, fresh-failure latency, fresh-rejection count, HITS + active paired rerank published-result count, HITS active paired rerank + fresh-rejection count, HITS active paired traversal metric-result + count, active published-read latency, active published-read result count, + active fresh-failure latency, active fresh-rejection count, active + direct top-k score count, active rerank published-read latency, active rerank + published-result count, active rerank fresh-failure latency, active rerank + fresh-rejection count, active traversal published-read latency, active + traversal fresh-failure latency, + active traversal published-check count, active traversal fresh-rejection + count, active profile graph-metric entry count, failed-build published-read + latency, failed-build published-read result count, failed-build fresh-failure + latency, failed-build fresh-rejection count, failed-build rerank + published-read latency, failed-build rerank published-result count, + failed-build rerank fresh-failure latency, failed-build rerank + fresh-rejection count, failed-build traversal published-read latency, + failed-build traversal fresh-failure latency, failed-build traversal + published-check count, failed-build traversal fresh-rejection count, + failed-build direct top-k score count, failed-build profile graph-metric entry count, + configured failure + repeat count, final retry count, configured retained-diagnostic ceiling, + bounded recent event/failure counts, retained expected-error failure-record + counts, retained failed-event counts, paired HITS event/failure/diagnostic + shape counts, + pre-drain/fresh/active/failed paused-metric counts, expected score + record count, synthetic fan-in shard count, min/max scores per synthetic + shard, active synthetic shard count, terminal merged score count, + active-shard published merged score count, mixed-active published merged score + count, active-shard fresh rejection count, + zero-generation, incompatible-generation, metadata-version, edge-filter, + missing-metric, duplicate-metric, extra-metric, non-finite-score, + non-finite-status, out-of-range-progress, identity-mismatch, and invalid-state rejection counts, terminal fan-in merge latency, + active-shard published fan-in latency, mixed-active published fan-in latency, + active-shard fresh fail-closed latency, + zero-generation, incompatible-generation, metadata-version, edge-filter, + missing-requested-metric, duplicate-metric, extra-metric, non-finite-score, + non-finite-status, out-of-range-progress, identity-mismatch, and invalid-state fail-closed latencies, + paired HITS fan-in metric-result count, paired HITS min/max scores per + synthetic shard, paired active synthetic shard count, paired + active-published metric-result count, paired mixed-active published + metric-result count, paired + fresh/zero-generation/generation/metadata-version/edge-filter/missing/duplicate/extra/non-finite/status-non-finite/progress/identity/state + rejection counts, paired terminal/active/mixed-active/fresh-fail/ + zero-generation-fail/generation-fail/metadata-fail/edge-filter-fail/missing-fail/duplicate-fail/ + extra-fail/non-finite-fail/status-non-finite-fail/progress-fail/identity-fail/state-fail fan-in latencies, + HITS failed paired published-read latency, published-result count, + published-score count, fresh-failure latency, fresh-rejection count, HITS + failed paired rerank published-result count, HITS failed paired rerank + fresh-rejection count, HITS failed paired traversal metric-result + count, active/fresh/failed pending-work summaries, fresh/failed status page + counts, terminal status truncation flags, fresh convergence/non-convergence + counts, iterations-completed value, positive-delta count, computed-at count, + local/planned top-k and status parity check counts, + fresh/failed score, metric, control, job-namespace, and attempt-record + counts, failed failure/event record counts, and parity summary. + Those footprint fields are verified, not merely logged: expected, fresh, and + failed score-record counts must match the generated graph shape exactly, + fresh and failed metric-record counts must match score records plus fixed + retained metadata and bounded failed diagnostic records, fresh and failed job + namespace counts must be zero, fresh and failed attempt-record counts must be + zero, the fresh terminal state must retain exactly one metric-control record, + failed failure/event records must match the bounded diagnostic window exactly, + and failed control records must stay within that same diagnostic ceiling. + Repeated-failure diagnostics + are shape-checked too: the terminal failed status must expose a failed last + event, the expected `InvalidGraphMetricScore` last error, retained failure + records with nonzero job ids, target/score generations newer than the + published generation, non-idle phases, retry counts, and the expected error, + plus retained failed events that preserve the prior published generation and + carry no score count. Compatible HITS pairs must satisfy the same diagnostic + shape on both authority and hub. + The emitted graph topology and active-generation fields are also verified as + result-level evidence: node/edge totals, source/sink/authority counts, + sink/cycle/bipartite/self-edge counts, max out-degree, expected score-record + count, active mutation count, active target generation, and active generation + delta must all match the selected family and workload knobs. + Published metadata is a result-level gate too: degree must publish converged + metadata with one completed iteration, zero positive deltas, and a computed + timestamp; PageRank, eigenvector, and compatible HITS must publish computed + timestamps, bounded iteration counts at or below `--max-iterations`, and if + they publish non-converged output, every metric in the family must report + `converged: false`, positive finite delta evidence, and + `iterations_completed == --max-iterations`. + Local-oracle parity is no longer score-only: each family metric must match the + local oracle top-k ordering and scores, and its planned published status must + match the local status for published generation, edge generation, complete + state, metadata version, convergence flag, iteration count, and finite delta + within tolerance. HITS must pass the same parity checks for both authority and + hub. + Optional budget flags can fail a run when local publish, planned publish, + cleanup, published read, fail-closed fresh read, HITS paired direct + published/fresh read, or synthetic fan-in latency exceeds the selected release + baseline; the public-read gates are `--max-published-read-latency-ns` and + `--max-fresh-fail-latency-ns`, and the fan-in gate is + `--max-fan-in-latency-ns`. These threshold checks run after the + family result JSONL is emitted, so a budget failure still leaves the measured + evidence needed to set or adjust the baseline. Separate disabled-by-default storage gates + `--max-storage-score-records`, `--max-storage-metric-records`, + `--max-storage-control-records`, `--max-storage-attempt-records`, + `--max-storage-failure-records`, and `--max-storage-event-records` fail a run + when either the fresh terminal footprint or repeated-failure footprint exceeds + the selected release baseline. Scheduler footprint gates + `--max-page-claims` and `--max-cleanup-ticks` fail a run when page churn or + cleanup scheduler work exceeds the selected release baseline. + `--max-rounds-executed` and `--max-failure-retry-count` make bounded scheduler + progress and bounded failed-build retry volume explicit release gates. + `--max-worker-steps` and `--max-coordinator-steps` bound how much work each + runtime role consumed before publish, cleanup, and failure verification. + `--min-families-run` fails the final summary when a release run completes + fewer metric families than the configured floor, making all-family promotion + coverage an explicit gate instead of an inference from logs. + `--min-split-worker-identities-with-progress` and + `--min-split-worker-identities-with-page-progress` fail a split-mode run when + too few configured worker identities make tick progress or actual page + claim/completion progress. The promotion evidence profile should set both to the + four-worker promotion floor, so promotion evidence cannot pass while silently + serializing page ownership through fewer worker identities. The rollout + runner should finish successful runs with a summary JSONL event that repeats the + configured latency/storage/scheduler budgets, marks whether any budget was + enabled, breaks that budget evidence into latency, storage, scheduler, and + coverage-floor categories, records whether the promotion profile floor was + enforced, and marks `deployment_shaped_release_gate` in the config row when + the invocation requests the promotion profile with all-family execution, split + ownership, reopen evidence, a four-family floor, split worker identity floors + at least as strict as the configured worker count, public-read/fresh/fan-in + latency ceilings, retained storage ceilings, and scheduler ceilings. The + final summary row marks the same flag only after observing all four metric + families, per-family split-worker progress/page-progress floors, and the + promotion fan-in floor: the selected shard count must be exercised for every + family, active/stale shard counts must reach the configured promotion floor, + and the observed primary plus paired-HITS layouts must be nonuniform. Worker + floors use the minimum observed family contribution rather than aggregate + worker totals. That flag is + release-tooling metadata for local evidence: it proves the run used and + satisfied the deployment-shaped local gate with all local promotion budget + categories enabled, but it does not replace hosted remote-owner, killed-owner, + or cross-range promotion evidence. The config and summary rows should expose + configured and observed deployment-shaped evidence flags so rollout tooling can distinguish + a correctly shaped invocation from a completed promotion run that actually met + the observed floors. The config and summary rows also emit the component audit booleans + `all_family_execution`, `public_read_fan_in_latency_budgeted`, + `cleanup_latency_budgeted`, + `retained_storage_budgeted`, `promotion_scheduler_budgeted`, + `split_worker_progress_floor_configured`, and + `split_worker_page_progress_floor_configured`, and + `promotion_fan_in_floor_configured`, and + `promotion_failure_churn_floor_configured`, and + `promotion_operations_floor_configured`; the summary adds + `public_read_fan_in_latency_budget_observed`, + `cleanup_latency_budget_observed`, `retained_storage_budget_observed`, + `promotion_scheduler_budget_observed`, `split_worker_progress_floor_observed`, and + `split_worker_page_progress_floor_observed`, plus + `promotion_failure_churn_floor_observed` and + `promotion_operations_floor_observed`. The latency observed flag is + true only when published-read, fresh-fail, and fan-in latency caps are + configured and the max observed surfaces stay within them. The promotion + operations floor also emits `min_observed_active_status_pages` and + `max_observed_active_status_pages`; the observed flag stays false unless each + family produced at least one active status page and the largest observed + active status page count stayed within `max_status_pages`. The cleanup + latency observed flag is true only when a cleanup latency cap is configured + and the max observed cleanup phase stays within it. The + retained-storage observed flag is true only when configured score, metric, + control, attempt, failure, and event caps are present, every completed family + stayed under those caps, retained score/metric/control/failure/event record + counts match the expected fresh and failed generations, and cleanup left zero + job and attempt namespace records. The scheduler observed flag is true only + when page-claim, cleanup, round, retry, worker-step, and coordinator-step caps + are configured and the completed summary stayed within them while publishing + exactly the completed families. The worker observed flags are true only when + the matching floor was configured and every completed family met it, plus + `promotion_fan_in_floor_observed`, which is true only after the final summary + proves every completed family used the configured shard/active-shard shape and + the expected nonuniform primary or paired-HITS fan-in layout, and + `promotion_failure_churn_floor_observed`, which is true only after repeated + failed-build retries, bounded diagnostics, paired-HITS diagnostic records, and + zero retained failed job/attempt namespaces match the promotion floor. The + operations observed flag is true only when every completed family exposes an + active page probe, the reclaimed page is fenced from stale completion, the + reclaimed probe page persists cursor-bearing progress, active status pages + carry leased, detailed, cursor-bearing, and progress-bearing page counts, + active status truncation is zero, terminal fresh/failed work is empty and + untruncated, and active progress is finite and strictly between zero and one, + proving the status sample is in-flight rather than empty or terminal. Rollout tooling + can therefore tell which local promotion-gate category is absent without + reverse-engineering every raw knob. When a deployment-shaped rollout + requirement is set, the runner should require the completed summary to satisfy + the observed deployment-shaped evidence flag; + a correctly shaped invocation that fails to produce the observed family, + worker, fan-in, failure-churn, public-read latency, cleanup latency, storage, + or scheduler evidence fails the promotion run instead of only emitting a + false summary flag. + The summary repeats the selected workload shape for documents, fanout, top-k, + synthetic fan-in shards, + synthetic active fan-in shards, active mutation writes, workers, max + ticks, rounds per tick, metrics per round, pages per round, max iterations, + failed rebuild repeats, retained diagnostics, status pages, reopen mode, and + tolerance, repeats the matching promotion floor values, records the number of + metric families run, records explicit degree/PageRank/eigenvector/HITS family + counters, and reports total and maximum observed latency across those families + for local oracle publish, planned publish, published public reads, + fail-closed fresh reads, and synthetic fan-in. When a release run requests a + four-family summary floor, those counters are verified so a promotion summary + cannot pass unless each family ran exactly once. The config JSONL row + emits the same selected workload, floor, and budget-category fields before + work starts, so release tooling can classify failed or interrupted runs and + prove a successful promotion run used at least the required workload shape + from either the starting config or final summary while still retaining the + per-family records for diagnosis. The summary row now aggregates + graph-shape and active-generation + proof: total actual and expected graph nodes/edges, source/sink/authority + nodes, sink/cycle/bipartite/authority-self edge components, maximum observed + out-degree, total successful-generation repeats and delta, min/max + per-family successful-generation repeat and delta counts, total active + mutation writes, and total active generation delta. + Those totals are verified against the configured document count, fanout, + active mutation count, and degree/PageRank/eigenvector/HITS family mix, so + promotion tooling can prove from the final row that the run exercised the + intended workload shape before trusting latency, storage, or public-read + evidence. The summary row now aggregates scheduler execution proof too: + maximum allowed total ticks, observed ticks, budget-exhausted family count, + combined/coordinator/worker-pool sweeps, maximum allowed drain scheduler + scans, observed drain scheduler scans, pre-drain scan totals plus maximum + allowed pre-drain scans, active-build observations, expected and observed + build starts, worker steps, coordinator decision count, coordinator steps, + maximum allowed page claims, page claims/completions, phase advances, expected + and observed publishes, expected and observed failed builds, maximum allowed + executed rounds, observed executed rounds, total configured worker identities, + total split worker identities that made tick progress, and total split worker + identities that made actual page claim/completion progress. These totals are + verified against the selected combined or split maintenance mode and selected + tick/round/page/metric-scan budgets for both drain and pre-drain scans, so a + release summary cannot pass while a combined run silently uses split-role + sweeps, a split run omits the coordinator-before/worker-pool/coordinator-after + shape, a split run reports impossible worker ownership totals, the aggregate + scheduler footprint exceeds the configured resumable bounds, or the run + finishes without build start, active-build observation, worker page, + coordinator, phase-advance, publish, zero-failure evidence, and + budget-exhaustion evidence whenever a family needed multiple ticks to finish. + Page-claim, executed-round, worker-step, and coordinator-step evidence now + also emits min/max per-family bounds and requires nonzero minima, so the final + row cannot hide a family that did no scheduler work behind aggregate totals. + Page-completion totals must cover phase advances, and aggregate coordinator + decisions must fit inside the observed coordinator step count, so the summary + row cannot claim publish/phase/failure decisions that were not accounted for + by scheduler execution. + When worker-identity floors are configured, the final summary also + verifies that each completed family contributed the required number of + progressing and page-progressing worker identities by checking the minimum + observed family counts, not only aggregate totals or per-family rows. The same summary row records total reopen count, + cleanup ticks, and cleanup latency. Reopen totals are verified against + `reopen_between_ticks`, so reopen-enabled release runs prove a DB-handle + boundary after every nonterminal tick, while non-reopen runs cannot report + synthetic reopen evidence. Cleanup tick and latency totals are verified as a + pair, so release tooling can distinguish no separate cleanup work from + measured cleanup work without relying on per-family rows. The summary latency + totals are verified too: successful runs must report nonzero local/planned + execution, public-read, fresh-failure, and fan-in latency evidence. The final + row emits min/max per-family latency evidence for those required surfaces, and + verification requires every minimum to be nonzero, every minimum to be no + larger than its maximum, and every maximum to be bounded by the matching + total. When latency, + storage, page-claim, cleanup-tick, executed-round, failed-retry, worker-step, + or coordinator-step ceilings are configured, the final summary re-checks + those ceilings against the observed maxima, so promotion tooling can trust the + final row without replaying every per-family record. Retained metric-record + expected totals are emitted next to the observed totals, along with the metric + slot count and fixed per-metric metadata overhead used to compute them. The + same storage block emits the expected fresh control-record total and the + failed control-record ceiling derived from retained diagnostics. Those + expected totals are re-checked in the final row against the generated + score-record count, the fixed per-metric metadata overhead, and retained + failed diagnostic records, so storage-growth evidence cannot pass while + hiding extra metric namespace entries or unbounded control records. The same + block now emits min/max observed retained score, metric, control, attempt, + failure, and event records; deployment-shaped retained-storage evidence stays + false unless every family retains nonzero score/metric/control/diagnostic + evidence, attempt namespaces are fully cleaned up, and the max bounds remain + within the configured ceilings. + Failed-build + diagnostic totals are included as status-level evidence too: total retry + count, recent failure/event counts, retained expected-error records, retained + failed-event records, expected and observed storage failure/event records, and + the paired-HITS equivalents. Those totals are verified against + `failure_repeats`, `max_failure_diagnostics`, and the HITS paired-family + count, so final-row release evidence proves bounded diagnostics + were retained and shaped correctly, not only that the underlying storage + record counts stayed bounded. The summary row also aggregates operational + scheduler-state proof: total pre-drain queued builds, min observed pre-drain + scheduler scans, min observed queued builds, total fresh-terminal pending work + plus its active-build/page, failed-page, paused-metric, and truncation + components, fresh terminal status page and truncation counts, total active + builds, active pages plus min/max observed per-family active page bounds, + active failed-page, paused-metric, and truncation components, active status + pages, active page-probe claim and reclaim counts, + active leased pages, active detailed pages, active cursor-bearing pages, + active progress-bearing pages, active status truncation count, the observed active-progress range, total failed-terminal pending work plus its active-build/page, + failed-page, paused-metric, and truncation components, and failed terminal + status page and truncation counts. Those + totals are verified against the families that actually ran, so release + tooling can prove from the final row that every family started as exactly one + queued build, every family had nonzero scheduler scan and active page + evidence, every active rebuild exercised a claim plus post-expiry reclaim of a + durable build page, active rebuilds exposed bounded leased page detail with + finite in-flight progress, and fresh and failed terminal states left + no scheduler-visible work, paused metrics, truncated page summaries, or active + page payloads. The summary row + also aggregates the public read-surface proof: + expected and observed primary pre-publish `MetricNotReady` surfaces, expected + and observed paired-HITS pre-publish `MetricNotReady` surfaces, expected and + observed primary published-read surfaces, expected and observed primary fresh + rejections, total graph metric profile entries, expected and observed + paired-HITS published-read surfaces, expected and observed paired-HITS fresh + rejections, total primary fan-in rejections, and total paired-HITS fan-in + rejections. + The same final row breaks active and failed public reads down by surface: + direct metric reads, search rerank reads, traversal projection/order/filter + checks, and paired-HITS direct/rerank/traversal reads. Those read-type + counters are verified independently from the expected aggregate surface + totals, so a release summary cannot pass by exercising only one public read + path while reporting the expected total. + Direct published-read score totals are included for active and failed + rebuilds, including paired-HITS active and failed direct score totals, and + are verified against top-k and family shape. That makes prior-generation + preservation visible from the final row instead of requiring per-family + result reconstruction. + It also records successful fan-in merge evidence with expected and observed + totals: total synthetic shards, min/max shard score counts, total merged + primary scores, active-stale published primary scores, mixed failed/building + published primary scores, primary nonuniform-layout count, paired-HITS merged + metric results, paired active shard counts, paired active/mixed published + metric results, and paired HITS nonuniform-layout count. Successful fan-in + totals are verified against top-k, shard count, and the + degree/PageRank/eigenvector/HITS family mix, including whether the selected + workload should produce uniform or nonuniform primary and paired-HITS shard + layouts. Promotion summaries now expose that as a single + configured/observed fan-in floor so deployment-shaped runs cannot pass after + silently falling back to a uniform or too-small fan-in fixture. Expected + aggregate rejection totals are emitted next to observed + totals, and rejection totals are broken down by category for both primary and + paired-HITS fan-in: active `fresh`, + zero-generation, incompatible generation, metadata-version, edge-filter, + missing requested metric, duplicate metric, extra metric, non-finite score, + non-finite status number, out-of-range progress, identity mismatch, and + invalid published state. Those category totals are verified independently, so + a final summary cannot satisfy the fan-in gate by producing the right + aggregate rejection count while skipping one malformed or incompatible shard + class. + Those summary totals are verified against the families that actually ran, so + promotion tooling can prove from the final row alone that first-publish + direct reads, traversal reads, rerank reads, status checks, active and failed + read paths, profile entries, paired-HITS read paths, compatible fan-in merges, + active-stale fan-in preservation, mixed failed/building fan-in preservation, + and fan-in fail-closed paths were exercised instead of only inferring that + from per-family rows. The + summary row also aggregates local-oracle parity and published-status metadata + proof: expected and observed top-k parity checks, expected and observed + status parity checks, minimum expected and observed converged status counts, + non-converged published status counts, minimum expected and observed + positive-delta metadata counts, expected and observed computed-at metadata + counts, and the min/max completed-iteration counts seen across the families + plus the configured maximum allowed iteration count. Those totals are + verified against the family and metric counts, so a release summary cannot + pass while omitting planned-vs-local parity, computed publish timestamps, + bounded iteration metadata, or the non-converged positive-delta evidence + required for fixed-iteration iterative metrics. The summary row also + aggregates storage-footprint proof: + total metric slots, fixed retained metadata overhead per metric, total + expected score records, fresh and failed retained score records, expected and + observed fresh/failed metric-record counts, fresh and failed control record + counts, the expected fresh control count, the failed control-record ceiling, + fresh and failed job namespace counts, fresh and failed attempt-record counts, + and expected plus observed retained failed diagnostic failure/event records. + Those totals + are verified against the generated graph shape, the fixed retained + metric-record overhead, the number of metric families, the compatible HITS + pair multiplier, and the bounded diagnostic window, so release tooling can + prove from the final row that fresh and failed terminal states retained only + published score output plus bounded metadata, removed job and attempt + namespaces, kept exactly one fresh control record per family, and bounded + failed diagnostics across the whole run. The + summary row also reports the minimum split worker identities with tick + progress, the minimum split worker identities with page claim/completion + progress, the minimum per-family active-worker page progress, and the maximum + per-family active-worker page progress. Promotion tooling can therefore + reject a run where one family silently serialized all page ownership through a + single worker while another family supplied the maximum worker-step evidence. + The rollout runner should stay out of the public Make/build target surface. Release + recipes can pass `--profile smoke` or `--profile promotion` plus the selected + latency, retained-storage, scheduler, worker-identity, and family-floor + budgets as rollout-tooling inputs instead of adding feature-specific public + build targets. + The budgeted smoke recipe uses loose local/planned/cleanup latency runaway + guards plus conservative public-read, fan-in, retained-storage, page-claim, + cleanup-tick, executed-round, failed-retry, worker-step, coordinator-step, + attempt-record, and four-family summary budgets, giving PR validation a fast + pass/fail exercise of every release-budget category without claiming + production latency baselines. The promotion recipe runs the larger + release-profile floor with conservative pass/fail budgets for published-read + latency, fresh-failure latency, fan-in latency, retained storage records, + page claims, cleanup ticks, executed rounds, failed rebuild retries, worker + steps, coordinator steps, and minimum split worker identity progress, plus a + four-family summary floor. Retained storage budgets now + include explicit attempt-record ceilings in addition to score, metric, + control, failure, and event records, while the cleanup invariant still + requires zero retained attempt records after fresh and failed cleanup. The + promotion recipe should require deployment-shaped evidence before accepting a + run as rollout-ready. + The required shape now includes the promotion workload floor, all-family + execution, split/reopen ownership evidence, four-family coverage, + split-worker progress floors, promotion fan-in and failure-churn floors, + promotion operations/status floors, public-read/fresh/fan-in latency + ceilings, cleanup latency ceiling, retained-storage ceilings, and scheduler + ceilings. That makes the local + promotion gate summary a required shape rather than best-effort metadata + before deployment rollout tooling consumes the larger hosted evidence. Both + the starting config row and final summary row expose the component booleans + for all-family execution, public-read/fan-in latency budgets, cleanup latency + budget, retained-storage budgets, promotion scheduler budgets, promotion + fan-in floors, promotion failure-churn floors, and promotion operations + floors; the config row reports the + requested shape, while the summary row reports the observed result. The + promotion recipe intentionally leaves local/planned wall-clock + latency thresholds disabled until deployment baselines exist, while cleanup + latency is now part of the deployment-shaped promotion requirement. The + smoke recipe keeps only loose runaway guards for local, planned, and cleanup + phases. The runner accepts `--profile smoke` and `--profile promotion` for + focused family or budget overrides. The process-owner summary emits top-level + `rollout_qualification_gate`, `public_read_release_gate`, and + `remote_owner_release_gate` booleans plus service, direct, and + failure/reclaim component booleans. The service gate is split into + `service_lifecycle_release_gate`, `service_multipage_release_gate`, and + `service_active_read_release_gate`; the direct/failure side is split into + `direct_publish_read_release_gate`, `direct_reclaim_release_gate`, and + `direct_exhaustion_fencing_release_gate`. The same row carries + required/observed service cleanup-takeover counts for killed degree, + PageRank, eigenvector, and paired-HITS cleanup owners, required/observed + service multi-page worker, coordinator, and takeover phase proof counts, and + required/observed direct reclaimed-attempt-completion plus + stale-attempt-rejection counts for abandoned direct page attempts; the + rollout summary should emit configured and observed deployment-shaped + local evidence booleans. Those JSON rows are useful evidence for rollout + tooling, but they are not themselves product build targets. CI should keep + graph metrics shaped like other features: focused tests for lifecycle, + cleanup, fan-in, operations, public API behavior, and process ownership. Full + promotion still requires hosted deployment-scale owner, fan-in, cleanup, + storage-growth, and latency evidence before any distributed-by-default + rollout. + Smoke keeps `maintenance_mode: "combined"` for backward-compatible PR + validation, while promotion starts with `maintenance_mode: "split"` so release + runs exercise the coordinator/worker-pool boundary by default. Promotion is a + floor, not just a label: a `promotion` run may raise workload knobs, but it + cannot lower the split/reopen, document, fanout, top-k, shard, active-shard, + mutation, worker, iteration, successful-generation-repeat, failure-repeat, + diagnostic, or status-page evidence below the profile baseline while still + emitting `profile: "promotion"`. Promotion + also requires `top_k` not to divide evenly by the synthetic shard count, so + the emitted shard min/max counters prove a nonuniform merge layout instead of + a perfectly even fixture; `top_k` must also exceed the synthetic shard count, + so every shard carries score evidence. Profiles set + workload defaults; latency, storage, page-claim, cleanup-tick, + executed-round, failure-retry, worker-step, and coordinator-step budgets + remain disabled unless a release invocation provides explicit + `--max-*-latency-ns`, `--max-storage-*-records`, + `--max-page-claims`, `--max-cleanup-ticks`, `--max-rounds-executed`, + `--max-failure-retry-count`, `--max-worker-steps`, or + `--max-coordinator-steps` thresholds or uses the named budgeted promotion + target. Synthetic + direct-metric fan-in also accepts `--synthetic-fan-in-shards` and + `--synthetic-fan-in-active-shards`; the smoke profile keeps the two-shard, + two-active-shard invariant, the CLI rejects active-shard counts below two or + above the total shard count, and the promotion profile starts with + `top_k: 33`, eight synthetic shards, and four active/stale synthetic shards + so release runs exercise a broader, nonuniform merge layout before hosted + cross-range qualification. The + CLI also rejects `--fanout` values larger than `--docs`, keeping generated + cyclic source targets unique enough for graph-shape evidence to be + meaningful. The promotion profile also starts with two successful generation + repeats before the failed-build churn phase and three active mutation writes + before the active rebuild, while smoke keeps a single mutation write for fast + local validation. + Independent of those optional thresholds, the harness treats scheduler summary + shape and graph topology as correctness gates: before drain the graph index's + persisted stats must match the generated node and edge counts, the edge + generation must match the target generation, and JSONL must expose source, + sink, authority, sink-edge, cyclic-edge, bipartite-edge, authority-self-edge, + and max-out-degree components for the selected family. Those emitted topology + components must match the configured family, document count, fanout, and + expected score-record count exactly. The active generation fields are checked + as part of the same release record: active mutation writes must equal the + target-generation delta, and the active target generation must equal the + first published target plus that delta. Before drain it must also see exactly + one queued build for the family. Scheduler budget shape is checked before the + optional release-threshold gates: observed ticks must not exceed + `--max-ticks`, executed rounds must fit within `ticks * + --max-rounds-per-tick`, page claims must fit within executed rounds times + `--max-pages-per-round`, and pre-drain metric scans must fit inside + `--max-metrics-per-round`. Maintenance-mode evidence is + a correctness gate too: combined mode must use exactly one combined sweep per + tick and no split-role sweeps, while split mode must use the + coordinator-before/worker-pool/coordinator-after shape for each executed + scheduler round. Split multi-worker runs must also prove that more than one + configured worker identity made both tick progress and actual page + claim/completion progress whenever the run performs multiple worker steps, and + must emit a nonzero min/max page-progress range for active worker identities, + so a release run cannot accidentally serialize all page ownership through one + worker while claiming a worker-pool shape. Reopen evidence is + checked alongside those role-shape + counters: reopen-enabled runs must cross a DB-handle boundary after every + nonterminal tick and disabled runs must not report synthetic reopen evidence. + The fresh drain must start exactly one build, publish exactly once, record no + build failure, claim and complete pages with completed pages not exceeding + claims, and prove worker steps account for claimed/completed pages. Completed + pages must cover phase advancement, and coordinator steps must cover build + start, phase advancement, publish, and failure decisions. The drain must also + advance at least one phase, execute at least one round, and record both worker + and coordinator progress. Pre-publish direct reads are also correctness gates: + the primary metric must reject both `published` and `fresh` direct reads and + graph search rerank reads with `MetricNotReady`, HITS must additionally reject + both hub direct and rerank reads with `MetricNotReady`, traversal published + projection must expose exactly one null/not-ready metric status for non-HITS + and both authority/hub null/not-ready statuses for HITS, traversal fresh + projection plus published ordering and filtering must each fail with + `MetricNotReady`, and non-HITS families must emit no paired pre-publish + counters. Pre-publish status is a correctness gate too: queued metrics must be + `not_ready`, expose no active build/page payload, and point their queued + generation at the current edge generation. + The synthetic fan-in evidence is also checked as a correctness gate: the + observed shard count and min/max score counts must match the configured + non-empty balanced layout, at least two synthetic shards must be marked + active/building for the active-published and fresh-fail fan-in probes, + compatible terminal and active-published merges must return the expected + top-k score count, mixed-active published merges must return the expected + top-k score count with failed status severity, fresh, + zero-generation, incompatible-generation, incompatible-metadata, + incompatible-edge-filter, missing-metric, duplicate-metric, extra-metric, + non-finite-score, non-finite-status, and out-of-range-progress probes must + each reject once, index/metric/status-name identity probes must reject three + times, invalid-state probes must reject twice, and HITS must additionally prove + paired authority/hub shard layout, active-shard, mixed-active, result, and rejection counts while + non-HITS families emit no paired fan-in counters. + Active/terminal work summaries are checked the same way: fresh and failed + terminal states must expose no pending work or active/failed page status, + active rebuilds must prove the page claim/reclaim probe, expose at least one + bounded active page without exceeding `--max-status-pages`, expose finite + progress, and report one active build with active pages but no failed pages. + Every emitted active status page must be leased and operationally detailed: + its phase/iteration must match the active job, it must include a worker + identity, attempt number, lease expiry, bounded progress units, and no page + error. Focused status coverage also proves failed active pages retain their + worker identity, attempt number, cursor, completed/total progress units, and + last error. The result-level leased-page and detailed-page counters must both + match the bounded active status page count. + Active and failed direct reads, graph + traversal projection/order/filter reads, and graph search rerank reads must + preserve the prior published generation for `published` and fail closed for + `fresh`; direct top-k reads must also return exactly the requested + query/index/metric identity, expected status state where stable, non-empty + configured-`top_k` score payloads, non-empty node ids, and finite scores. Rerank + published checks must attach per-hit score details with the requested + index/metric identity, prior published generation, finite score components, + and consistent missing-score markers. Traversal published checks must also + return metric payloads and status entries for exactly the requested metric + names. Each active and failed traversal gate must record three published + checks and three fresh rejections. + Query profile is checked as an explainable + read-surface gate too: non-HITS families must emit three graph metric profile + entries with exactly one direct metric source, one traversal source, and one + rerank source, while HITS must emit four entries because traversal profile + includes separate authority and hub statuses. Paired HITS active and failed + reads must return both authority/hub published results, exactly + configured-`top_k` score payloads for each side, measured paired direct + published and fresh-failure latencies, one paired direct fresh rejection, + separate authority/hub rerank fresh rejections, and paired + traversal projection metric results, while non-HITS families must keep those + paired counters at zero. Storage footprint evidence is a correctness gate: + successful and failed terminal score-record counts must match the expected + graph shape exactly, terminal job namespaces must be gone, successful publish + must retain one control record, and repeated failed rebuilds must retain only + the exact bounded failure/event diagnostics allowed for the family. +- Local promotion-profile evidence should cover every graph metric family under + the CI-sized profile floor: `docs: 128`, `fanout: 4`, + `top_k: 33`, eight synthetic fan-in shards, three active mutation writes, + four split worker identities, reopen-between-ticks, eight maximum iterations, + two successful generation repeats, five failed rebuild repeats, sixteen + retained diagnostic/status slots, and a four-family budgeted promotion + summary floor. + Degree, PageRank, eigenvector, and paired HITS each prove local-oracle parity, + split coordinator/worker-pool sweep shape, all configured worker identities + making tick and page progress, active and failed `published`/`fresh` read + behavior, repeated successful-generation storage growth, cleanup/storage + footprint bounds, nonuniform synthetic fan-in layout, active-shard and + mixed-active `published` merge plus `fresh` rejection, zero/stale/ + incompatible/malformed fan-in rejection, identity-mismatch rejection, and + invalid published-state rejection. The PageRank promotion run also proves + fixed-iteration non-converged metadata at the iteration cap, eigenvector + proves the same single-vector substrate can converge under the promotion + shape, and HITS proves paired authority/hub promotion-shape reads, + diagnostics, fan-in, compatibility, and invalid-state rejection. + The all-family promotion evidence profile should include published-read, + fresh-failure, and fan-in latency capped at one second, retained storage + ceilings of 2500 score, 2600 metric, 32 control, one attempt record, 16 + failure, and 20 event records, plus scheduler ceilings of 4000 page claims, + 600 cleanup ticks, 1000 executed rounds, five failed-build retries, 7000 + worker steps, 2000 + coordinator steps, a four-worker minimum for both split tick progress and + split page claim/completion progress, and four active/stale synthetic fan-in + shards across primary and paired-HITS probes. Those non-local/planned/cleanup + latency thresholds are deliberately conservative local promotion evidence. + Larger HITS graphs, including the previous 1000-source stress shape, remain + separate performance and deployment-scale qualification rather than the + default CI promotion gate. +- Promotion-scale rollout still needs deployment-sized graph indexes + with one coordinator, multiple worker pools, and forced killed-owner churn at + scan, contribution, reduce, publish, and cleanup boundaries. Reopen between + ticks proves durable restart after each scheduler tick, but it is not a + substitute for killing active owners while they hold leases or page attempts. + The direct process gate now explicitly counts abandoned page attempts whose + stale completion is rejected after a replacement worker completes a newer + attempt. The service process gate now also counts 27 worker phase proofs, 31 + coordinator phase proofs, and eight takeover phase proofs across the existing + two-worker 130-source degree/PageRank/eigenvector/HITS service runs; the + remaining gap is exercising the same shape through deployment-sized service + owner churn. + The synthetic fan-in probe should prove the rollout runner sees normal merge + compatibility behavior, and unit-test hosted fan-in coverage now covers a + nonuniform eight-shard degree/PageRank/eigenvector direct merge with four + active/stale shards whose unpublished high-score targets stay invisible, + active-stale degree/PageRank/eigenvector traversal + projection/order/filter/search rerank published merge plus fresh rejection, + compatible HITS authority/hub direct, traversal projection/order/filter, and + authority-rerank merges over an eight-shard layout with four active/stale + shards and prior-pair preservation, remote HITS + generation/metadata/edge-filter mismatch rejection, and missing remote HITS + status rejection through the real + cross-range table reader. Those checks are still not a substitute for hosted + cross-range promotion layouts with larger shard counts, broader mixed + active/stale shard states across traversal/rerank surfaces and paired HITS, + and broader injected incompatible statuses. + Increasing synthetic fan-in shards gives the local runner a cheap preflight + for larger shard counts, and increasing active synthetic shard counts + preflights broader active/stale shard + mixes, but successful local synthetic fan-in still does not replace + deployment-shaped hosted fan-in evidence. +- Keep pass/fail gates separate from raw benchmark numbers. Correctness gates + should require local-oracle parity, no leaked attempt output, bounded retained + namespaces, no unbounded status payloads, and successful cleanup after owner + churn. The promotion evidence profile should enforce retained-storage, + page-claim, cleanup-tick, executed-round, failure-retry, worker-step, + coordinator-step, published-read latency, fresh-failure latency, and fan-in + latency ceilings as a conservative release guard. Local/planned/cleanup + wall-clock latency should still be + recorded as release notes until real production baselines exist, then promoted + to explicit rollout thresholds per metric family and profile. +- Promotion-scale fan-in should use shard layouts beyond the focused hosted + unit coverage: larger mixed hot/cold shards, active/stale shard mixes for + rerank surfaces and paired HITS, and more varied incompatible HITS + authority/hub pair rejection. Unit-test hosted coverage now proves nonuniform + eight-shard degree/PageRank/eigenvector direct fan-in with four active/stale + shards, active-stale degree/PageRank/eigenvector traversal/rerank fan-in, + compatible HITS direct, traversal projection/order/filter, and authority + rerank fan-in over an eight-shard layout with four active/stale shards and + prior-pair preservation, remote HITS + generation/metadata/edge-filter mismatch rejection, and missing remote HITS + status rejection, while + the rollout runner should have fanout graph-shape, merge-layout, and + active-staleness knobs so promotion runs can + move beyond pure star topologies and two-shard synthetic fan-in, exercising + denser PageRank/eigenvector cycles plus multi-authority HITS graphs while + preserving the same local-oracle parity check. Release JSONL + now records the generated topology components and verifies them against graph + index stats before metric execution, so promotion notes can distinguish larger + graph-shape evidence from merely larger metric budgets. +- The rollout runner should not introduce a public job API. It should drive only the + same public index/metric config surface, public action/status routes, and + internal service owner boundary used by the focused gates. + +### Long-Term Non-Goals + +These should stay out of the graph metric roadmap unless a later design changes +the core assumptions: + +- Synchronous PageRank computation on the write path. +- Query-time full-graph centrality scans. +- Exposing partial distributed job output as queryable scores. +- Metric-specific query APIs for every new centrality algorithm. +- Retaining many historical generations by default. +- Hidden PageRank boosts that affect retrieval without explainable config. + +### Roadmap Exit Criteria + +The graph metric framework should be considered production-complete when: + +- PageRank supports local and distributed materialization through the same + graph-index-owned metric API. +- Degree, eigenvector, and HITS reuse the same distributed executor model + without adding a second storage/query/status framework. +- Direct metric top-k, graph projection, graph ordering, graph filtering, graph + search rerank, query profile, any future standalone explain surface, and + status APIs are covered by public e2e tests. +- Remote coordinator and worker owners communicate only through durable + graph-index job/page state; duplicate coordinators cannot double-publish, and + worker loss abandons only reclaimable page leases. +- Failed, abandoned, or exhausted builds preserve published scores across + restart, including compatible HITS authority/hub pairs. +- Dirty markers survive restart and eventually rebuild through bounded + scheduler/runtime ticks. +- Cleanup resumes after restart, keeps diagnostics bounded, and prevents + completed, failed, abandoned, and unpublished job state from growing without + bound. +- Cross-shard direct metric top-k has deterministic merge behavior, and + retrieval/rerank metric merges either prove globally comparable generations or + fail closed. +- OpenAPI, generated clients, and public docs describe freshness semantics, + convergence metadata, phase progress, runtime ownership summaries, cleanup + behavior, and failure status. + +## Resolved Design Defaults + +- Graph metrics are owned by graph indexes. +- PageRank is the first v1 metric kind; later metric kinds remain explicit, + opt-in graph metric configs. +- Edge scope defaults to all edges in the graph index. +- The resolved metric config always records `edge_filter`. +- V1 supports `mode: "all"` and typed edge include lists. +- Metric projections return `null` before first publish. +- Ordering, filtering, and top-k metric reads fail with `MetricNotReady` before + first publish. +- `metric_freshness: "published"` reads the latest complete generation, stale or + fresh. +- `metric_freshness: "fresh"` requires the published generation to match the + current edge generation. +- `fresh` with no published generation fails with `MetricNotReady`. +- `fresh` with an older published generation fails with `MetricStale`. +- Fixed-iteration non-converged PageRank publishes by default with + `converged: false`. +- Invalid score output fails the job and preserves the prior generation. +- V1 keeps only the latest published generation, subject to snapshot safety. +- Old generation cleanup happens immediately when graph metric reads are + snapshot-safe. +- If reads are not snapshot-safe, old generation cleanup uses a deferred cleanup + queue keyed by eligible cleanup time. +- The v1 deferred cleanup fallback uses an internal 60 second delay. +- Direct graph metric endpoints always return metric status. +- Graph traversal and graph search execution carries metric status for every + projected, ordered, and filtered metric dependency so fan-in can prove + generation/freshness compatibility. Clients should set `include_metric_status` + when they need that status as an explicit response field. +- `metric_status` is a map keyed by metric name. +- Retention controls are future debug/admin options, not v1 user-facing config. diff --git a/go/pkg/sdk/oapi/client.gen.go b/go/pkg/sdk/oapi/client.gen.go index a4b32f47a1..5f7d7c4adb 100644 --- a/go/pkg/sdk/oapi/client.gen.go +++ b/go/pkg/sdk/oapi/client.gen.go @@ -2757,6 +2757,354 @@ func (e GraphMatchOperationLimitExceededErrorStatus) Valid() bool { } } +// Defines values for GraphMetricBuildPageStatusRangeKind. +const ( + GraphMetricBuildPageStatusRangeKindContributions GraphMetricBuildPageStatusRangeKind = "contributions" + GraphMetricBuildPageStatusRangeKindFull GraphMetricBuildPageStatusRangeKind = "full" + GraphMetricBuildPageStatusRangeKindJobControl GraphMetricBuildPageStatusRangeKind = "job_control" + GraphMetricBuildPageStatusRangeKindNodes GraphMetricBuildPageStatusRangeKind = "nodes" + GraphMetricBuildPageStatusRangeKindReverseEdges GraphMetricBuildPageStatusRangeKind = "reverse_edges" + GraphMetricBuildPageStatusRangeKindScores GraphMetricBuildPageStatusRangeKind = "scores" + GraphMetricBuildPageStatusRangeKindSummary GraphMetricBuildPageStatusRangeKind = "summary" +) + +// Valid indicates whether the value is a known member of the GraphMetricBuildPageStatusRangeKind enum. +func (e GraphMetricBuildPageStatusRangeKind) Valid() bool { + switch e { + case GraphMetricBuildPageStatusRangeKindContributions: + return true + case GraphMetricBuildPageStatusRangeKindFull: + return true + case GraphMetricBuildPageStatusRangeKindJobControl: + return true + case GraphMetricBuildPageStatusRangeKindNodes: + return true + case GraphMetricBuildPageStatusRangeKindReverseEdges: + return true + case GraphMetricBuildPageStatusRangeKindScores: + return true + case GraphMetricBuildPageStatusRangeKindSummary: + return true + default: + return false + } +} + +// Defines values for GraphMetricBuildPageStatusState. +const ( + GraphMetricBuildPageStatusStateComplete GraphMetricBuildPageStatusState = "complete" + GraphMetricBuildPageStatusStateFailed GraphMetricBuildPageStatusState = "failed" + GraphMetricBuildPageStatusStateLeased GraphMetricBuildPageStatusState = "leased" + GraphMetricBuildPageStatusStatePending GraphMetricBuildPageStatusState = "pending" +) + +// Valid indicates whether the value is a known member of the GraphMetricBuildPageStatusState enum. +func (e GraphMetricBuildPageStatusState) Valid() bool { + switch e { + case GraphMetricBuildPageStatusStateComplete: + return true + case GraphMetricBuildPageStatusStateFailed: + return true + case GraphMetricBuildPageStatusStateLeased: + return true + case GraphMetricBuildPageStatusStatePending: + return true + default: + return false + } +} + +// Defines values for GraphMetricConfigKind. +const ( + GraphMetricConfigKindDegree GraphMetricConfigKind = "degree" + GraphMetricConfigKindEigenvector GraphMetricConfigKind = "eigenvector" + GraphMetricConfigKindHitsAuthority GraphMetricConfigKind = "hits_authority" + GraphMetricConfigKindHitsHub GraphMetricConfigKind = "hits_hub" + GraphMetricConfigKindPagerank GraphMetricConfigKind = "pagerank" +) + +// Valid indicates whether the value is a known member of the GraphMetricConfigKind enum. +func (e GraphMetricConfigKind) Valid() bool { + switch e { + case GraphMetricConfigKindDegree: + return true + case GraphMetricConfigKindEigenvector: + return true + case GraphMetricConfigKindHitsAuthority: + return true + case GraphMetricConfigKindHitsHub: + return true + case GraphMetricConfigKindPagerank: + return true + default: + return false + } +} + +// Defines values for GraphMetricConfigRefresh. +const ( + GraphMetricConfigRefreshBackground GraphMetricConfigRefresh = "background" + GraphMetricConfigRefreshManual GraphMetricConfigRefresh = "manual" +) + +// Valid indicates whether the value is a known member of the GraphMetricConfigRefresh enum. +func (e GraphMetricConfigRefresh) Valid() bool { + switch e { + case GraphMetricConfigRefreshBackground: + return true + case GraphMetricConfigRefreshManual: + return true + default: + return false + } +} + +// Defines values for GraphMetricEdgeFilterMode. +const ( + GraphMetricEdgeFilterModeAll GraphMetricEdgeFilterMode = "all" +) + +// Valid indicates whether the value is a known member of the GraphMetricEdgeFilterMode enum. +func (e GraphMetricEdgeFilterMode) Valid() bool { + switch e { + case GraphMetricEdgeFilterModeAll: + return true + default: + return false + } +} + +// Defines values for GraphMetricEdgeFilterStatusMode. +const ( + GraphMetricEdgeFilterStatusModeAll GraphMetricEdgeFilterStatusMode = "all" + GraphMetricEdgeFilterStatusModeTypes GraphMetricEdgeFilterStatusMode = "types" +) + +// Valid indicates whether the value is a known member of the GraphMetricEdgeFilterStatusMode enum. +func (e GraphMetricEdgeFilterStatusMode) Valid() bool { + switch e { + case GraphMetricEdgeFilterStatusModeAll: + return true + case GraphMetricEdgeFilterStatusModeTypes: + return true + default: + return false + } +} + +// Defines values for GraphMetricEventKind. +const ( + GraphMetricEventKindDelete GraphMetricEventKind = "delete" + GraphMetricEventKindFailed GraphMetricEventKind = "failed" + GraphMetricEventKindPause GraphMetricEventKind = "pause" + GraphMetricEventKindPublish GraphMetricEventKind = "publish" + GraphMetricEventKindResume GraphMetricEventKind = "resume" +) + +// Valid indicates whether the value is a known member of the GraphMetricEventKind enum. +func (e GraphMetricEventKind) Valid() bool { + switch e { + case GraphMetricEventKindDelete: + return true + case GraphMetricEventKindFailed: + return true + case GraphMetricEventKindPause: + return true + case GraphMetricEventKindPublish: + return true + case GraphMetricEventKindResume: + return true + default: + return false + } +} + +// Defines values for GraphMetricFilterOp. +const ( + GraphMetricFilterOpEq GraphMetricFilterOp = "eq" + GraphMetricFilterOpGt GraphMetricFilterOp = "gt" + GraphMetricFilterOpGte GraphMetricFilterOp = "gte" + GraphMetricFilterOpLt GraphMetricFilterOp = "lt" + GraphMetricFilterOpLte GraphMetricFilterOp = "lte" + GraphMetricFilterOpNeq GraphMetricFilterOp = "neq" +) + +// Valid indicates whether the value is a known member of the GraphMetricFilterOp enum. +func (e GraphMetricFilterOp) Valid() bool { + switch e { + case GraphMetricFilterOpEq: + return true + case GraphMetricFilterOpGt: + return true + case GraphMetricFilterOpGte: + return true + case GraphMetricFilterOpLt: + return true + case GraphMetricFilterOpLte: + return true + case GraphMetricFilterOpNeq: + return true + default: + return false + } +} + +// Defines values for GraphMetricOrderDirection. +const ( + GraphMetricOrderDirectionAsc GraphMetricOrderDirection = "asc" + GraphMetricOrderDirectionDesc GraphMetricOrderDirection = "desc" +) + +// Valid indicates whether the value is a known member of the GraphMetricOrderDirection enum. +func (e GraphMetricOrderDirection) Valid() bool { + switch e { + case GraphMetricOrderDirectionAsc: + return true + case GraphMetricOrderDirectionDesc: + return true + default: + return false + } +} + +// Defines values for GraphMetricOrderNulls. +const ( + GraphMetricOrderNullsFirst GraphMetricOrderNulls = "first" + GraphMetricOrderNullsLast GraphMetricOrderNulls = "last" + GraphMetricOrderNullsNullsFirst GraphMetricOrderNulls = "nulls_first" + GraphMetricOrderNullsNullsLast GraphMetricOrderNulls = "nulls_last" +) + +// Valid indicates whether the value is a known member of the GraphMetricOrderNulls enum. +func (e GraphMetricOrderNulls) Valid() bool { + switch e { + case GraphMetricOrderNullsFirst: + return true + case GraphMetricOrderNullsLast: + return true + case GraphMetricOrderNullsNullsFirst: + return true + case GraphMetricOrderNullsNullsLast: + return true + default: + return false + } +} + +// Defines values for GraphMetricQueryMetricFreshness. +const ( + GraphMetricQueryMetricFreshnessFresh GraphMetricQueryMetricFreshness = "fresh" + GraphMetricQueryMetricFreshnessPublished GraphMetricQueryMetricFreshness = "published" +) + +// Valid indicates whether the value is a known member of the GraphMetricQueryMetricFreshness enum. +func (e GraphMetricQueryMetricFreshness) Valid() bool { + switch e { + case GraphMetricQueryMetricFreshnessFresh: + return true + case GraphMetricQueryMetricFreshnessPublished: + return true + default: + return false + } +} + +// Defines values for GraphMetricRerankMetricFreshness. +const ( + GraphMetricRerankMetricFreshnessFresh GraphMetricRerankMetricFreshness = "fresh" + GraphMetricRerankMetricFreshnessPublished GraphMetricRerankMetricFreshness = "published" +) + +// Valid indicates whether the value is a known member of the GraphMetricRerankMetricFreshness enum. +func (e GraphMetricRerankMetricFreshness) Valid() bool { + switch e { + case GraphMetricRerankMetricFreshnessFresh: + return true + case GraphMetricRerankMetricFreshnessPublished: + return true + default: + return false + } +} + +// Defines values for GraphMetricRuntimeStatsRole. +const ( + GraphMetricRuntimeStatsRoleCombined GraphMetricRuntimeStatsRole = "combined" + GraphMetricRuntimeStatsRoleCoordinator GraphMetricRuntimeStatsRole = "coordinator" + GraphMetricRuntimeStatsRoleWorker GraphMetricRuntimeStatsRole = "worker" + GraphMetricRuntimeStatsRoleWorkerPool GraphMetricRuntimeStatsRole = "worker_pool" +) + +// Valid indicates whether the value is a known member of the GraphMetricRuntimeStatsRole enum. +func (e GraphMetricRuntimeStatsRole) Valid() bool { + switch e { + case GraphMetricRuntimeStatsRoleCombined: + return true + case GraphMetricRuntimeStatsRoleCoordinator: + return true + case GraphMetricRuntimeStatsRoleWorker: + return true + case GraphMetricRuntimeStatsRoleWorkerPool: + return true + default: + return false + } +} + +// Defines values for GraphMetricStatusPhase. +const ( + GraphMetricStatusPhaseCheckConvergence GraphMetricStatusPhase = "check_convergence" + GraphMetricStatusPhaseCleanupOldGenerations GraphMetricStatusPhase = "cleanup_old_generations" + GraphMetricStatusPhaseComplete GraphMetricStatusPhase = "complete" + GraphMetricStatusPhaseComputing GraphMetricStatusPhase = "computing" + GraphMetricStatusPhaseHitsHubContributions GraphMetricStatusPhase = "hits_hub_contributions" + GraphMetricStatusPhaseHitsHubReduceRanks GraphMetricStatusPhase = "hits_hub_reduce_ranks" + GraphMetricStatusPhaseIdle GraphMetricStatusPhase = "idle" + GraphMetricStatusPhaseInitializeRanks GraphMetricStatusPhase = "initialize_ranks" + GraphMetricStatusPhaseIterateContributions GraphMetricStatusPhase = "iterate_contributions" + GraphMetricStatusPhasePrepareGeneration GraphMetricStatusPhase = "prepare_generation" + GraphMetricStatusPhasePublishGeneration GraphMetricStatusPhase = "publish_generation" + GraphMetricStatusPhasePublishing GraphMetricStatusPhase = "publishing" + GraphMetricStatusPhaseReduceRanks GraphMetricStatusPhase = "reduce_ranks" + GraphMetricStatusPhaseScanEdgesAndOutDegree GraphMetricStatusPhase = "scan_edges_and_out_degree" +) + +// Valid indicates whether the value is a known member of the GraphMetricStatusPhase enum. +func (e GraphMetricStatusPhase) Valid() bool { + switch e { + case GraphMetricStatusPhaseCheckConvergence: + return true + case GraphMetricStatusPhaseCleanupOldGenerations: + return true + case GraphMetricStatusPhaseComplete: + return true + case GraphMetricStatusPhaseComputing: + return true + case GraphMetricStatusPhaseHitsHubContributions: + return true + case GraphMetricStatusPhaseHitsHubReduceRanks: + return true + case GraphMetricStatusPhaseIdle: + return true + case GraphMetricStatusPhaseInitializeRanks: + return true + case GraphMetricStatusPhaseIterateContributions: + return true + case GraphMetricStatusPhasePrepareGeneration: + return true + case GraphMetricStatusPhasePublishGeneration: + return true + case GraphMetricStatusPhasePublishing: + return true + case GraphMetricStatusPhaseReduceRanks: + return true + case GraphMetricStatusPhaseScanEdgesAndOutDegree: + return true + default: + return false + } +} + // Defines values for GraphNodesResultKind. const ( GraphNodesResultKindNodes GraphNodesResultKind = "nodes" @@ -3105,6 +3453,24 @@ func (e GraphRowCountTarget) Valid() bool { } } +// Defines values for GraphTraversalMetricFreshness. +const ( + GraphTraversalMetricFreshnessFresh GraphTraversalMetricFreshness = "fresh" + GraphTraversalMetricFreshnessPublished GraphTraversalMetricFreshness = "published" +) + +// Valid indicates whether the value is a known member of the GraphTraversalMetricFreshness enum. +func (e GraphTraversalMetricFreshness) Valid() bool { + switch e { + case GraphTraversalMetricFreshnessFresh: + return true + case GraphTraversalMetricFreshnessPublished: + return true + default: + return false + } +} + // Defines values for GraphWorkBudgetExceededErrorDimension. const ( GraphWorkBudgetExceededErrorDimensionExploredEdgeBytes GraphWorkBudgetExceededErrorDimension = "explored_edge_bytes" @@ -4791,6 +5157,24 @@ func (e JoinType) Valid() bool { } } +// Defines values for LegacyGraphQueryMetricFreshness. +const ( + LegacyGraphQueryMetricFreshnessFresh LegacyGraphQueryMetricFreshness = "fresh" + LegacyGraphQueryMetricFreshnessPublished LegacyGraphQueryMetricFreshness = "published" +) + +// Valid indicates whether the value is a known member of the LegacyGraphQueryMetricFreshness enum. +func (e LegacyGraphQueryMetricFreshness) Valid() bool { + switch e { + case LegacyGraphQueryMetricFreshnessFresh: + return true + case LegacyGraphQueryMetricFreshnessPublished: + return true + default: + return false + } +} + // Defines values for LegacyGraphSearchResultKind. const ( LegacyGraphSearchResultKindLegacy LegacyGraphSearchResultKind = "legacy" @@ -8031,6 +8415,23 @@ type BackupRequest struct { // Example: portable type BackupRequestFormat string +// BatchCommittedFailure Additive details for a committed batch that needs operator action. The +// open string code is forward-compatible with older SDKs; clients should +// treat unknown codes as non-retryable when `retryable` is false. +type BatchCommittedFailure struct { + // Code Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + Code string `json:"code"` + + // Message Actionable operator guidance. + Message string `json:"message"` + + // Reason Optional stable reason within the failure category, such as `build_budget_exceeded`. + Reason string `json:"reason,omitempty,omitzero"` + + // Retryable Whether replaying the document mutation is safe. Committed repair outcomes are false. + Retryable bool `json:"retryable"` +} + // BatchRequest Batch insert, delete, and transform operations in a single request. // // **Atomicity**: @@ -8118,13 +8519,19 @@ type BatchResponse struct { // Deleted Number of documents successfully deleted Deleted int `json:"deleted,omitempty,omitzero"` + // Failure Additive details for a committed batch that needs operator action. The + // open string code is forward-compatible with older SDKs; clients should + // treat unknown codes as non-retryable when `retryable` is false. + Failure BatchCommittedFailure `json:"failure,omitempty,omitzero"` + // Inserted Number of documents successfully inserted Inserted int `json:"inserted,omitempty,omitzero"` // Status Durable commit outcome. `committed_pending` means requested visibility or // participant propagation is still completing. `committed_repair_required` - // means the primary write committed, but a terminal enrichment failure needs - // operator repair and will not be retried indefinitely. + // means the primary write committed, but a terminal background materialization + // failure needs operator repair and will not be retried indefinitely. Inspect + // `failure` when present; retrying the document write is unnecessary. Status BatchResponseStatus `json:"status,omitempty,omitzero"` // Transformed Number of documents successfully transformed @@ -8133,8 +8540,9 @@ type BatchResponse struct { // BatchResponseStatus Durable commit outcome. `committed_pending` means requested visibility or // participant propagation is still completing. `committed_repair_required` -// means the primary write committed, but a terminal enrichment failure needs -// operator repair and will not be retried indefinitely. +// means the primary write committed, but a terminal background materialization +// failure needs operator repair and will not be retried indefinitely. Inspect +// `failure` when present; retrying the document write is unnecessary. type BatchResponseStatus string // BedrockEmbedderConfig Configuration for the AWS Bedrock embedding provider. @@ -9210,8 +9618,11 @@ type CreateGraphIndexRequest struct { Enrichments []EnrichmentConfig `json:"enrichments,omitempty,omitzero"` // MaxEdgesPerDocument Maximum number of distinct visible edges materialized per document after source precedence and identity deduplication. Zero uses the server safety limit (currently 1,000,000). Independent aggregate reconciliation budgets bound work across overlapping source manifests. - MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` - Resolvers []GraphResolverConfig `json:"resolvers,omitempty,omitzero"` + MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` + + // Metrics Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + Metrics map[string]GraphMetricConfig `json:"metrics,omitempty,omitzero"` + Resolvers []GraphResolverConfig `json:"resolvers,omitempty,omitzero"` // Source Artifact stream materialized into graph edges. Each source artifact is limited to 16 MiB and 1,000,000 relation items so live apply, repair, split, and restore share one bounded admission contract. Artifact-backed graph sources require index_capabilities.artifact_sources=true and are rejected by serverless deployments. Source GraphArtifactSourceConfig `json:"source,omitempty,omitzero"` @@ -9576,7 +9987,8 @@ type CreatedGraphIndex struct { Enrichments []CreatedEnrichmentConfig `json:"enrichments,omitempty,omitzero"` // MaxEdgesPerDocument Maximum number of distinct visible edges materialized per document after source precedence and identity deduplication. Zero uses the server safety limit (currently 1,000,000). Independent aggregate reconciliation budgets bound work across overlapping source manifests. - MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` + MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` + Metrics map[string]GraphMetricConfig `json:"metrics,omitempty,omitzero"` // Name Name of the created index Name string `json:"name"` @@ -9606,6 +10018,7 @@ type CreatedGraphIndexConfig struct { // MaxEdgesPerDocument Maximum number of distinct visible edges materialized per document after source precedence and identity deduplication. Zero uses the server safety limit (currently 1,000,000). Independent aggregate reconciliation budgets bound work across overlapping source manifests. MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` + Metrics map[string]GraphMetricConfig `json:"metrics,omitempty,omitzero"` Resolvers []GraphResolverConfig `json:"resolvers,omitempty,omitzero"` Sources []CreatedGraphArtifactSourceConfig `json:"sources,omitempty,omitzero"` @@ -12317,6 +12730,12 @@ type GlobalStatefulQueryRequest struct { // FullTextSearch An Antfly query expression retained as syntactically validated JSON and compiled by the query engine. FullTextSearch RawQuery `json:"full_text_search,omitempty,omitzero"` + // GraphMetric Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + GraphMetric GraphMetricQuery `json:"graph_metric,omitempty,omitzero"` + + // GraphMetricRerank Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. + GraphMetricRerank GraphMetricRerank `json:"graph_metric_rerank,omitempty,omitzero"` + // GraphQueries Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH operations. Keys use the versioned GraphIdentifier policy. GraphQueries GraphQueries `json:"graph_queries,omitempty,omitzero"` @@ -13011,8 +13430,11 @@ type GraphIndexConfig struct { EdgeTypes []EdgeTypeConfig `json:"edge_types,omitempty,omitzero"` // MaxEdgesPerDocument Maximum number of distinct visible edges materialized per document after source precedence and identity deduplication. Zero uses the server safety limit (currently 1,000,000). Independent aggregate reconciliation budgets bound work across overlapping source manifests. - MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` - Resolvers []GraphResolverConfig `json:"resolvers,omitempty,omitzero"` + MaxEdgesPerDocument int `json:"max_edges_per_document,omitempty,omitzero"` + + // Metrics Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + Metrics map[string]GraphMetricConfig `json:"metrics,omitempty,omitzero"` + Resolvers []GraphResolverConfig `json:"resolvers,omitempty,omitzero"` // Source Artifact stream materialized into graph edges. Each source artifact is limited to 16 MiB and 1,000,000 relation items so live apply, repair, split, and restore share one bounded admission contract. Artifact-backed graph sources require index_capabilities.artifact_sources=true and are rejected by serverless deployments. Source GraphArtifactSourceConfig `json:"source,omitempty,omitzero"` @@ -13083,6 +13505,9 @@ type GraphIndexStats struct { ExpectedGroups uint64 `json:"expected_groups,omitempty,omitzero"` FreshGroups uint64 `json:"fresh_groups,omitempty,omitzero"` + // GraphMetricRuntime Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers. + GraphMetricRuntime GraphMetricRuntimeStats `json:"graph_metric_runtime,omitempty,omitzero"` + // Incarnation Opaque identity of the desired index incarnation. Clients may compare it for equality but must not interpret its contents. Incarnation string `json:"incarnation,omitempty,omitzero"` @@ -13279,6 +13704,378 @@ type GraphMatchQuery struct { Return GraphReturn `json:"return"` } +// GraphMetricActionResponse defines model for GraphMetricActionResponse. +type GraphMetricActionResponse struct { + Status GraphMetricStatus `json:"status"` +} + +// GraphMetricBuildPageStatus defines model for GraphMetricBuildPageStatus. +type GraphMetricBuildPageStatus struct { + // Attempt Current attempt number for this page. + Attempt int64 `json:"attempt,omitempty,omitzero"` + + // CompletedUnits Completed work units for this page. + CompletedUnits int64 `json:"completed_units,omitempty,omitzero"` + + // Cursor Opaque resumable cursor for this page. + Cursor string `json:"cursor,omitempty,omitzero"` + Iteration int64 `json:"iteration"` + + // LastError Last page-level error. + LastError string `json:"last_error,omitempty,omitzero"` + + // LeaseExpiresAtMs Unix epoch milliseconds when the page lease expires, or 0 when not leased. + LeaseExpiresAtMs int64 `json:"lease_expires_at_ms,omitempty,omitzero"` + PageId int64 `json:"page_id"` + Phase string `json:"phase"` + RangeKind GraphMetricBuildPageStatusRangeKind `json:"range_kind"` + State GraphMetricBuildPageStatusState `json:"state"` + + // TotalUnits Estimated total work units for this page. + TotalUnits int64 `json:"total_units,omitempty,omitzero"` + + // WorkerId Worker id that owns or last failed this page. + WorkerId string `json:"worker_id,omitempty,omitzero"` +} + +// GraphMetricBuildPageStatusRangeKind defines model for GraphMetricBuildPageStatus.RangeKind. +type GraphMetricBuildPageStatusRangeKind string + +// GraphMetricBuildPageStatusState defines model for GraphMetricBuildPageStatus.State. +type GraphMetricBuildPageStatusState string + +// GraphMetricConfig Published metric configuration. If kind is omitted, the metric name must be a supported kind. +type GraphMetricConfig struct { + Damping float64 `json:"damping,omitempty,omitzero"` + + // EdgeFilter Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied. + EdgeFilter GraphMetricEdgeFilter `json:"edge_filter,omitempty,omitzero"` + Enabled bool `json:"enabled,omitempty,omitzero"` + Kind GraphMetricConfigKind `json:"kind,omitempty,omitzero"` + MaxIterations int32 `json:"max_iterations,omitempty,omitzero"` + + // Refresh Serverless accepts background only. + Refresh GraphMetricConfigRefresh `json:"refresh,omitempty,omitzero"` + Tolerance float64 `json:"tolerance,omitempty,omitzero"` +} + +// GraphMetricConfigKind defines model for GraphMetricConfig.Kind. +type GraphMetricConfigKind string + +// GraphMetricConfigRefresh Serverless accepts background only. +type GraphMetricConfigRefresh string + +// GraphMetricEdgeFilter Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied. +type GraphMetricEdgeFilter struct { + Mode GraphMetricEdgeFilterMode `json:"mode,omitempty,omitzero"` + Types []GraphEdgeType `json:"types,omitempty,omitzero"` +} + +// GraphMetricEdgeFilterMode defines model for GraphMetricEdgeFilter.Mode. +type GraphMetricEdgeFilterMode string + +// GraphMetricEdgeFilterStatus defines model for GraphMetricEdgeFilterStatus. +type GraphMetricEdgeFilterStatus struct { + Mode GraphMetricEdgeFilterStatusMode `json:"mode"` + Types []string `json:"types,omitempty,omitzero"` +} + +// GraphMetricEdgeFilterStatusMode defines model for GraphMetricEdgeFilterStatus.Mode. +type GraphMetricEdgeFilterStatusMode string + +// GraphMetricEvent defines model for GraphMetricEvent. +type GraphMetricEvent struct { + AtMs int64 `json:"at_ms"` + Kind GraphMetricEventKind `json:"kind"` + PublishedGeneration int64 `json:"published_generation"` + ScoreCount int64 `json:"score_count"` + Sequence int64 `json:"sequence"` + TargetEdgeGeneration int64 `json:"target_edge_generation"` +} + +// GraphMetricEventKind defines model for GraphMetricEvent.Kind. +type GraphMetricEventKind string + +// GraphMetricFilter defines model for GraphMetricFilter. +type GraphMetricFilter struct { + Metric string `json:"metric"` + + // Op Semantic comparison operator. Named values keep generated SDK enums portable and readable. + Op GraphMetricFilterOp `json:"op"` + Value float64 `json:"value"` +} + +// GraphMetricFilterOp Semantic comparison operator. Named values keep generated SDK enums portable and readable. +type GraphMetricFilterOp string + +// GraphMetricOrder defines model for GraphMetricOrder. +type GraphMetricOrder struct { + Direction GraphMetricOrderDirection `json:"direction,omitempty,omitzero"` + Metric string `json:"metric"` + Nulls GraphMetricOrderNulls `json:"nulls,omitempty,omitzero"` +} + +// GraphMetricOrderDirection defines model for GraphMetricOrder.Direction. +type GraphMetricOrderDirection string + +// GraphMetricOrderNulls defines model for GraphMetricOrder.Nulls. +type GraphMetricOrderNulls string + +// GraphMetricProfile defines model for GraphMetricProfile. +type GraphMetricProfile struct { + // Freshness Effective freshness mode requested for this metric use. + Freshness string `json:"freshness"` + + // IndexName Graph index that owns the metric. + IndexName string `json:"index_name"` + + // MetricName Graph metric name within the index. + MetricName string `json:"metric_name"` + + // QueryName Name of the graph query or graph metric query that used the metric. + QueryName string `json:"query_name"` + + // Source Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + Source string `json:"source"` + Status GraphMetricStatus `json:"status"` +} + +// GraphMetricQuery Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. +type GraphMetricQuery struct { + // Index Graph index that owns the published metric. + Index string `json:"index"` + + // Metric Graph metric to read. + Metric string `json:"metric"` + + // MetricFreshness Whether the latest published generation may be stale or must match the graph edge generation. + MetricFreshness GraphMetricQueryMetricFreshness `json:"metric_freshness,omitempty,omitzero"` + + // Name Optional result key. Defaults to the metric name. + Name string `json:"name,omitempty,omitzero"` + + // TopK Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated metric snapshot. + TopK int32 `json:"top_k,omitempty,omitzero"` +} + +// GraphMetricQueryMetricFreshness Whether the latest published generation may be stale or must match the graph edge generation. +type GraphMetricQueryMetricFreshness string + +// GraphMetricRerank Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. +type GraphMetricRerank struct { + // BaseWeight Multiplier applied to the existing hit score before adding the graph metric feature. + BaseWeight float64 `json:"base_weight,omitempty,omitzero"` + + // CandidateCount Bounded retrieval window scored by the graph metric before offset and limit are applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. An explicit value must cover offset plus limit. Larger windows improve promotion recall at predictable linear score-read cost. + CandidateCount int32 `json:"candidate_count,omitempty,omitzero"` + + // Index Graph index that owns the published metric. + Index string `json:"index"` + + // Metric Graph metric name to blend into the search hit score. + Metric string `json:"metric"` + + // MetricFreshness Whether stale published generations are acceptable or the metric must be fresh. + MetricFreshness GraphMetricRerankMetricFreshness `json:"metric_freshness,omitempty,omitzero"` + + // MissingScore Metric feature value to use for hits that do not have a score in the published metric generation. + MissingScore float64 `json:"missing_score,omitempty,omitzero"` + + // Weight Multiplier applied to the graph metric score before it is added to the existing hit score. + Weight float64 `json:"weight,omitempty,omitzero"` +} + +// GraphMetricRerankMetricFreshness Whether stale published generations are acceptable or the metric must be fresh. +type GraphMetricRerankMetricFreshness string + +// GraphMetricRerankScoreDetails defines model for GraphMetricRerankScoreDetails. +type GraphMetricRerankScoreDetails struct { + // BaseScore Hit score before graph metric rerank composition. + BaseScore float64 `json:"base_score"` + + // BaseWeight Weight applied to the base score. + BaseWeight float64 `json:"base_weight"` + + // FinalScore Final hit score after graph metric rerank composition. + FinalScore float64 `json:"final_score"` + + // IndexName Graph index that provided the metric score. + IndexName string `json:"index_name"` + + // MetricName Graph metric used as a score feature. + MetricName string `json:"metric_name"` + + // MetricScore Published metric score for this hit, or null when the hit was missing from the metric generation. + MetricScore float64 `json:"metric_score,omitempty,omitzero"` + + // MetricScoreUsed Metric feature value used in the formula after applying missing_score fallback if needed. + MetricScoreUsed float64 `json:"metric_score_used"` + + // MetricWeight Weight applied to the metric score feature. + MetricWeight float64 `json:"metric_weight"` + + // MissingScoreUsed True when metric_score was missing and the request's missing_score fallback was used. + MissingScoreUsed bool `json:"missing_score_used"` + + // PublishedGeneration Published graph metric score generation used for this hit. + PublishedGeneration int64 `json:"published_generation"` +} + +// GraphMetricResult defines model for GraphMetricResult. +type GraphMetricResult struct { + IndexName string `json:"index_name"` + Metric string `json:"metric"` + Scores []GraphMetricScore `json:"scores"` + Status GraphMetricStatus `json:"status"` +} + +// GraphMetricRuntimeStats Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers. +type GraphMetricRuntimeStats struct { + AcquisitionCount uint64 `json:"acquisition_count,omitempty,omitzero"` + DurableProgressTicks uint64 `json:"durable_progress_ticks,omitempty,omitzero"` + Enabled bool `json:"enabled,omitempty,omitzero"` + ErrorTicks uint64 `json:"error_ticks,omitempty,omitzero"` + HasLease bool `json:"has_lease,omitempty,omitzero"` + IdleTicks uint64 `json:"idle_ticks,omitempty,omitzero"` + LastAcquiredMs uint64 `json:"last_acquired_ms,omitempty,omitzero"` + LastActiveBuilds uint64 `json:"last_active_builds,omitempty,omitzero"` + LastBudgetExhausted bool `json:"last_budget_exhausted,omitempty,omitzero"` + LastBuildsStarted uint64 `json:"last_builds_started,omitempty,omitzero"` + LastCoordinatorSteps uint64 `json:"last_coordinator_steps,omitempty,omitzero"` + LastErrorName string `json:"last_error_name,omitempty,omitzero"` + LastFailedBuilds uint64 `json:"last_failed_builds,omitempty,omitzero"` + LastMetricsScanned uint64 `json:"last_metrics_scanned,omitempty,omitzero"` + LastPagesClaimed uint64 `json:"last_pages_claimed,omitempty,omitzero"` + LastPagesCompleted uint64 `json:"last_pages_completed,omitempty,omitzero"` + LastPhasesAdvanced uint64 `json:"last_phases_advanced,omitempty,omitzero"` + LastPublished uint64 `json:"last_published,omitempty,omitzero"` + + // LastRetiredInputRecords Consumed intermediate records retired in the latest maintenance tick. + LastRetiredInputRecords uint64 `json:"last_retired_input_records,omitempty,omitzero"` + LastWorkerSteps uint64 `json:"last_worker_steps,omitempty,omitzero"` + LeaseAcquireFailures uint64 `json:"lease_acquire_failures,omitempty,omitzero"` + + // LeaseExpiresAtMs Cached expiry of the currently held maintenance lease, or zero when no lease is held. + LeaseExpiresAtMs uint64 `json:"lease_expires_at_ms,omitempty,omitzero"` + LeaseKeyHash uint64 `json:"lease_key_hash,omitempty,omitzero"` + LeaseOwned bool `json:"lease_owned,omitempty,omitzero"` + + // LeaseRenewAfterMs Earliest time the runtime will renew its maintenance lease, or zero when no lease is held. + LeaseRenewAfterMs uint64 `json:"lease_renew_after_ms,omitempty,omitzero"` + LostLeases uint64 `json:"lost_leases,omitempty,omitzero"` + Notified bool `json:"notified,omitempty,omitzero"` + OwnerIdHash uint64 `json:"owner_id_hash,omitempty,omitzero"` + + // RenewalCount Number of durable maintenance lease renewals completed by this runtime. + RenewalCount uint64 `json:"renewal_count,omitempty,omitzero"` + Role GraphMetricRuntimeStatsRole `json:"role,omitempty,omitzero"` + RuntimeIdHash uint64 `json:"runtime_id_hash,omitempty,omitzero"` + Shutdown bool `json:"shutdown,omitempty,omitzero"` + Started bool `json:"started,omitempty,omitzero"` + TakeoverCount uint64 `json:"takeover_count,omitempty,omitzero"` + TicksCompleted uint64 `json:"ticks_completed,omitempty,omitzero"` + TicksStarted uint64 `json:"ticks_started,omitempty,omitzero"` + TotalActiveBuilds uint64 `json:"total_active_builds,omitempty,omitzero"` + TotalBuildsStarted uint64 `json:"total_builds_started,omitempty,omitzero"` + TotalCoordinatorSteps uint64 `json:"total_coordinator_steps,omitempty,omitzero"` + TotalFailedBuilds uint64 `json:"total_failed_builds,omitempty,omitzero"` + TotalMetricsScanned uint64 `json:"total_metrics_scanned,omitempty,omitzero"` + TotalPagesClaimed uint64 `json:"total_pages_claimed,omitempty,omitzero"` + TotalPagesCompleted uint64 `json:"total_pages_completed,omitempty,omitzero"` + TotalPhasesAdvanced uint64 `json:"total_phases_advanced,omitempty,omitzero"` + TotalPublished uint64 `json:"total_published,omitempty,omitzero"` + + // TotalRetiredInputRecords Consumed intermediate records retired at completed reduction barriers. + TotalRetiredInputRecords uint64 `json:"total_retired_input_records,omitempty,omitzero"` + TotalWorkerSteps uint64 `json:"total_worker_steps,omitempty,omitzero"` + WorkerCount uint64 `json:"worker_count,omitempty,omitzero"` + WorkerIdHash uint64 `json:"worker_id_hash,omitempty,omitzero"` +} + +// GraphMetricRuntimeStatsRole defines model for GraphMetricRuntimeStats.Role. +type GraphMetricRuntimeStatsRole string + +// GraphMetricScore defines model for GraphMetricScore. +type GraphMetricScore struct { + Node string `json:"node"` + Score float64 `json:"score"` +} + +// GraphMetricStatus defines model for GraphMetricStatus. +type GraphMetricStatus struct { + // BuildCompletedUnits Completed work units for the active graph metric build, or 0 when idle or unknown. + BuildCompletedUnits int64 `json:"build_completed_units,omitempty,omitzero"` + + // BuildCursor Opaque resumable cursor for the active build phase. Empty or omitted when idle or when the phase has no cursor. + BuildCursor string `json:"build_cursor,omitempty,omitzero"` + + // BuildIteration Iteration number reported by the active build lease, or 0 when idle or not iterative. + BuildIteration int64 `json:"build_iteration,omitempty,omitzero"` + + // BuildJobId Durable identifier for the active graph metric build job, or 0 when idle. + BuildJobId int64 `json:"build_job_id,omitempty,omitzero"` + + // BuildLeaseExpiresAtMs Unix epoch milliseconds when the active build lease expires, or 0 when idle. + BuildLeaseExpiresAtMs int64 `json:"build_lease_expires_at_ms,omitempty,omitzero"` + + // BuildPages Active leased or failed build pages for the current build phase, capped and ordered by durable page key. + BuildPages []GraphMetricBuildPageStatus `json:"build_pages,omitempty,omitzero"` + + // BuildPagesTruncated Whether build_pages was capped before every active page could be included. + BuildPagesTruncated bool `json:"build_pages_truncated,omitempty,omitzero"` + + // BuildQueued Whether a local or distributed build is queued after the currently published or building generation. + BuildQueued bool `json:"build_queued"` + + // BuildStartedAtMs Unix epoch milliseconds when the active graph metric build started, or 0 when idle. + BuildStartedAtMs int64 `json:"build_started_at_ms,omitempty,omitzero"` + + // BuildTotalUnits Estimated total work units for the active graph metric build, or 0 when idle or unknown. + BuildTotalUnits int64 `json:"build_total_units,omitempty,omitzero"` + + // BuildWorkerId Worker id that owns the active build lease. Local builds use `local`. + BuildWorkerId string `json:"build_worker_id,omitempty,omitzero"` + + // BuildingGeneration Edge generation currently held by an active build lease, or 0 when idle. + BuildingGeneration int64 `json:"building_generation,omitempty,omitzero"` + ComputedAtMs int64 `json:"computed_at_ms"` + + // ConfigFingerprint Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every SDK preserves all 64 bits. + ConfigFingerprint string `json:"config_fingerprint,omitempty,omitzero"` + Converged bool `json:"converged"` + Delta float64 `json:"delta"` + EdgeFilter GraphMetricEdgeFilterStatus `json:"edge_filter,omitempty,omitzero"` + EdgeGeneration int64 `json:"edge_generation"` + IterationsCompleted int64 `json:"iterations_completed"` + + // LastError Last build error for the current failed target generation. + LastError string `json:"last_error,omitempty,omitzero"` + LastEvent GraphMetricEvent `json:"last_event,omitempty,omitzero"` + MaintenancePaused bool `json:"maintenance_paused,omitempty,omitzero"` + + // MetadataVersion Version of the published graph metric metadata schema. + MetadataVersion int64 `json:"metadata_version,omitempty,omitzero"` + Phase GraphMetricStatusPhase `json:"phase"` + + // Progress Build progress for the target edge generation, from 0.0 to 1.0 + Progress float64 `json:"progress"` + PublishedGeneration int64 `json:"published_generation"` + + // QueuedGeneration Pending edge generation waiting to build, or 0 when no build is queued. + QueuedGeneration int64 `json:"queued_generation,omitempty,omitzero"` + + // RecentEvents Recent graph metric events, newest first. + RecentEvents []GraphMetricEvent `json:"recent_events,omitempty,omitzero"` + + // RetryCount Number of consecutive failed build attempts for the current target generation, or 0 when no failure applies. + RetryCount int64 `json:"retry_count,omitempty,omitzero"` + State string `json:"state"` + TargetEdgeGeneration int64 `json:"target_edge_generation"` +} + +// GraphMetricStatusPhase defines model for GraphMetricStatus.Phase. +type GraphMetricStatusPhase string + // GraphNodeSelector Select graph nodes using exactly one explicit, exact selector form. type GraphNodeSelector struct { union json.RawMessage @@ -13289,6 +14086,9 @@ type GraphNodesResult struct { // Kind Stable discriminator for the graph result shape. Kind GraphNodesResultKind `json:"kind"` + // MetricStatus Graph metric status metadata keyed by metric name when requested. + MetricStatus map[string]GraphMetricStatus `json:"metric_status,omitempty,omitzero"` + // Nodes Traversal result nodes; requested paths are stored on each node. Nodes []GraphResultNode `json:"nodes"` @@ -13564,6 +14364,9 @@ type GraphResultNode struct { // Key Document key Key string `json:"key"` + // Metrics Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score for the node. + Metrics map[string]interface{} `json:"metrics,omitempty,omitzero"` + // Path Exact ordered traversal identities from the start node, terminating at this node's fully qualified identity. Present only for traversal queries with include_paths=true. Path []GraphPathEndpoint `json:"path,omitempty,omitzero"` @@ -13689,16 +14492,34 @@ type GraphTraversal struct { // IncludeDocuments Include each result node's stored document when it exists at the pinned snapshot. A dangling graph identity omits document. When false, document is always omitted. IncludeDocuments bool `json:"include_documents,omitempty,omitzero"` - IncludePaths bool `json:"include_paths,omitempty,omitzero"` - Limit int `json:"limit,omitempty,omitzero"` + + // IncludeMetricStatus Include graph metric status metadata in the traversal profile. + IncludeMetricStatus bool `json:"include_metric_status,omitempty,omitzero"` + IncludePaths bool `json:"include_paths,omitempty,omitzero"` + Limit int `json:"limit,omitempty,omitzero"` // MaxDepth Maximum traversal depth. Defaults to one hop to keep fan-out explicit. MaxDepth int `json:"max_depth,omitempty,omitzero"` + // MetricFreshness Freshness required for projected, ordered, and filtered graph metrics. + MetricFreshness GraphTraversalMetricFreshness `json:"metric_freshness,omitempty,omitzero"` + + // Metrics Graph metric names to project onto returned traversal nodes. + Metrics []string `json:"metrics,omitempty,omitzero"` + + // OrderBy Sort traversal candidates by graph metric score before applying limit. + OrderBy []GraphMetricOrder `json:"order_by,omitempty,omitzero"` + // Start Select graph nodes using exactly one explicit, exact selector form. Start GraphNodeSelector `json:"start"` + + // WhereMetric Filter traversal candidates by graph metric score before applying limit. + WhereMetric []GraphMetricFilter `json:"where_metric,omitempty,omitzero"` } +// GraphTraversalMetricFreshness Freshness required for projected, ordered, and filtered graph metrics. +type GraphTraversalMetricFreshness string + // GraphTraverseQuery defines model for GraphTraverseQuery. type GraphTraverseQuery struct { Index string `json:"index"` @@ -16222,7 +17043,19 @@ type LegacyGraphQuery struct { Fields []string `json:"fields,omitempty,omitzero"` IncludeDocuments bool `json:"include_documents,omitempty,omitzero"` IncludeEdges bool `json:"include_edges,omitempty,omitzero"` - IndexName string `json:"index_name"` + + // IncludeMetricStatus Include graph metric status metadata in the legacy graph_searches result. + IncludeMetricStatus bool `json:"include_metric_status,omitempty,omitzero"` + IndexName string `json:"index_name"` + + // MetricFreshness Freshness required for projected, ordered, and filtered graph metrics. + MetricFreshness LegacyGraphQueryMetricFreshness `json:"metric_freshness,omitempty,omitzero"` + + // Metrics Graph metric names to project onto legacy graph_searches result nodes. + Metrics []string `json:"metrics,omitempty,omitzero"` + + // OrderBy Sort legacy graph_searches result nodes by graph metric score. + OrderBy []GraphMetricOrder `json:"order_by,omitempty,omitzero"` // Params Deprecated graph_searches traversal and path parameters. // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set @@ -16240,8 +17073,14 @@ type LegacyGraphQuery struct { // Type Deprecated discriminator used by LegacyGraphQuery. Type GraphQueryType `json:"type"` + + // WhereMetric Filter legacy graph_searches result nodes by graph metric score. + WhereMetric []GraphMetricFilter `json:"where_metric,omitempty,omitzero"` } +// LegacyGraphQueryMetricFreshness Freshness required for projected, ordered, and filtered graph metrics. +type LegacyGraphQueryMetricFreshness string + // LegacyGraphResultNode Deprecated graph_searches node response with an unqualified string path. // // Deprecated: this type has been marked as deprecated upstream, but no `x-deprecated-reason` was set @@ -16288,6 +17127,9 @@ type LegacyGraphSearchResult struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Matches []PatternMatch `json:"matches,omitempty,omitzero"` + // MetricStatus Graph metric status metadata keyed by metric name. + MetricStatus map[string]GraphMetricStatus `json:"metric_status,omitempty,omitzero"` + // Nodes Result nodes. Optional for compatibility with v0.2 responses. Nodes []LegacyGraphResultNode `json:"nodes,omitempty,omitzero"` @@ -17489,6 +18331,9 @@ type QueryHit struct { // Score Relevance score of the hit, normalized so higher values always rank first. Score float64 `json:"_score"` + // ScoreDetails Optional score provenance for ranking features that changed the final hit score. + ScoreDetails QueryScoreDetails `json:"_score_details,omitempty,omitzero"` + // Sort Sort key values for this hit. Pass as search_after or search_before // to paginate to the next/previous page. Values preserve their JSON // types. Present for ordered result pages, including cursor-only @@ -17561,6 +18406,9 @@ type QueryHitsTotalRelation string // QueryProfile Detailed execution profiling for a query. Present in the response // when the request sets `profile: true`. type QueryProfile struct { + // GraphMetrics Graph metric freshness and generation details for metric-aware query work. + GraphMetrics []GraphMetricProfile `json:"graph_metrics,omitempty,omitzero"` + // Join Join execution statistics. Join JoinProfile `json:"join,omitempty,omitzero"` @@ -17788,6 +18636,12 @@ type QueryRequest struct { // FullTextSearch An Antfly query expression retained as syntactically validated JSON and compiled by the query engine. FullTextSearch RawQuery `json:"full_text_search,omitempty,omitzero"` + // GraphMetric Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + GraphMetric GraphMetricQuery `json:"graph_metric,omitempty,omitzero"` + + // GraphMetricRerank Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. + GraphMetricRerank GraphMetricRerank `json:"graph_metric_rerank,omitempty,omitzero"` + // GraphQueries Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH operations. Keys use the versioned GraphIdentifier policy. GraphQueries GraphQueries `json:"graph_queries,omitempty,omitzero"` @@ -17992,6 +18846,9 @@ type QueryResult struct { // Error Error message if the query failed. Error string `json:"error,omitempty,omitzero"` + // GraphMetricResults Results from direct graph metric reads. + GraphMetricResults map[string]GraphMetricResult `json:"graph_metric_results,omitempty,omitzero"` + // GraphResults Non-empty canonical graph results keyed exactly by graph_queries operation name. Keys use the versioned GraphIdentifier policy. GraphResults GraphQueryResults `json:"graph_results,omitempty,omitzero"` @@ -18023,6 +18880,9 @@ type QueryResultBase struct { // Error Error message if the query failed. Error string `json:"error,omitempty,omitzero"` + // GraphMetricResults Results from direct graph metric reads. + GraphMetricResults map[string]GraphMetricResult `json:"graph_metric_results,omitempty,omitzero"` + // Hits A list of query hits. Hits QueryHits `json:"hits"` @@ -18039,6 +18899,11 @@ type QueryResultBase struct { Took time.Duration `json:"took"` } +// QueryScoreDetails Optional score provenance for ranking features that changed the final hit score. +type QueryScoreDetails struct { + GraphMetricRerank GraphMetricRerankScoreDetails `json:"graph_metric_rerank,omitempty,omitzero"` +} + // QueryStrategy Strategy for query transformation and retrieval: // - simple: Direct query with multi-phrase expansion. Best for straightforward factual queries. // - decompose: Break complex queries into sub-questions, retrieve for each. Best for multi-part questions. @@ -19008,6 +19873,12 @@ type RetrievalQueryRequest struct { // FullTextSearch An Antfly query expression retained as syntactically validated JSON and compiled by the query engine. FullTextSearch RawQuery `json:"full_text_search,omitempty,omitzero"` + // GraphMetric Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + GraphMetric GraphMetricQuery `json:"graph_metric,omitempty,omitzero"` + + // GraphMetricRerank Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. + GraphMetricRerank GraphMetricRerank `json:"graph_metric_rerank,omitempty,omitzero"` + // GraphQueries Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH operations. Keys use the versioned GraphIdentifier policy. GraphQueries GraphQueries `json:"graph_queries,omitempty,omitzero"` @@ -19961,6 +20832,12 @@ type StatefulQueryRequest struct { // FullTextSearch An Antfly query expression retained as syntactically validated JSON and compiled by the query engine. FullTextSearch RawQuery `json:"full_text_search,omitempty,omitzero"` + // GraphMetric Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + GraphMetric GraphMetricQuery `json:"graph_metric,omitempty,omitzero"` + + // GraphMetricRerank Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. + GraphMetricRerank GraphMetricRerank `json:"graph_metric_rerank,omitempty,omitzero"` + // GraphQueries Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH operations. Keys use the versioned GraphIdentifier policy. GraphQueries GraphQueries `json:"graph_queries,omitempty,omitzero"` @@ -20185,6 +21062,9 @@ type StatefulQueryResult struct { // Error Error message if the query failed. Error string `json:"error,omitempty,omitzero"` + // GraphMetricResults Results from direct graph metric reads. + GraphMetricResults map[string]GraphMetricResult `json:"graph_metric_results,omitempty,omitzero"` + // GraphResults Stateful graph results keyed by operation name. Legacy values are possible only when the corresponding request used graph_searches. GraphResults StatefulGraphQueryResults `json:"graph_results,omitempty,omitzero"` @@ -31431,6 +32311,19 @@ type ClientInterface interface { // Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName} (the `CreateIndex` operationId). CreateIndex(ctx context.Context, tableName string, indexName string, body CreateIndexJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // ExecuteGraphMetricAction Execute a graph metric operational action + // + // Refresh, rebuild, delete, pause, or resume maintenance for a configured + // graph metric. The metric configuration remains owned by the graph index. + // Refresh and rebuild durably enqueue bounded, resumable maintenance and + // return the aggregate shard status without waiting for graph-sized work. + // `delete` clears materialized metric state and durably disables automatic + // maintenance. A later refresh, rebuild, or resume action re-enables the + // metric and can publish a new generation. + // + // Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} (the `ExecuteGraphMetricAction` operationId). + ExecuteGraphMetricAction(ctx context.Context, tableName string, indexName string, metricName string, action string, reqEditors ...RequestEditorFn) (*http.Response, error) + // LinearMergeWithBody Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge // // Synchronize and keep Antfly in sync with external data sources like Shopify, @@ -35105,6 +35998,29 @@ func (c *Client) CreateIndex(ctx context.Context, tableName string, indexName st return c.Client.Do(req) } +// ExecuteGraphMetricAction Execute a graph metric operational action +// +// Refresh, rebuild, delete, pause, or resume maintenance for a configured +// graph metric. The metric configuration remains owned by the graph index. +// Refresh and rebuild durably enqueue bounded, resumable maintenance and +// return the aggregate shard status without waiting for graph-sized work. +// `delete` clears materialized metric state and durably disables automatic +// maintenance. A later refresh, rebuild, or resume action re-enables the +// metric and can publish a new generation. +// +// Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} (the `ExecuteGraphMetricAction` operationId). +func (c *Client) ExecuteGraphMetricAction(ctx context.Context, tableName string, indexName string, metricName string, action string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewExecuteGraphMetricActionRequest(c.Server, tableName, indexName, metricName, action) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // LinearMergeWithBody Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge // // Synchronize and keep Antfly in sync with external data sources like Shopify, @@ -39684,6 +40600,61 @@ func NewCreateIndexRequestWithBody(server string, tableName string, indexName st return req, nil } +// NewExecuteGraphMetricActionRequest constructs an http.Request for the ExecuteGraphMetricAction method +func NewExecuteGraphMetricActionRequest(server string, tableName string, indexName string, metricName string, action string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "tableName", tableName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithOptions("simple", false, "indexName", indexName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithOptions("simple", false, "metricName", metricName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + var pathParam3 string + + pathParam3, err = runtime.StyleParamWithOptions("simple", false, "action", action, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/db/v1/tables/%s/indexes/%s/graph-metrics/%s:%s", pathParam0, pathParam1, pathParam2, pathParam3) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewLinearMergeRequest calls the generic LinearMerge builder with application/json body func NewLinearMergeRequest(server string, tableName string, body LinearMergeJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -43369,6 +44340,21 @@ type ClientWithResponsesInterface interface { // Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName} (the `CreateIndex` operationId). CreateIndexWithResponse(ctx context.Context, tableName string, indexName string, body CreateIndexJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateIndexResponse, error) + // ExecuteGraphMetricActionWithResponse Execute a graph metric operational action + // + // Refresh, rebuild, delete, pause, or resume maintenance for a configured + // graph metric. The metric configuration remains owned by the graph index. + // Refresh and rebuild durably enqueue bounded, resumable maintenance and + // return the aggregate shard status without waiting for graph-sized work. + // `delete` clears materialized metric state and durably disables automatic + // maintenance. A later refresh, rebuild, or resume action re-enables the + // metric and can publish a new generation. + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} (the `ExecuteGraphMetricAction` operationId). + ExecuteGraphMetricActionWithResponse(ctx context.Context, tableName string, indexName string, metricName string, action string, reqEditors ...RequestEditorFn) (*ExecuteGraphMetricActionResponse, error) + // LinearMergeWithBodyWithResponse Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge // // Synchronize and keep Antfly in sync with external data sources like Shopify, @@ -49463,6 +50449,82 @@ func (r CreateIndexResponse) ContentType() string { return "" } +type ExecuteGraphMetricActionResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *GraphMetricActionResponse + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *BadRequest + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *NotFound + // JSON405 the response for an HTTP 405 `application/json` response + JSON405 *Error + // JSON429 the response for an HTTP 429 `application/json` response + JSON429 *Error + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *InternalServerError +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON200() *GraphMetricActionResponse { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON400() *BadRequest { + return r.JSON400 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON404() *NotFound { + return r.JSON404 +} + +// GetJSON405 returns the response for an HTTP 405 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON405() *Error { + return r.JSON405 +} + +// GetJSON429 returns the response for an HTTP 429 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON429() *Error { + return r.JSON429 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r ExecuteGraphMetricActionResponse) GetJSON500() *InternalServerError { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r ExecuteGraphMetricActionResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r ExecuteGraphMetricActionResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ExecuteGraphMetricActionResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r ExecuteGraphMetricActionResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type LinearMergeResponse struct { Body []byte HTTPResponse *http.Response @@ -54602,6 +55664,27 @@ func (c *ClientWithResponses) CreateIndexWithResponse(ctx context.Context, table return ParseCreateIndexResponse(rsp) } +// ExecuteGraphMetricActionWithResponse Execute a graph metric operational action +// +// Refresh, rebuild, delete, pause, or resume maintenance for a configured +// graph metric. The metric configuration remains owned by the graph index. +// Refresh and rebuild durably enqueue bounded, resumable maintenance and +// return the aggregate shard status without waiting for graph-sized work. +// `delete` clears materialized metric state and durably disables automatic +// maintenance. A later refresh, rebuild, or resume action re-enables the +// metric and can publish a new generation. +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} (the `ExecuteGraphMetricAction` operationId). +func (c *ClientWithResponses) ExecuteGraphMetricActionWithResponse(ctx context.Context, tableName string, indexName string, metricName string, action string, reqEditors ...RequestEditorFn) (*ExecuteGraphMetricActionResponse, error) { + rsp, err := c.ExecuteGraphMetricAction(ctx, tableName, indexName, metricName, action, reqEditors...) + if err != nil { + return nil, err + } + return ParseExecuteGraphMetricActionResponse(rsp) +} + // LinearMergeWithBodyWithResponse Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge // // Synchronize and keep Antfly in sync with external data sources like Shopify, @@ -60089,6 +61172,67 @@ func ParseCreateIndexResponse(rsp *http.Response) (*CreateIndexResponse, error) return response, nil } +// ParseExecuteGraphMetricActionResponse parses an HTTP response from a ExecuteGraphMetricActionWithResponse call +func ParseExecuteGraphMetricActionResponse(rsp *http.Response) (*ExecuteGraphMetricActionResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &ExecuteGraphMetricActionResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest GraphMetricActionResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest BadRequest + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest NotFound + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 405: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON405 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON429 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest InternalServerError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseLinearMergeResponse parses an HTTP response from a LinearMergeWithResponse call func ParseLinearMergeResponse(rsp *http.Response) (*LinearMergeResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -61947,2728 +63091,2809 @@ func ParsePredictResponse(rsp *http.Response) (*PredictResponse, error) { // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7P39chy5sS+KvgqiryMkzepuUtLM2OaE424OxZmhrS+Lkr2XF3Wb6Cp0N4ZVQA2A6mbPhG6cv84DnDhP", - "uJ/kBDIBFKoa1R8kJdtnrxWxPGI1vpHITCQyf/nbIJNlJQUTRg9OfhtUVNGSGabgr7+w9UX+lprFW//Z", - "fs2ZzhSvDJdicDJ4v2Dk9O0FuWFrcvFiPBgOuP1cUbMYDAeClmxwMrixDQ2GA8V+qbli+eDEqJoNBzpb", - "sJLaRtktLavClqXfZy/YD/Of+J9vXpav5dtf3un3g+HArCv7qzaKi/ng06fh4LKe/swys2N8Z1RPuSAa", - "CxM7oCHRdbYgVBMlC3ZimKDCTBSjOVNEKjJXsq5OmJj3zMa1ted8NvtIzua9okLTzI76IrctJDo2TZkJ", - "376emx180Ey9piXbYz9rzZTtsmf+tWtpzwX4WS5ELlli1p9sA7qSQjOgt9M5E+Y9KyupqOLF+oOgS8oL", - "OrXtWFIVhglj/0mrquAZtWM++lnbgf8W9S0FezMbnPzXb4PfKTYbnAz+P0cNmR9hOX3015qpdbq3c6Wk", - "Gnwabm/gQsyYYiJjZ7SiGTdrV+0jzKy9rDA18ovtkuSsYiJnIuNMW3rjvh2SuYYIVYyYZmikbsZmN2UB", - "dARr9o4ZtR6dzpKb+YoLXtYlUbYQyVlB18QeBpZJkevx1g0sse7g5GnYOC4MmzNld+7TcPA9zW7q6hUz", - "NKeG3nWvti1wbw9uoTeXGWuQTNZFToQ0hGlDpwXXC2IWjPjZktK1SWhtFlLZJZ+ymVSMaEOV4WJONM8Z", - "YbMZy4y+y5qLupxafjLzy02ygtvpEb2A4a0oN75X2CCOHOeOWzIc/M/RqTCzYj3yKzZ6Lc3oJfKcjXG+", - "VUxbmlxxsyBLWtSMXNsOr4kUxZqsFkyEYRE6p1xoA4uY1UrZimENC+ScJV1bxpoxlttpRKxA2DH/18C2", - "PviYYANATPk79kvNtHkw6tlCJDlQgu3s03BwJsWs4NkX6NjNkGSuR42LHy+qYlrWKgM6NMwO7wUTmr1j", - "FeXKUnelmNa1erhD1tN+7yT+rrhheoNDFbzkhuVkteAFI5TkttkRFzm7JYpNa17kJKMmWzBN6uqQA3VZ", - "z+dM27a7DCx1eg5gYD9xpqjKFuuzWmmpLg19QN6Varx3TV+wSrGM2jl6gUhCm3Z6lAuWk5lVTphgypYc", - "ITshjl5scWr4lBfcrL9zkoaJvJLc8hwhV1asExB6geI/DQcXdode1QbmeMdTSMV6D3n7Qei6qqQyLIdO", - "rdTE4e4nbrdI1wuxpAXPCZKbPV58XisYrhWwlhqrQq5Lu15Z6JaUXJeWJDfW4cFZQrL1XnKwOhhOpXRV", - "uizDswvgEkNQIoy2NFByQ2RtMlmySApOGamUXDLROXdBYryjMzPy4xu9wQb2kxm1uBFyJUbLp7HksDwt", - "DN5KBlpY5XdNFnTJ3EAtuVORk7LWxg9zWnCRF2s40rxPkDRd9omT1npfMrXkGburhrIfde/o8QFI3NID", - "kDkuKtd9GqIjeW5XZFoDU5FFYcV4Xc0VzRlZUA0LvmYgjJZMzXGtP4+SEyhiRnlRK2bHDjzb67MHMO2g", - "dMOFyfblte8HPKxpvT61KRuae/+2/BvokLjAxt7+CkvDTOHUP7tm5DslGnolzB2XwStmFjJ/Lc1pUcgV", - "yz//ULBHOB4U+wS5axZAtCht7cheS/ODrEX+JdRGJ+PtmGbQ56fh4C1dF5Lm76V8SdWcfTntdSrzNWG3", - "Vs/XqLs6ectyovmvDFVBO0RQNr6n+Y/UsBV9uAMK7b7wt+j+43nq1aByyvLcMkCpiGKKihv7hxWJ3N5c", - "FDO1EiCNSC1qDUzUK2JhIg+uFLRa3aoMwDQeaVJxYYe58LqlnY+B0RpZyULO1yRbUDEPerhlvlQTVQsB", - "wtFNxW3Ie14yWZsvvi9/xV25ZVndqGgd28iaGF6y3CozWxnnPTgdDOO+7G4/W9N+4v+93cpLIxWdsw/C", - "6kuR6vCxdyGdVB0TV5XANMGo4X7SoPtTopFYhBSjIH+R15IK2ck40Mjn4i8HEkk4ulxUtWnxnXB+pfJy", - "o8163lHDXuK99F+N90R0buUKNczfoD8rtT+AcfV+JtX+hYpWBFaK5qSiZrFNz7WXB2/Ws5KJa6LpzDJD", - "VIb+TTQuWLUPolIyY1o//Ia0mt4tZ+wy4nV6WtvrsnBXM+TXLCfslmYGtsIePrsBOcsKauW/HbSiaFm4", - "57XrTjqK63ObFm5HZm/NBdM6enT5ATnlgw9xODDs1hxVBeWd2onXkB6lOHr3aZj9xczRvrMXabq2bJGi", - "kdbbALi+Eu66PGydlR13cltU+2W6Er7Wqw+X78nrN+8tPfgrOvkAsiUepGZa2//CxY+6pq8EXPSnDA6p", - "u+OjCJquW9UvXoyvBG4UCDSvAJ/fLmitH5Kb93XQT2BOxPqvUm2aQlkY5mdi5aAn4FPHgyulibZ3vbZc", - "vCCZLAput3xIMmpoIecjLjKqhLNegVIKdioqAm3iClhpqCyvzXjBsXjMXkEh+QJ3/Y0u/r0u/feTQJFp", - "9hXLOX0PhT43s456bdiS7d7SieuaMJHJ3L9huybDW/ELlgHVgeuCkhVThuNTMhV6lVrkD5pZGrQ/DonO", - "aEFRfTSqzgxcYVEPAYUNjVYwMEuWNxxv3rQCY2qebB2tBXAlVuWRYkvOVkQbVmnLDS2nsPJS0Rmy4Ckj", - "NMtYBXxXj7hunsmnUhaMAh/0Q5jwPP1iH8Y4ZfBcBxMEBrTpCNDQyX+1Gm5MqRL9Gz4NcZX/6golVhmf", - "RzfH9BOfL0YFW7LCCqi6MJZJUk2wAsuB4y+4tpoD7uBwwA0rdUIwhmFRpeh6AGQ0o3VhJn2bHL8YQUm3", - "II0h0l0XFnZIpFJsxhSoLwvJs4SnwnCQWvhLvE2Ftec5E4bPOFNIA3BqRxlVVkwCQXNRwwkaDCP/CKs6", - "8dl6ImltFpMlU249NsYA9LfjsLU27C+OYCWMOLFNb+AftCAF18ZyHFwA7Sxe6Aly0Nb8EtHKJpm6qVoS", - "DatmJKH6Ju5toxNLOqkm/75YuyYdZ7KMWDCW70H54D8DKxoNeucR+Ivbgs6xvyDK3l1sR0eO1BZU5GBz", - "X3BhYD0pmUqrMueEemcQ7YnBPWw4tjEYDjQX84JNAkGWdWF48+dMMTaxiqX9N2dFPqlkwbN14j3EzcDq", - "vHWCBC4XoLi7oY1waI1pRmO1aIjxek/Cig7BkaxgqPVwMamUnCumNfzlf7Ojpbxg+bZxsmpzlKfAQgl3", - "p3f7mO0NxHbV4VVZmi7/blVmp3iuqCaG3rDk8cuZobzAtvKc48l5G/WBYrcz8FCSuPqETmVtnILNqkGC", - "5HL3djkpUxsGrGuE7inNnLlV9gUpuVXEUG8YbEr74QCsPZPS3gbniRc+ENZhqMAvkQTs0brC2leDfTnk", - "B8F/qXGeVktEyagUK/C0wkVD0Qz1l7vzO0szntehj1oPq4aB2BKW1eSb48mU1Jr8+fLNaxjb5eU50bWa", - "0Yzp1AB1OFN7DdEdwS4nEuhK5+izlwWFSe53hGGqGTVsLsEQEpxwpCwmGS0Ke0AKinbhoT3WWodzPRgO", - "nJ8B/tE8O2LRhgNsP8h9TMc9MFuJQ/1pboYdjRb8iYCJ4IvQcKBveFX1MpC5YnP0ZKizG2Y2NZbpfJLJ", - "GlXazcvMXNk1JFCCPLbkoflcwGSFmRimSv1k81QNB7ejuRzZryM7vJF0YnUEzhdWQ5nRQjN7smXW1//r", - "oPbnMqtLUPiB33FNpjib1HmeKVluNvZSWhkEFAFUruz9yzVjV3MmVUnN4GQwKySNGsa7RzwjX+bbrw+Z", - "px3VhOqJ25uN8f0A/VsNrWhGunHEDujwhq0TWwrzBdfkx3bzhm4d0Ak4B9cJZrLxkzH5AQQ0SNkRCFQC", - "u01oQ1K4FdxqjMAhcJCg11Fuj5HXLDkt+K8sdw4SoBp9BwZ4W21cUaXZY9yK8Q1bP0FzZSaXzpgGFcZJ", - "hsjW+y2qnbHd9pzrqqBr8piN5+MhmYUCdu4xKR++4DqTKsVow3nJGIEyW87R56ZCXU8n0QZuEd27OHho", - "4x3cZgapN9q6MNoeX4FXD11PR63OE4zdyITMrKp/6tE1cj8aq5tx3p2OOqLQnuKYSaaFYVjTF9Swd3Z5", - "Nvl8mi1eGqoMED95fHH5hvzh2+On+NpgNYAlIwW/YeRqIORq9Pv8anCvE5LWRF5TlHtgBrADifc3dehT", - "NHIu8v1mca8ppPSUHTty0G7EQuoxF1lRa75kX4Ix7N6au+xKfHIfs9svNp+77FPj5dm5JRVzNlWUZ5Of", - "JT5YbGWMvnTU9J9tvU9D25JU3CxKbLbY45U8Fh6nofqnj8NtQib0Q5JyhnR48N40Mg364GTGC2ex3Tb6", - "d3QFz2128hktmMipmkCjS1rsvwhnruqFr7k5f19kRFdU4XO/LYkinxo2WXBt5FzR8s6zh2bgECTU9xeB", - "b+mmTzwxnf6C4WhP8dpw9JTZj2tjt7x/XK5APLY5kxNf8U6j8432DgvUxYSkRC1Shk4ZkWJM3vkQlFoU", - "TGtyDdX1NTxe11VVcHcx3aGNJhVEbCtx31I5s31iW2Dss11s7UCPyTnNFo4Hgkp5N903WG2AvdqRfNd9", - "0QCdmDQ6MaF4Cx8fZHuMz1p3L25Z3j4nvUfkc8uekovJlmugfwXyl0B3GXUGRNwLI8nUHvusqNHUGYzJ", - "T+9zOS1lzg7hUyrnghbcrF/Zigk2zQqGdjXbsptC1lRrkTO5prWR1+SxM9k/sTcmS28lNQ1Z/bSumHop", - "5y/lnOgbZrIF2qikYAReqGwVkVtKdS7yQ8IKzciMFoUmlqfDkRToxABO0lxkhuiM2jHA12tCixVda/io", - "7ciqSslbbkdy7WPI7NDcCGyHYJ7QhM+IiMYyJhdzIZU70dIs7BVvXTEIKNt7X6Ticy5Sp9p+91xuQfVi", - "Mlc8b91ZHyM5n5CrQUHNsJDiavDkSpwjwdjPz38//v3vv/7jcPT02bPx10//+PXV4ErcR++tVPQq1x7x", - "jzhMEoqQx09HT5892ToHfS+LS5+seNeIiAeRXL3iQfNfE7rmK3rbeex1NzznvFQr3Dv3GWPwmgE+aR36", - "4/ss0Ge5JIfguo6GfcDV2D1B79klvFh/Gg5qwc3+PMzL9g+21iYDC/qEbXWnNnE3bR3muVNbB6PDprLe", - "8KXkTb2P2eqh5ZrIjRbMyWor3AWJmiRMG/yHvb4RmmK+j42q2RPn19Fhqd6Kamf9ZExOpxA6ZFdRSDHq", - "G9p48wH8AGKmy4TN4nTJwGcH7geGGv2lRb433vQYKJELpQ86Gi61s1wOG7XFWS7vwKmcaTzBqnpUkheb", - "qkhyGe+hedDbfhaJ5PlP27wyJXq9lvZPHpu3+EyY91vf5AGdQx2fOxi9dwt0tiPL23KqnEbjzSGeFYyJ", - "DwFsQv3i1rkmliOMv4Rx1+STnC3RZSxp6sN5hDL/vG3SdcoWWZf/1BFN5Gyif6mpSmlHdnByRtzv/7xx", - "AoWmXhnEvGDR6SuZUTzrcE5dl0NCl/MhKbkYkpLeDpGFDeNT8OSLTENxqzJszuRv7pd/1hJ/2q55eEfA", - "jivPusK32/hxTEIIem3YyZUYkVewIfqEbNkEXZeOwIY49da+2Fa+9zLxhLTFYGNxaonEtvHLtvAjkyet", - "m8WwpcPZIqeCFmuDg+1aD69E6zG6HKCKgVIB5ZYXm3Co/IFC3wD732hGdu9sowN3KxnEZrbBcBAG7n+I", - "P8RzwD/DJMBdqDPw9Pt4n7l2Q7MECKSJ5nli920NREcBb3NntNIIm2T1F2elgNOptzl2dKgKPWgL8rPt", - "IJgM4EnUDzyyBtCO75S9dw+8rjUYDlZc5HIVvkzc3x/vcbMtGdW1Ygeui+NN/cuRfo0Im+XWoz1xcJFt", - "kAycD8l2lzdXKNrbzpSS1xA/DMRsgFDPBEMGNXOUM8WXLI/2y7abUdXGZBiTt/W04Jn3+tXgCisrQ7gw", - "kqDOSqAxKD900YydNdBEsZJyQZiYc8FGciUwir1NyzioCbgkNB7LXScwJkBzGWK3rEN20VL7GcGlyJZy", - "rkXQ8piEoA20xSLAQWfvcjbjguMUqMK43gpX5PTtxd1uQHbwaW7e2sBLYEubKGmyrOzlzY3CMi97jctQ", - "9FsVcmNHAaBiTF6AqxjLiaqF4SUbEprTyqqSw+TcFcukysPWcb9cOadzIaHPcdJ3b8mCT+Gk4DOWrTOM", - "KDr4OMNSDTcaVXKlJy5SCp27g+ituUCpevgVp6czQ9WcGejzoXtyyz+Z0uxmxgH4oTE2f46OMkujdTWx", - "W1uWTOQs/7wdhlX8rL1AbNTn7UIbWrDP04Vei2wCZ9T5cWzzWW0zjf178SQ2QfJO8tVgZgqMQxMsDiFg", - "gAzFxXxIFKsKuoZ/gnAz2QIxS+7BEKMhgpmkfb53O+CF4iSv8cXLAe/4ZmMF/f67FgYbvKeTboq2CPFF", - "CMQwgGBFmXQ8Prba+NPxcTy4XNYYjtR7fdh3bIhMloglcP6itIDoXS7s4KBwg7dpj9TQwxEMQ/SSlbqA", - "TJPD3qOP+PgujxBusI28bmTFRPe5wPcpL5HUD62AFx1xtvfmkQlO8hC9cICkg2v8/aZhssWkrqLzdeeD", - "0LSFj8wTbZUvdyl9QBoO/VQLqu8loENLTlh+rgEvWHYDHyfIgiZWowm97eGvi4QzKuSc+Fqa0Jnx0dG1", - "AhWx6cjdEIw9x4JB+JvtePyw3CTn+mZSp/38L/mvwdcJoca4INO1YfqBx3CYw/OSa45wJrHIeOB1Yfn8", - "8wjdHissxlS4kAvCZ87O04Jnw2DqJcvvwS3YbQXBdRO44T20XjlTDMwPn6HpBaOFWazvxd6icOOUbKK/", - "1MwF6Jl18L9kGlxx/AEILYzJmXNQsRdTgFZUAKYDvP+XGs3n09o02HUwmkox45D4IHpV34f5w6gmJmmB", - "e8HtXyUX1EgVzCQ4D6QuNDmacWQhCYLtTkYQN6qCaoPPDZMmLO+ujZW8YNpIsft1GeH1muJQWWt7t/ks", - "JClk/nl4BLg6uQX8LO0XVAimJjNaFFZl+6ydADHkkbtHMJWC6w9wMz+Q+xBdqz//FmVVLF0XZtL7vPqS", - "GqZNpNC5dlxM8sgZKpt3bneObDfETwEVUpxDCFd+YInUMz+dURFMBHtOzdYZKbn6V5tVoMf7c43QbtDs", - "p4XMbiwv+ByNw6XlXrLJtxlOxQOfRiV/Rke7SaTLptT8zSB9S0Qp/ZVAPJIP1G902KavSJ19aLpJzgct", - "yJMZF3MraHlKpXwbja4FAhwkP9VaZhyQlgP29qZ+DhZIagxTttX/338dj/5IR7OPvz399tPv7iHR0xOL", - "Ii23TagpdsAkPvuu9N2pX2yjFxdSfELAcjhsWYFahgAFsOQPcpWulCylX+QD4reV4TOwjPv67q7WNVjf", - "1YQGFne9AGm2DHL0Afct2GL2UrHe+dI+WNk24Hdnb/ueM4u0DHz3MeIFyzKGxe+8UMb9B5t0/IaxoM4J", - "90EX2hLrnqtsi8ZLDHTuaf9ewsa1xbWuP48Gu9lBo7I8xMhB6+kMvxtsaQsSKKMREdfKKXZbFTzjxrGN", - "kW2I0Awa4WLewPg2TOmzkMBE12VJ1foBNAdnkPrMJjvXS7C3NY/G9x/657XdKYb4UZ/nEqiYlkV9D6HR", - "NPDAUgMaXjLlLJaHDe9HRasF8U089MjwGXgC5qL7EVDckhdgdxX/vrEK3RcfZGAOC/weo0KhEDCfHpZ6", - "cXgT6ojxMBq5RMRxX5looxgtETbFyDgoh8yBnFg+Z/ciHHwL/Szn2D+2fx4FyzBVfhZBa6ShBb7h7vte", - "qdjM3e9d0N2G38bDCjwHLTpRrJTms+xeF7arMYomvZUELdYum1zbhaTKaOLQ21XWIvWwtulLc4CjmB9G", - "X+CGG0zw3L+Tj2fCgd/P5YEbvt9S6BVTgAgUBZ3nIeFRmv28OH/77vzs9P35ixOAl32HzyO0iBsiXGjD", - "aD6+8hFHjcMggrABptCYnALCIoKGy6IBm8TVIVTkV8I7IhWQJw/idbV/hlK+d9cgeIV2PJTsDxN7GArL", - "CVNBFCXlYqQrlvEZz0go2ozZNpHyD8wWNBWDcGY/2+PvLANS7R23BlVfcnGTIiLmQmi3YnouaeF8AT8F", - "rCi58/Lzoy/Y1C3p7QReS27NxMgbloIp9JEg+DuGnDcxKcLh4G1iI/1SM8VT7u1/xR/s/jpUaxfsjADY", - "bpOYhrccuRLo4zfed3kRfL4JgkuAJKp1Gpv0kSaCmlrRghRUzGs6d6jcaVxEwLFsrRsEzw5Ovj4+Pt5A", - "+MblgwBDqIhRqUumFozmyRUE4NKdsUXNEb+E8p+GgxU3iwkqDcFw4Ia2z3l36oYUO0/+uQA7E0C1+e7I", - "0oG4eXDscRzRGvF+O0pZb5rg3EiBh20b6puSI2SZHkfmuT1GfTFz3qaWe8Lpj+rbBtMj3kRvBcLwdJ6W", - "iBH/9dLovuwXEF1j7gsfgkdsi/ummGUPbmu32+1r2hnEjz4xnu8enKbVTW7PL3L68VUS2BHLo4E5T5vK", - "+0cWVdtjkGdNYaegzTvDxjFuyO0SWSBE+Ac45+OuPP/UBfKbAKw6tuaoe6tcaFV+36rbIF95dq1YwZbp", - "sJZ3/icrn4JvQ6SoOsnqKfju87Vr7R4C9xFazSxmsijkqq4mHodVbwMSxtKjuiJN8QNxccEgBChhu+lr", - "AYJnJzlhjhTXKgZguCcbkFdX4kCJ5ddmh/rXZiuXXkbcj6tAM92jEh5wUoodJESqWMEFc0x4K6/ZQzHh", - "UthhNNpJBxbzoOPTaanFXba2Ekq2W/AEu6v+D65cXDu9hWZWrM8Wtbhhqgns2BP9w1Z743ClPw03I9P5", - "pFZFGgX6w7uX3gOnwdI/fXsR0pV6vMJHC2Oqk6OjQma0WEhtTv5w/IfjR0/G5AxCEjTgoGhmyJJTcvr6", - "/Q8v/3Ny8fqH83fnr8/OJ7YfJpZcSQGqIrjEOD2ugd9O9tG6JSuekhqlzFnRUhcGM34LLg8J4Gu7XFYz", - "gVqW+9WajckLrArs8BHUfuRQrMC7FPS60ZRqgCbHFr5zMCIYTuTe8wl1DYMIhn/qowx3Vh/9JmjJPh2N", - "ySWiqc7qgmSKAYiTT7/u0WQw2hdQ2vmSYVPt9fKT3Iw16lDYJmxV8kQjGcZJ0P1a+fxO4ytxJTaKARhR", - "xoRRHhvm/fu3AOzOMxeC5RrQTZPwXIqwR4YzRTIKTubjK2F3CdcQ8IFLWsGuvHn9+n+67zlXLDNSYbSV", - "Jo81L3lBlS22kCvypihoSclKqhv9BMb81VdnvuNXsCcnX311JUYElvCEXOIuw18jyJwXxmlZuN18D6Iw", - "rXlhRlwMiXBj8nrgEwxhXDuygMGveFEQagwrKwALKiTNt5JGMzc3bFwWN9yXT0/IK1bamdv1YriKz0Yl", - "F7Vh5P37l1Dq2Ql5y5TmGvI2vGVTq5fn1FBLwLYERu3OCj5fGJKzvA4ZJXzOBO9dj8/1GXiSY4TYVQsz", - "P2IwvecXLpZ2nnpw8s1xOK+Bfj11WTkHtDXAVD2Q1G3JVEGrcLGy9TWrqLvkDuwqDYJdsSl1DFkukFTP", - "IfFXzFnvdBqa/GE7jwN+e6SJpxby6mU4ELYDWKOmRU1qbRtuaNxK0Ivo3DSdR6p4zxkij0ukkf8gVSAD", - "fw5+YPZSy/wBeAkDsf067pbqCEmPlBuUF1w7LIEF6ot67RIfVr4/CdqZOOSk5kCT6ZzBLEZMjJZPx9+Q", - "x7//9g8k56V+MiS0KEavuOAvX41efjtaPiOPn//ha/zRtoetQBCgPaQsby5R1+6sMkdH4bBeN6d1fMih", - "eDZoDkF3zMnz8Gn4oFL9s0vtZ4dJ7e4EGhqM5MBjLwj6dqPZjCftESaWeGM0zZo3rpNu8T8m7T1OYd55", - "0/FTCSp2pAjGFoQwAL8yH3sVxa7t7q4MzR/wJSOnFzFTG/zLkdtOYqK3O22WDcCXCeY3f+cfJ21uW0hU", - "RAi1weiaUtKkELfumfLoR1aWdPR8ZNntAxCh1Spsx3UK/vtMCqNkoYmiIpclhIDxlu/a4+Px8ejZ+PjJ", - "eOu1/9mOa79pUtd2nBqtFhiS4rlidgw+mZYlzDbFZLQo4OlyW56s4cDIanKT2BdZjW6ItisPUpoqWjLD", - "VHpzbSNV6i0vK1ite5q5q3Xk7of9HSQqvbfysplruSOxvDwqrVI2VYzmI8qPytsp5SOsjDx0+bRHY9um", - "AyblVzJRQMiDNGVUMQVU4TH08WlzY272NgXJy+wdLFPMEMgi5WK/REg9BQd+g+ucvr2Y/OX8P8fEaYl5", - "tGi5ZJid33loEKkge4CsDWJLZ4qBckILPT5AxoVJNrvSCLoxAQgEiSkgh5tbiRdN3b5pcnhyRlLp3pLG", - "5JIZ9LfyjlkeMaqUkAeNIm4naqWhGY5GIFokQ0AP51f/whKk73z2n0uPx9PEWVCVLSZUT9ayxmdxSBOx", - "kir395nhYGHKYjC0bIEpnjl0GcsaB8PI/aXg4gaRZWD2+E/wlLRn1msUtkohp2l8mYr/JXW8HNpHyQy1", - "KrnHlPCn7DEeIIeFwZatjOwQRQnWCi7Fk01oCDRk5BNq+lA9Fiz0tKLaWT7y1u7Y9Ri5BdmYFbutuGJ6", - "vx5c4TF5XRcFKRkVGqfU25+oC5cctxWL3876sSW5UifrXDSYtkpAv89esB/mP/E/37wsX8u3v7zT7/dH", - "o/mpLqkY+ZTkqIhs7e7sojHOYlqHzYPMFMRwbU9N15QiOpMVF3PIfcvKyqyHhIsFU+6VFl5PZ3bdo5b3", - "frR9G+rAKqR3JdjzlVxFsPA9eKkb+KfxpEaIG6PkimBDY/IXtnbZZMMia/JYKvLoK7QL0qLA3/STMfkb", - "WgBtecet8fUaXj6Rceix3xztVwqdINxiNYa+Zhj6O1t+hFzZfYJOnBMq4A00Wc0EVUqu0IbWT80NH6s1", - "U2kSe2MH5TlykqZ+lguRy91AR+7EDD3iUehzGLOLJJcFDvZ3bhaXwJP2N4g73rdpCocEpin/rbfK3g3h", - "x0ikI0ZCc7Zqs5DKg+dgKtkTYnWib79+jPM8sf9BFtq5gv7nhz///O7Hv/34j+NFdVmaV9MP3yw//PTD", - "Zfb0r9/+4zyX2Y+vv/mr+P5n9uHP8vLD4s30h5tfL8vbm7/+9U9/6ss/pMO6dDTCglFl5Y1Xh7rcgVwa", - "K/Y1y2rFijX5X//H/014nF+8eSakc8pFeyr5j4sqW39v/39x8dPr4j+f/7nIT/eiBDfiYdiJj7tN1Z6d", - "e8HT3CjQTu4R77PurB/rhVxZvSZjTyB1sfcFRvSl4DGa5hhJh4PXtGR5/E7c8cnMpNB1ie9+VHhcprdK", - "5nUGz+BVbTSZskI2OW1Lj67ChOLZwuotCdSlyHM0mU7PX0dp/+jcteolE3OziC9WPRsW+kyeTvcj+v+n", - "k+HZrwBdaQB3PQzHKJrdNPFs6KlvOSailXqFKtZzqNZAN2BFBuA0Wi2sGK+Lwiff3BJTvTHgdz1pTPE7", - "oGz50a6QyaKz7bbx+iDosG7DQSaVqisTf6qFl9zx12bvJy4rJzqWg9twwUtuJuw2w3Sq22aHVK3PAlrK", - "ZRohxgcL+SGMprglGLZe1oY2kGghKbKQq2ErufbM6ukBj4cC3rTi0xqySEuAuyJ1BYE0EENmdQEqMBqo", - "jjJONxmIq0Ku/QkIF4oogbdrbeLyQsN6hnbSK1PnXLaeSHv0G02CQ6CRhNpq4RFo80D6Z4mt+UHfYCGS", - "h9jHdlZQMmVmxZiIe9Mh7cIJQctMsDD84duvj4/BmSxhaIgMGwivuH1of4cy/SOzIgNfw7C1zoJEg3xu", - "h3SXgSYfwmuzuKzxzw1X3TRW5ZmVBWrpIJOxLmk7C+DdWXmTvtVBAvgdFXmoVSlm56xj6nOJkJUsAmAj", - "4IviGD/GgtGV2cxN2kyomzxIT7kI3cPlf9Bt8cQwQQEQh+apnMwdru07c9CeKd79Pc1u6uq0APXm/JZr", - "o8+kmBV8W2SCk4edG59LU+IXawotTyg2PWHQdvJY9uCzvGRzmq3Jon3FQfxry+bH4K1ybfu9Rqh2RQW+", - "GyfNL01u3aQFf436cZgATPLjTmc/mHaTB9X3ErfZv+4XYiZTSfLttSEOP+loQXitcL/jm7pToEEo4cq3", - "iGf5dHwMwGMbU3fb1JfBHn+ObrOdHOm1NkyNsNTo2fGzb0bHT0dPk68q/pqdgk6rK+/1XWtMnePhY+Es", - "W5YrlfGr2QwgfE30Z69KfSDkEjIP+BL+dpNYOP385OioXCMkxRHujJuuPnLTP9o+b7wZJpYXvod8Qd7z", - "cHMQyHg0yp28zvAY7e/kZnjJtKFltcVQ4vY5ssS0lqGZ4funxyfPj0+Oj/+xp6Wmc14aeosHFlYp2rb+", - "Y/OSa/POaf6JXMK4OwkRLBip7L5b3dgrEm7m8B6iUX8uqeAzps3IXjUgM9be1oroWOND1AXWegoSsLsx", - "gt2aSVYrnWJ/Dhwpk8JwUSOdYtkxODgDzIDdPiHji7+fDgK5jvfcD71ltV85++AHERbtPM2xT0lgesGq", - "OHKVEK/PKrS1Ar3ZqXtsZq+fjvwwezpYP6xCNMcJ5DGqEjxP0Kpy7uu+m0kELVg3A+3bJz+nRjnemN2w", - "absAcXtIuy+hxkabzdPH+jUYW1B8fBoOpGB7WDP2GPan4X5N9I3wY9h3lw38tJzyeS3ru+oF/lox2SJp", - "HK2He1b0MKmYoVyEazT1o/GOVGPy3uV/xlehTIqMF9wBVAtpHBAnqwqaMXgJKOQcPEgc0V28GB8oGF92", - "GyCrhcSnIMifDuhjkApOI/SbJ3MPrJnssEeBco1Owsz/t9Sh4v3oZ1VRKNsBFLplp/uM+Vy7zYcFzDF4", - "xz9WRfIUaRKMHkR6JFY9vhJnC2kphsJbBBfzWe08W0CbczqBhswJR17R4yJ4/Y/bjneDDQVstHyGt7Bg", - "4nn2h67JJ4bSOR39g45+PR798WPzz/Fk9PGr36VJVQhEathcsosXaHvy7mAsJ9fsFqP4Jlxek6ZylHYT", - "EPyWTK2vhGBmJdUNWCjdMtqLmVvGMXmDNySvJ2jC7LpkPlElxg3i9Q2eJpj+jsx4wfRaG1aSD+8uSEXN", - "QoeH3CkTjDq8nmZwREnpQlOiZfz6+I/f7jCdtTVd74Qcaarb1F90QYZcHdeo+16fkHME0g82iWqx1sCA", - "tKCVXkhI56VNYrWGkN4TLToih+xA4Jl67YdzfULOlNQ6FOkwRy7I6Q/f+9E91gUkh/YkndeACYqWIocp", - "MwQAxowKNB47gx9w77V/EXGdga/dG9EM1u6A64prQmsjRzkzyELhuj7DfANzniEq6riVCeTLXBa8Bd0z", - "gOBwAM6XlVMpmcg1bOOlJcE8psCGxk7Itf1+cnR05Nb9yJLmNTgPl/RXKcjl8xNyDbcQJO+R5RJQ6sjI", - "IxwDlP9RynnByFkh65y4QZ+Q67neVRWdrMN+O84T3rTAXXOIe8y0yx4QPxkDyJALzczjEJc9b1FwucFn", - "t/ZN6rBjt+WeETa3xbnSYsRki1ZAdPuommxhpTpTZgjxyMalUwjxXBGXt2eHEo05kZy3qvO7PTWy5Bk3", - "66++Ahr56iuXOkkvqMq/+uqEvGmaAbsrVAA/W3tPsaUwozlVnGls4pUnP/hZ21Y+aCtDIkvssxHAP5NM", - "liU35PGzt2eY5dN1kAEvwPZXihvm3Gt/kqtWOzBfmuEALbeGiTwdE69egoMzg6dRu/hMk59enpFw4UNT", - "m/NdyaSEZDxGKhz7lXg2JmfRVxxK3Kuz2Q1JZZXGjFdUGI3lAPNVGH0lno/JKTgqWGJ2X4mus4yxfNjq", - "Ftej1cOV+HpM3satu5wgSOeA5L9QUshaF2sXvQviJHT/jZVvANi3JoWUFWFC1wq9iII9UgOOraUkK/+E", - "96yIxubuTLgRb5kCBiky5mgHKcft2dRSKNMn5Ko+Pn6ekW9KTQpqmMggcdNZtL3xFp6Q//+z43bRC5hF", - "BJUT2nx+HJz2VlJpM8osRT1ecgqbEqb7BEf8Y00VFYYx7QZ8WhR+Q91eEPeGbmdKHlN/OAgFcoSfkKZB", - "dp1tro2VFq5vYFmPBVsRvLVBGF7ZXXRo6CJnZSU70ySPvfc56+yT3X5NZ8zN63sm2IwbP6t3LK8z243T", - "Xnz8tocpzp2g5Eue1y339RF5ZWUpm814xsHH3aWHII/rKoezQ+GKbLfWBZbURlpFMNtyKMFvrOEiuqKO", - "cWjP8pGRYet1hf8eETDU2t5vvNuDnYrdMMPsjYqtml+80043DARZo/amq5Pf/+GP7r3/RBb5xIF/gRUL", - "x2Arwc9Pnz13uA2Dk+fHwwErKS+cj8H/cD2MM1l6L4KTwZ/lQpAX6H5A59BnVmsjS7BVVoqVvC4HH52P", - "w8nX33wbOnj2TdQBFaynAyoYuSy5WaS6+AixJd3URm76GwYSSLweQcOQixfgiYg1xuRFiMSlcF0s5dKr", - "PsjDQAiDFHktDUMN47UUI9g122DYG80LfF3jmOvalnxhuwFyMPSGQb5n5mKkl/AmjUQQpLq9kECGeXQJ", - "BBK07QSPmPgVrzVa7bQmyPqNo24TyW7KOCDBfKChvgzJPZlTUfBj+ApVU24UVWufIio8xq9lrdrppFKZ", - "kbt+5lVyn9s9+2T+Lou/XfO6133M14Xd/55pQyplT3vmyOADyHWhG0KwN0o7ARg10z5+8yoctauB/Qtk", - "XMHs0YAs5LC/pABNi1BnjYs3tGqEEBmRG8YAK6EkCPNLp8Wa6IVUpkUqGg1+BbvlmYQ3e6vvFush0ZJk", - "eB32j25NqjapjEsEDlnfu4zm35ZnJAgIUgMVbMl2xpFcrkX2EgradrzquY3fJPVTS1hcjMAw1lCmlzkY", - "hPZKirl88f1Im3XhTRhSIQd6n2rUKnwre2RcdA6frSN8AYfsAd5UI/x1hEqbop6Ou6qv928DtQCQgJy3", - "DKqYIEC96hsNpGIRHE3cEAQdIOwh6E/++ub8cJ1qakYFv2k1+Ph3XGRD8jtImPk7yJj5O5rn7+UlM0Py", - "u6ouClANzmRZSgG+4VYzwkldiEyxJlszU9pqTGylh8T2o4dkafk5nL4PsAONsmy71syg5M9z4rzfl4wA", - "C/Qxx48TowmNCYeXgKzNb4OX7N5vyvIc1dBCh2P/NgAv/cAv7LkbDpoVgiKyGpwM7EohVvPC/jWGmQ5C", - "3tinn4a+ILryhIIF1eZvnK3g+cmVHjw7fvbt6Pj3o2d/eP/02ckxPj59+mgbwQGFQ9ozGr8ucU9wKJs+", - "lryyTX7c850nkP5+sAzuWtn3YoXSf08sOR2ixR0mlmFpkCAUi3dqNlRNgw9tx5Z2d0tntx6Ta/xgWO5d", - "da6dB7bTgVmOCWbwiUiqKxFd6ohdKuqy2XLtsvO4OxN4HUftd4Cpr68EdmRpu1K8tNId2U2og+YqCols", - "uaBF5HoXbheCMas1e+7nvb4so4CY7panJOo7kEmSFeu2gSr0CpaIzqq0vnVmkg758kR4pz2Oa+/nkPM9", - "y5XMbu4YQf33S+Ia6AueBmuFLxu53gLQ2kkymEMPyYpNA478EG44LcddPSTnZ5fEUH1DlCy8Bev87BnG", - "s0A8lP0+7gskzuSCKTaGQY+WX58cDwkF29zYcEPFCH8w7NaMls9OjrGZFzKDygtjKn1ydJTLTI/pSo9d", - "1UyWR1NcjqMCkkaAHWxe85wdYVDrKDizjV2kSCJCa2Nw7YAs1wU8pcxxe2o9YlSb0dMBGsyYNpPwjo/N", - "TZZfb0ZpgfI/0fxX1jJsPx0mvUZMtiCAnQCG/bDfIbOskURWhpe2iFkoWc8XVW3SwXk5L5lIu8K8qU1V", - "GxJKQHebZIYL2tYqW+HqoQG9YwgpJavgFJUp2R0NXMJdNH/FxOnFCKwBhiOfbJ0TkMzjHk5e1aYn5dAZ", - "7FiYMxQltugwpPdz0UieDwz9B4hLGHb84zDLIzq29D4D9ofC+nFg1NnFi2EUq1YpCfZ7+1UqcvruNdpE", - "lvKGBWyZDYJ+NCSPtp23R9AYFajfxU5+rU5P373uuOGnjs5eUW3+UKVjwedJQn3v2Br+HriiXy4PB+FX", - "IRzSR51Bx6c30Xn7OMfvT7Q2cvPtyXXvZxkgOH3a5Gtb7Rpsa3PBfwVrsrJXZkB3h7Ue+a3Wwyux8fn0", - "3WvHcd3jR9iZUUMO+sgWS0Ym2ibV9q3VQxy/PXHgAee5yZDgjcsd/yG+DCIgC7WHlmn3cG9p0DWP4CFN", - "tKe7iK/d5TxnqlgHltJyTcYFBhr1buj4B7wLlTKnBYh45LDPo39//blgBYbQXDURbDUpuGB6N45iSB0h", - "CdQFgx/UbTAwHJNht8Y7NgRWO04COhpViyzpfN5hX74gmbIFXXKpxvuHZ24Ln/5eyuIHy2H/6jE+O8JN", - "yiKNQzyVUu+EsPseCn0aDoCLJ4xW3ccp213fOBkV/aM8aCw+Dm4nxBww9Xp322dS/FwLsDC36k2E3Fn3", - "BdcbdfVC1rheh9X8lF46bbarJqcEIvW5mCMgskeB8PDpOcsUoxpCu7lw/8YwD4+bqDOpmEYPB5Cf42Q2", - "3554uwYo4XtFl+wS5PChGG9/Z9NWxTTMWzKgHnolWLuJ85UKoqi/f3f6txACb7Vtq2U/SXryxtD/7R5+", - "8D+5GMUTUuV/yul6SKrVn1aM3QxJVf6plMIshqRa/2nNAPvcM9EKIKBW9n9K+z/p2HFdsaKAvEU7EWwd", - "Dm1Tg2gEj3RQkQlexW4heZ4MdoRdLPPCY7RZhhjVJI+nssiHhBsKcdZU3TCln6QBZA9HaetuJtqF448B", - "MKpSfEmz9WgmM6B1VADR7u2xlO2t1d5EwwuUuxBdMlNXABD1dEwu+VyQuiLUhJvN1PYI9xls9YhW/Ahe", - "cH9kpoFssLIjp3oxlfDAm7oj0YqPsY1x0yitqiOvuTr8qZT1/fu1Ye+omPfgrE/XhvVAxTgPXUQ2af7o", - "PjKc0YKJnKoLYZhyQrkbwYElRnQFaUldQcTFpoZNFlwbOVe0JHQ+V2xOPRE2UWOihnEuZK0AD2A9GA7s", - "qQHhJsCa/EtNlQFxB2cndT4a99BNGeKDfLey+VAfvFQhI4Z0CAdBE0UAMDobwz840zslNXSdknqd/hIr", - "GxJ+N+8iYxJ9xpjskq7R7CoLKUY5g2A5luOvFc3wxTmkPD8J8OFJT7jJ+GT08T+S/m9nFF6gbdevnNdm", - "h+3c0syMlnpEq0rJW8z4iP4O/vhSkjWtxBQB9mKrTZ7AZGic1SMnP60rpl7K+Us5J/qG2Xs2PvUKb+HG", - "OCaurwRpEqGzwjYlCLMDg1djLjIDjyohfuvJ2PYMJU4ILVZ0jc4JtUlVBTs21IjmeOIBEHHY+H6Gw/zO", - "uZ3yGRHRYNsGMqdEQ1eD4SBqOU3oeXbWckXsxHh7t0Pj/VJT2VRG4HWO73tWFGL4LiI22pVtfNXXzpDY", - "YOZs+pHR+QSD5/q68/bRgs67kXbDdIcOMWqvpFhuCC6irXcMPuJtyhZc5A/VrzaTbGGZcMiMRU064pAW", - "xSgrQOMPLkEuCAfyofq3Gu2WDFoddtfroJE54+edh9QYT8nZizNSyaKwwqlY33lYIRN/J+4Ylx17AW+t", - "YMhx7p7DBlN/GPnZDonziNkDoafDYG1X3gZg643JWcjVdzWopDZzxfTV4Dsyq02t3OBcDWS8NM/hroiv", - "UW2rhW8gOSpAocHo1DQCxVs/sqhodFIhDtPymebIOnuB4+6b+mMhza6+FGtMDhh2A10W0hzQD8Zso7tS", - "QmP4B1PSIW26Mp7i4t7dGXC+f/ZXQxv74QaxPX+WRjiDMPP0pJ1XrkM+YRnjS7ueeOrCvZ9r2HUcTnK+", - "+HR82KlfYQSabxzeliAMDQ6da/GOR6zfUhCtxsY2JTWUBeXiTAr0Lmlb1qSY+MiFDZUdy6O5DyI/YP+E", - "vSg0QIFcuCcOEKYgd0/IKcpfo9ZY3GoHKi/szUrO/OOareC7PyHvfVlALVg7eRukuyvssPdaxd2bukPl", - "g3raFVdWcwVFKlHD/kjgx1CpldjeTmEwjFeoGQD+0TSfFvAhI04i2My51hZc3KCvbbOksJ54v5ENvppd", - "TKsbZWEjN4OYoy3emaqnIYh7ZtuBoe2q984W6gEMbTrvoV7zqgnd6cZedR8mXPxNa025FEcZBBpLsWRK", - "U5+mc2P9jMvjt2P1/IDOXI1eiKsAzmSkLCKEK/uXG6n2+HUA4s7KCl7U4NmWcJNkVRCfv/8o39nigBMp", - "i0lGiyIZLYShL8Ch7egy8KG3rM2vKMLV5PD89RgOEQQIyYL9ydZ4khxq6DMVQ+z7sTI4Z15zo1pzbagw", - "3V7CD0/2Dmm1XZzRokh6OsQ0CEu6g/zOGvro+Mu5FXIEFEVVmJV0MSEungLh2XF9TsjV4CdWFHIIWO/2", - "3r2W9f/3agC+Mt77yTUKvuL6hPzXb1cwyKuBrW7YrUE/OPxXaPJq8Okj8LMmNrOjS7t0AJYj+nEnNrDX", - "Kat5F2iPcO+tcav5lqpEepCP7Sxy2uQ/aynG7+jqVYiva9LF8dKudmMg8OUblxmAaeJifgSfP7X39Z1M", - "3a/sV38i/BHQTOSYBWg7b9mE3Qi0a0U2PCYN8HD0SQ5jSTdtUTgN4d+Bq/gA0lZOOY0X3DyfeHPmaR7M", - "Zx6F7TFY/cH30ijKhUE3LqpvJllBVXhaPSGn+gYRRwDVPP7NVlix6QSbPvEWPLtyKzYlj91B01GhSRSr", - "JlXnhxmfwyBm9up9Qn4AQ8GHdy8DpT32SCNGIvgXN2v4UckCx69ZSYXhWRjTOeZkCz8cLQFS1C8HwIPZ", - "u5sDu4LuPSbTRiP2F3jEJd+/evbNtjaMYmyjuv3YMmNO7VXAfRB0yedhWcHXdKMBTI9qFAWKK1xV2Ddn", - "imFN4dhelxhjrPMEUgGK7VAAWPP8NlkV1m6LJef2UsdgVs2naB086lXzZxj01sOg9/fZ6RwEOCfoI3Mx", - "I9cMzOr5BL5eE64R+PEoANV2z5HH3QU8paKIfocmrpo49PA27kOmyHmTQNxXqqT9wjTBFyQwlHUWEV+O", - "HYmGmjCDHzw6t3MJyaV41DiGYBRhI8IhEgPf5TGXyJPhlaD29ltWxl3fZrQopjS7sSsBFnagSW1UnRkI", - "gXWuIRVVOoAydgEJoyVNRHJzbRCuXBaYIRHKx6Cbe6/5lcBW+te866vfIutNao2I+uP+uS4b/pxw6oej", - "4TjZztRKtmw7dyUoTdyw1CPON8M9QOG9/ta0AR7FplZifCXeKraEJeYCPf0g4glC/qAqpq8MgdoNfvrx", - "LjjzDTa+a/Kbz4GDpIxIuAgGnMAoVrupoMcY8MVz+C9or40pgtxwkUdCp8n2OgIfdlwH6eDUSromCiKk", - "MCQ7BAkLaQi7rSCGcsGuRNP9I90alrf120MvFTKAdTJz4KekCroN9O0tUyPv+ZJtcMIA/AaRa86DGuKi", - "/M1kRH5lSnrm5/2x4SUCM/oE1PEEmmOdc7kzlecGcN2ndiad3WkOHKBclOag5SHfwmwL0bZbgOV8Tp6t", - "Vwd2a7qjNgvF9EIWeY8Qyh2EgisFOwC8upNwCzMWPN2ZsWAnFH+aVg5PvZbK2/bQGdj2hz5/mGRqdyfO", - "Rn8BHemwLPeb2kijLYbgyIamuSDfW54zvhKAZ1UppkHFhYBa7uTjk6EvDz7bdv0QmgC42Uk2MydE17MZ", - "v42C1vLQ+CNN3GlDFuA7o1MND2zuN3voT7JWS8Aq7C0cFINIKWkyPo0Tj9r7Z3t/WEZA/sEUgFCgL3aH", - "gx3MJ/47H9+WV5idqRyZ8g8S6MwnFWtt9Ta3lJD0ZeVc+qAiwDlYNdXn5crR1Q+i+1y6CWiuMRwH4raU", - "3EAKc+GTgFEHLRLnMIOQPzhgrlFMrDyVZhGaE3lPjU0Xmf1Pw4MIpodO7bIPtjWpBUILbKogNJF9MR0P", - "cNc0ej0587pEmBRbG2PzBzd2bcEohC0JQXqztO4ZYNKBeYVs3YgNZv9JqKDF+lfH1NDz/Up4BAoA368M", - "L2l8M9XGEvp8jf7LnvB1uGiSdhZnveVWt08e3jNngUePFr3iJlvYGWAWXbhjY2PRm5kLYLbHGYXed5Ar", - "BsMjbS0jSc413PFalvDI4Q3AkybhUlc6h5atAaiuMLhjvnIQdq4dt2h7+Z1e+sI+NT1GEHdT02/1uqsU", - "GzV7FhoAH3LK4Z+YgAEoQHMH8+t6bpxy9nLL65JpMhf3BsFiPugOgbZgYwKavb2uZ7Kc4sDtBRQHz8SC", - "CsSv0xHY/eY0XPiqo1AXY0P7Hrt6M6u3p0mibOpW77ZkZVXvu2ve+/NzXlq2wnIMV0mOVCHfxJXCYH4A", - "aXcZ0V1iG8hQHac8a6PibXq6HqCB1YXhk2qhqE7CPhTghwSqAhQCeec0fRw1hEnBPZjlEQtCSBOw1e+J", - "h7D/oDtnrZsD4yHOlHt+8go58C045510+E0PDZe7z34oWRsWAqa2Pqraku8xQdO+rd+TVd6hox7Sf+Mi", - "6PKIiPBucRQ4gLPOEPdWRDCQH119IZd9azYnxGWpwteQR4pBuOyjIVmsK6tRY+5Vd5igSPzDo3vsmRVX", - "gBHaN9vvlQRAH1tmriDlWjNtf9QTBBfIkmvyKPTy6Mm9xnoXIXdA+/V0AuaotL3qBYO+wJO7no5CyZ2z", - "z31FmP0Dc5SN5+BwBjeY+AZpdw9VtMStHP3J52UMVvzfHAF0GxT7PwMGNCB77oEDijIbsiu7NQWuc91Y", - "fdfj/4bh/G8Yzv+G4UzDcB4Et5lMWnDo0Wr8J7flCCz8myVmOjAybNbFrMlp2mTPczh3kH2oroZXoq58", - "oiU3ar8+6HMoZ8SO1tVGNGFAF6BTuXSMFEpeCYAWdMGq9gSYCPvfEnkayW5H5oVDVy2KNvrm2zje6OmG", - "yEUYMPdzSrzeDd+0Iyu3p1D4EklB+sBk3th7TxH4nyvWQlKpPPRN45kP3krKcIi0DsmzWsjpcfx/08Te", - "STsu0UFZzgij2aJFRns71Nk6uAPY2k7PulbODJ8nwy3Jll1+QQ39UcnQzcY2Z7Wx90tIJ7ROhz/nXN9M", - "gMe3zKJbohpymU0QxO+ACh7AZVLwGcvWWZHGmIeHnPRAIQrAUew+vS4YLcxiPVlKAysAsmOjco8z/c9W", - "8Z1k9vAIXev0iFzWCMgg0TeunjjhOHYFW7kRciXS/djd07iHE0CPnbh97SteFXQ9yWg9X5hJXW0t1VDh", - "b8mwfio06NgBwyhZDtc4TRQ9KxxV6Z971wnak8COM/Fa5qzvSFB40IEb4ow7l9t9xhu9626+ono3m8MO", - "klV+M3smDjt+28gtUVxRLiYBiCu9fw4Ha5LLknKRnCMeJ7TrJAsUjGo2sVdTXSu277IWVpFMjknInB0w", - "T0VnpneD7NmZFJLm+w7Le48nxVn6F7jRHNBJh7b9tjYT30HkEAzdR+VfmqsDguchVBlXBtDiAypvERJM", - "5B4YYWd+70MFysOw+0NXShuqjJ/TAeSICvz+/bTDxzaaO5jD97HuaAWiQe6idYyW66X2w1jioRt/ICeq", - "GFMTV0en0Qq27UM34TlOfe/++9d9k8O0Wt+xBZc9evz7tb11/4PPfWjfkrMV3t3DrZYYWclCztGRANF+", - "Q7amTY85GHN75bb6UaSV4cRa2nnfqd1IoUhtEURR3qXdmIf37/3dmm4dmb1ATF3tn0DS99/YUBPw2+0e", - "udydMA7qEKjWBU0cLr3Nv3MG2X3zkBcq7TMAzb5Dw9UXsP22EkW6pA9ggpspWf6/1CTrEy9ZLem+JtkD", - "7HsrAAaLVptr7x70xc1dbgnCg1/ky0Z5MeEzn962a1X+Sa4slSyoyAvWpFbASzyal9stXJ+Q06lUhvBZ", - "ZKdySXSxAd0O6b3WN7xqNXB5w6tuX8PmEcIsGEb3Xnv4Z2Z7beC1Xa68ThPR07l2WUq0oXNwUxDg9Mzz", - "BqDHH/ON9WmPdjAchEG0rTMb5R7MBhkSifUYIVt5WF0eBiS8pEXSr2vaJPmSqnkwW+qQGdC3eCW4CGiH", - "PnsL+OQbUkptyLNvQltf0jT5Ra2RgX33mSN3GQn9Fmy1EhrF53Om0PO0Vj5ZaAO+vGE5/EzWQjfag8yF", - "bon2tBceYCNs9LZt7tndkAWzmLQc29p+WTnHjE/e79SWtxI0CyDe3pGt8d3z6kHC7dOe0V+Zkk5+Q6qN", - "qpBrqx1GHDkFodIUDDpmRB3oe4oE0aTRwWUTOS2kgIdoyDJQMJ1OvLkI6tAempfTnQDSN2e3kziqZVcT", - "F7aGm9hZXK+dl7OHDfrwW3TRtEw7Z4byAt5KanQYbGluyaB57H3PwCQ/VijcUK5mmWJmgmdgp/OMLXvZ", - "Jn3nq7yzLhZzo/DVOyfF7d6WA/Le082/2RGhhh5wHWhW97/P1n+frQc9W44Wk0cMwH/vlk/AAQf3pBLw", - "sJNLTsm1g0O9xsAdIhW5Pnvz0/m7AHZ6ncwt0JsQABHDmZgXXC9Gy+fj46CRD8nT42dfk5yX+snQFQR/", - "hIKLeU0LKL0lQ4DDJM9keRS8iY6gmba7f4wIP+jguwcg5JPB5kjboQHY3SbWfy+A7Ptm5d0Sp0PS2uub", - "Xt6E/x3cA96IYt14/3ag77e70Lt01xAEEUHhBzg5IzHJcEQ2wSfLhan4G8GVcE4yOVN8yTS5jlHzMVl2", - "x8nSIS9ed/bjGiPRfUBZSMcxJm9hjzHwWhaRZ5ELMgW4cXsBd56413FExeauxwO0ynfLxxY+eFT/JLdN", - "RGol6WeTIoQLJE6ey1YoV+venmx9LwR+R7efDTW9jVfuV+P89YsdF3wgOU0KKeawrxSvkiW9xSA1l1Qt", - "3sbXb16fD4aDy/en794PhtDHxweBPMeNeMcUbUezHsBcFVT+sswV+0xz1ydD//O9eSq2k46hSgzhX5lt", - "po9uehJ7nd1m2/vPbrr5+x3e/YPnhlGs+GGRYlEgC9WaaY0xr+dLWtSYahVd23+pHS6vyK9EAD4IOPC4", - "fQ6eFxYpowWfYmAkiIZMKjSd6Zt/z8CwT32rPq8VO781mOImsrkngovmEwCLahHmb592AkXHdXs2v50Q", - "4J6ZEjLX3v4vKSGDwVY7TNNuzywE+NS+8qc3tYK1St0g36uaRbuNJAhJkB2DQ1ff2NLvbmRq2MAkenZq", - "P3qgixauccsRrD8D0nmjTSUzIMFLT19iJQ2ORmnsv59aEVpumg38aozG7NCLNZEizSPTPbzClWteeAAb", - "fG/A570Rzlt7/T6ZyQmHAonKEIL3FQdMTZdyxccWKDa3F+o1MfRWClmuvyNXAzDqXw0A9TvADDiII0xx", - "E9ptTemRBgO53TormHzWmyvhlMdO/jpPQ4MY7nI48FRkdUykoUj9ZFgCVHv8J3UYrIoKO/+p8x81imbY", - "Hswm/eC3Be27e+HfBLsBDPic+BTZkL9PqptZIVce8IDr+BHMQULpJpcWrikmjRoG1D8AKxqGpL52inii", - "8gz+mCBU8XhwSFrjLM92mj5a+Oc7j1OwWLhiDZrmhwudPDT49LqZ5cMlRkQ7hw80Als31+QKq10N0k02", - "D447VXRX9EK2Z5l6q70MxnUU1amAnWZnIR5uxucjvN21YI+IXguzYBpT0/Eo75FHI4eqpKKoyW/MMJzV", - "3WYnV7A9vRsu8j2gH12Nv9jSvfytw0EhLXMEotqhd58R8UD48LdN3L+95Dg9yzdqp+OA1hZ0aTUm35JD", - "ivLJz6Isju60RfBS/fjaKaRp93bcnt2K6hZaOhvP7TXECdoTz9z0kayYoHykS1oUVwN7kK/wTeXEXiiO", - "4CZ/MpX5erJk2dUAeeT+B7t5ztpvhxvzXISzdgAqWCCtjrRCZx/EwAaaC0MbtpnpFonmCXBjC+xXePhy", - "ZzjeiEgrkU6DETJv4UU0Z6gDmRizD+SR2+VEn2PSWcTlociYXPn3SpZfDVyKWkoKiNpWcgrZIby0hJgD", - "loOhKPA7EmebtRXwIZE81swhxgAiN9yUnwzJVaTi+cpXAoz90ZEMGjwEdFkaFtJg+zmGAV0NahFyhTYN", - "CelGwTeR/toHA6DcpGoUHfKDVOTy+TDgsbHcr4eI3ypYTn5iNP8ePC+a5IF+bTCqfcnU+ko0Kf0wRXiB", - "2DPotDEmF4YsGcTxa/fNCikIf1+TiqmSa6tPDq8AMa6sMRdQ9EuT2uVtbd4AmXZz77qZBI+iYaxiD+NF", - "3EFTuv+pOpIj+7tltbj/rhtFaL7nVAZE4ARiegwzjCpiA0GM5tLHht2aIeElnQMe2RBzoRVcMFKynFOA", - "W2sQkXei7LQgineIQ9vph3cvD6nzyo6pVeGjXQaI5jot5myqKM/weSfyDtsL0A0bgbqY0n3nYNod9mLA", - "eWt2g5Pvqu22icCve4AJ4dgJdYk8g5oTunIpvgbNYlX8L2wdLdKm75g920Z6TCwK2T68OWvTvnFbccX0", - "hIt0pnA4vrUwvCBQ1GFyuPynv392vECMr+fHJKdr/WRMzsvKrB0bEpartC1Stsr+976OXhRpRCyeU9P8", - "2QWpeMXgINywdVJBapjRFt27KQVQmAAseU6zRfyDdxOiRNdTzUxwo7RLL+2tLeprb7jwt6EOrEvaF73x", - "LpWrSZNCMv30/dun4ZZpjpxKLlcOKntM/sLWGOVpwrJrSED46Cvc7sYP68mY/A3B22x59xiDgA9/vnzz", - "Oly2/HZpwsWCKWfjkisBt9sGo60Zhv7Olh8BGmIA8bad+GciqkkzXyKoUnIVUHt71s0fxb0tAnCKguVE", - "fyke1elxbybVALV9Bi6VM4FZN3VFlWYxKlyXT/1QF4UVLF9qvVr97b1aAf7yMyxWBwszWpsfFa0WX2ph", - "ms72XhXALP8MK4JY7t3ViCezacHAF1a9oA4WAd+FUbaNEEkBnlyEQVM7fgI5wSFhZpQnzclGbxDoRJ/E", - "/fZbZJrPntljj4BFajSpalVJnQRNYELxbAFvyilfI9TdqKBzlpOmLMkZQFeD7uyXH6fEw7HbU7Kch1aj", - "N9S7oVI5qJPWm8Fxd+f/5vBQWisFYKAhTWgbNfQ4YX0+ANemh3t3j9pm6MtI0xlLAjpa5QkX+VDysvci", - "xUsufLKhklaVi71tVMmt5zytGQ9jJr+1fo/YimB/t9dPMnGX02B7zU0W17zCuuyhuFuf9r2lbBnRLha4", - "dTH2q5yazz710juISV9cBITOb0a0QZ5LmExf/IUAVgzoP4WWhGYZq1CBsqSYk4i/AyKMT5+1ZMoQbtAX", - "nmugzVEwueZN7gYA4ORoN4hQ8DBXSo3mh9O3F0MwalhFwD+FkhVXTT6CwXCA89/Lky8kyQon1TlR97xV", - "7sei23CBKY7tgjXaHHt8JT5oNqtdIuA4lTFiQDJakkxapdIlju7Gt3zQDGwzgF2DgTmRgpRwRGpHuHxz", - "fJw0UgOQzBbdfl99ITqIXexpyKAZsbZwe3QStsUYkRGWtALnErlCS9qVaOp/R3IJhKJYxagh1/bjNeEC", - "M7guWLfBC5ykex+/EjmfgVnRuItEbOWEaJwf2rqVSwoUZaixhf6G8N1xiWYDeMkLqrhZ25KvYvNKUxqs", - "K/oI0NWPljxn8kpcif+UNeA10TxvMIx8NSNDfpLuJFywWDdA5LeGmzcY7OFhdnDy/A9fD5sHvcj9hRbF", - "qOSCF2Xb60VC8pNItEfS4lPwPgtduUKNREgKUlGXGOCderILmOVYIjI++Ks6ZnAmLzymNVUGSBle+ZTU", - "2ld1WTYF0BZGZPr85vxXLubkx5rncLvXkOt8RC5LexfNqaGaGU0eYzKMp8fHf7GHWD85IU9Hz13zsNcs", - "53UZVbBFR09f+dLPR0+Po+IQGNRtnjXFnx7/Ryh9JV5BeAtOBYmZTJnddjtlWhSs4LoEAzEXdonsbWrJ", - "1ILRfEwc0lxYAnZboUkXgm/hnYsGMMWImuzqIHS3hLRXMIhLGASsUXCR9M+k5KuvJKqbMFSiGC18NNBX", - "XwHnkrUhuVxhYmFotYymtlowzIJ5IXK+5HlNizDr24yxnLhXOQfVrcnjcOKnBYN4OATbLahhIluH5dAe", - "JK2Adcfx2cZslde25cR6WiqDd+vRgtHlGh6LC0lxSz5o5pRP98BxfRTmy65DygYQkRh6BEhqJQU2gf1X", - "BTeGi/nJlbi+vp5SvbgSb99cvidHvtWj5dOoWSiGUGHg9fNLzcBA0lrpoDDCkxvkL4YndytCMqbtWlC9", - "FtlCSSFrXayHzUCuRLQ+2kHYBQd+cG23NON3Mrzc2dGqunLSyxLOab6kwkByRkuJ5Lyg2vDMpeHCI3ba", - "tx7ksZD2ylID+mxI+eBvKbDPb5DSgv8u1PGkBSUAnLmiCkMUfLJeh5QJFiPFSsoFqD1aQ+7LvFbYU7Oe", - "T9p89Xkn82waIKTJ3zvpfaV8i1mRL//6EnLfbub81WNyCY4Dmly8vjx/9/7ow9sXp+/Pj16cvzx/f95K", - "0HslouZCJKPXz1oZfpecBnzBqFPYuFde8LgRgESaQSJN2xj1SVexJWeuvYZQxGvyH+RaZ1IxfU3+1//5", - "f/lO3a9PxlcC7JzuFR0cLq6lmGCO3+sj+++cFcyengCMjUwfMxlAise3PxKXeslqCkY2ftpOs4pRNsk7", - "nzbvekWLCTw3/8lN/RrfHhmJ1s3lNm6/527PyRqWDzOrJ99799JcQUW9xKL7R2JgLSx7yeDw6MGWK6vV", - "JXv1YDAH0GLis6vtEVbrqrQe4rxwtvv+mYzTHvnwENP0aW0WATIxehF16S/srUWuRmgPDgnmOmkWDBNU", - "GAALGNAMM5ftMAgPBxXVeiVVjs8iTdR6pp6bt/9D69UblcdI6qF8KsW2tmIh9aLxwf0Sll+wFW4BuZj5", - "p93cRzfbC9vQubdg9kpECvU5rOBtUBtSUuPSUNoaIB1LZrpvLz/Lhcgl2+kfF2bWbxHP2zfbQy2Y+b/n", - "e91ru/cFQHk3Lxbb3+46ViXFTK1A74XQfNCTIUVzs7IdQ8VnXdp0l//U14bkGu96ebj3Kve6pisG5hJa", - "jGaKMSKi0TVj2K93cglCeeQk6/J4/CzOUwf+IwpJSIo5XJYzKqRA8GIn6BGomuYki93NEym+F7W4mVjV", - "uWWjfXr87Oso4cO3Xych7Zx36H6pjxpbsk+DgJJpJ+z6CzwxZ67WW6wUOzHDfXdrHsacoyvcpGRG8Z2e", - "mC9c8VdY+lP7ar3HSfE+dM2km+t7muG/Y4Cpl+N+6xYJwEYCCQyj7XdO07Rx1QGbnsO+bUzpEN3QIo/g", - "mwe2vELLhhbNQsl6vggqq3PxbaL4kj6gLj3mXua8c186WhvnBZZKE5PAL7QLkXTNK1k5kTDSFFxcycVk", - "xfh8YbrPERtZTzp5oIaDqp6GG8B+VAtzfdtUa8gWl3ZCleEzmpmIHrYHjrxorKlAI+HXZfBl3UIo9lYP", - "z+FtiyP07m11JltAXpDmVQkBwxdScQOm3t30MBzcjpqZjDAXCCo0LnrO9xLCDJr+HmmSWp3vCE8NZZvP", - "aBPE4Fsi2l2+fOimkXMXro/fKiZyiM1Cg2BpB6j298c4df3AxjfXhwaK5duvDwWJHg5Qju13Lgwrq6IX", - "IVBWk5sOi9+evLb/1pFvPBDuwujaX1g2pLdVWDppHgR2Wqy51Ggt8XDcK8mCFNyjLFMhFioF2mWYaMKc", - "784vA6tsmIe39k1y7qCBtg62n1cmUnruJrF9fNkb4uj6svd4fG9ywk3q3UbaeF4nEA6y5Y6rTRN1dIRJ", - "JqFKO1xIM2UCTgckQu/yEQBK2wa/lPIZcku35aLUerL8Esr8v4wrTFKT77jFHK64p6a3qbYHxajp705K", - "evwaGmtj2iGOOafvtILVXvG7qjeRKNxLXvkF+gxya4vogBdy3+VbwDprJWe+uxRBt6Eg7SvX9r6Uk0in", - "9/m4uOejwQahNTNJN3XPEncAvuHm7xpJcvFxy8Mjez/zCr183HN7Wy0fuLnhYHbUN6s/MsUdvwArNm48", - "y+ep0+Sr77GGLunYQYt4hnWiq0w+P2wbzvM5e4UuP00rqVxHLqQSE+wVHgUkmFjSvzZf+7z1LFsRUMIO", - "vZ/cA5zu3jN7LfPNmVUUIaC2i46wa7tI7YtJy3++f2RSTEa+koeLyI05HWLWwp7vyFyD8XNSFVT4rJW7", - "KctXe+tqNXQVn/N9t7NHDLlTDEdhf3Fqz/H7dcV6XCZdYnfgUpOKqQbpZ48k7zmEqWWGLDm+a0IzbVYY", - "Z333gK6om9i7OHMgGSJvbD85y+vmubCTKB6x3IimM2bWDg31cVYrxYQp1uTp8Pj42P7/E3TL8XdnOp8r", - "Nsck65kUGS84ksa0zufMaDKFTI8rqW68S4e7pFUQh4cjLqngM6aNbuWnf3oM/7crRb1iWhZLpvbfOiCG", - "d65a//4dqmDtFIn3tw7UZUkV//UeZsktN6stWtxFc2VsWagaX1aWx8xiK5ugwrv8Nom7HsYxtvv6tL9L", - "7Mbryt7OsJ2L3F5usLEsexgH2M4o7va+c7hs3LdGZ1+iKL98Hz//QEAhIpUItirWwfbt7N4hkj0Lcgwv", - "ZOgGBOlVLUu0nyAGLbT8L+f4H0l//iViAPosfXcPBUg/e7yOAKtau5daoX/JaIID4sO6/Pd+F96AlXOI", - "pZS8sUQvpBghumjTinb+Jl2LRUJ7a7JjNb5Tiqd2DDDQNw2rPW90BzznxfBF28vOwB4jMgAHpwUmE9r9", - "4tOGqtya+iAFvz4BC+Mew4vA3lJYZ1az8zuEgEnwmuO837YiyLU1lZ04FrjhGTtsmSolLY27hBsHoIyk", - "JtgS+cmJKTbvW3T7r2oi2GoCfr89Oe5YCf5k3aRlvfMzvGSyNrv3MTywpH+q9uwugqrc9Bba58wdBLvX", - "iMVU7ryMaT25YetkNpXTv186T0twwr54EeXsRdYyQhj/AGKmx+QHWhSYaRu8kP9+OTk9Ozu/vJz85fw/", - "JxcvkLaFNJYZtX2S1rJWI+xvdMPWI56nRSi66SaiY56PvCeEvT95d14foa6fjymkBqYrPc5k+YhIRR5B", - "TPFCanPyx+PjYwxpfsXFxZsnnYwl7cqDnfkZHLJzs8Tp9XUsulnmu63x5fnZu/P30VLvWmfXdrPcSbge", - "Bv59yOi2qEQ4EYyEbxCF7EGUimKQpifCPaaXGiy0PcJxpJ3sJlq3nQtS7+vn6Il/efny6P3LSxjl5fMW", - "lJRHLj0htj6UOP375ZCAuIY/MQo90Mg+KIwA2L6gFXvBsgRu4D5PbqGJ7ovbLjgukKEXTeLf4N7ah/TY", - "XrEocp8wkcncRdsTXHwHCAHuhuAIQbQdJCmomNd0DrGRMy5QrR3vq/ANng7u++TWXq/IZBfBLyu5anDx", - "WB4eKR1oEaxc2xCLLGXi/IMt25eF/ytld31BDXM5weynB8G+zKlhADwPfgMq/XAh2tnlbJ2RrTRI+XLs", - "0IJ9hoL+VytIfq75kk1cxz0etnEGhD07bZqGxIkP23ho8iEXKskAmNDsNbwdOtfvtwuqWVJl0kwt8ZUx", - "ysNAlCwKWRtS2XrAh9xT5N9PX45cBnZwjXSBes4bfeyKTVp+NQg86jAz4YoM0sIhrftgIwzxWFDdOBY5", - "62Dw0lkTtkRw3hgKrABQd4Aqw9y8NS9cyh73xeeeir+1Rpg+TXZ+71hFOeRD96lpz9OQh6et9VAMRoEz", - "N5QLDBoBUMUFt6uFdgLww5Y5g3krRrMFw1s9xrugrRQtnakHxbz1PAADmCgYMWQoDtl00480IW9FEiTd", - "J4suU3mL6vkc8gGTnNkpucT5UA1QjRfOIjIeDBPJPrfrvdAKnrgwN3v0Pg53JXiGBWlmFje1MaePfecG", - "ozcjP7tdKZCSV+vzW5oZRxLO/Svy+CMZVTnIKbMO/vhy6rzrnIFBZBQcANJvyZhtKgltCC5oGEgFKdCw", - "f02YHVOxJuyXmhaaGKrmzPhf06i+m60kwpQWaw0Pq76fxqjvXxeMhOhBFy+xXz7ZeHAJbzxcX4zr8w52", - "WKlJiePYR3wwO6t6cELUzriSKzRstidNZS3v4zdTx4KluBC+4jvn5LjJZTAsHm656AlJVou1i23GBom7", - "QSOdAR7elBHjTGFUk3khp5DIz/cWc1OfZ6YWIQJmELZDNmOFY6w1F/MJJFyFBwUAcg5/K1ZKY1tqf9aG", - "Fqz5C14c1p3+HMJ3yTV4d8KXWsDR9Z/SLDvl2N3S9Wxh0Lk28zQ0CQxHOpOVlUn4eiRrk8mSaZIzw1TJ", - "BXNZP/IQp44LKZjWrRwcvrMmd9yUaTNhs5lUZp8p7Ml8DuUO0Uaiec5XGQYp1RFISAEIKImSO/Art0q2", - "BhV+sRxWRISSDZuRZjNuu2dczJmqVPK+7aywtj8fTN/BTWlqx3cHSPYyWvHcLMiC3dKcZbykxZi8BpOl", - "ayr43hBTg7D2UNeEwtO4A8GPE68ej/5IR7OPvz399lNfqtWYhpHimxzYnXcIxfSiiBcVNADMZey0J/AZ", - "jML7d6/DvtwWWEcKR/4dXREk+cbVP5wGPJJhgyPUVrfXiO5R0nWIzkb3TCMNLbwmyAsWk+PEUyLc5eAa", - "DP7l3fO/79RCnuDes0CLJsjFHgXNjCkAetQ0PBUjgTUJKYhd8p3kxIcE7hHg5BlWb4aY2BqAVKHYSLGf", - "ne57w6seCewzHvcOPwwRMFvd2vkA7I3e052k1r+/x6TiDjG47vZAycyS87BNmLG5LOoQtxdU9MA7IvVo", - "94B5kJoTlIhJ5/3KuGtHH61ZFfM7zH674jpy4tVuwiDFvNB1GL4/Nyi8zTD2fhI7QBNImdddZtJN7RzP", - "aUgkFWFrIzgKcnVIH+zpI3BtkcOB1YYXhdUbHCgAwFC+rotiYxHx0LRnv3kwe27RB7yO+Vv0nUK8UIw2", - "7pLOlzPfY/GAPGlQNX3NsGLBsRtVgUaJ2JdDOX6z91DEOgymu30nYXhDwFwQuIjFGjhMxTDrgK9VrB0A", - "9d4jxUb2GGm7cxyU5YZC4jLB6DEZyN6dR7JjjxEEQeqcgoAjBdwRMAk36FOAvQ03CuNCviMBdQjPwKAi", - "UsgVU84zCdwBiuaEgSBzgZeoFbZO097L4fRmSPy9Xbi1dQqHaxrEhnP5YZqsMLkTzQltkn0joBdOuejh", - "xp6gJi4x8h6UDDlHmrPikdCFFCN/aV/vuxS16NVfkkfICsYtimokzpsrlVNo/gU4YPc5DnlajwzfKSmT", - "uneXurbrsp2jGTHXhmNs0kijdDYcMN7KRr5F1+lGHYoUuz3u2C32H1nMdtzPojjS5FWN6wdNM5W79u6d", - "ZgosbAk3UDS7td1AsUdUERpEhzF5U3LjM3UsPS8PFQCDxl0/9HeQn85Icmz/p6Q3kMMD8r9ExT097zAJ", - "Hj/YyWiWM0kh7YDslo2geDbRv9TU5YLvOiRCNYJh38HWtIww38hjhxogivWTMfmgGaRy0Fywq0Gcecko", - "yoUPoMUCEUAcPiWTs5cXb4fkTcXE6UVojQvB1MRl9t/SaC593IqJWvatNPPc0sR5nRU8Z1QQH/KOwb1u", - "uTChT9xSyzYfr2Rr0HCs7Yw3j9b+HmB+N+DRa/P4zZQs+4+Bnw55HF59ngx2hm5Hg/NlgL3vrb/u9loL", - "AwMcOpfuImVhMLLf17uZHbv9YrPb13nN79sHwU3S6ReHXgvHg+ZMBkdwK3ABA648IQA1A3B5N+UJueGF", - "bL6U/ISUvGDwx8yckBljxv57nZ+QNQLnRZRaDoaDG3S0sMtkl2adzvPxQmYX+YMwfJ4f5FG1kZ6nh7M5", - "ZSdEFC14kYcT0rHdU6WtnmOLjJDcfAGpMLEV9bbOKBjAR1B5t/r+KKkebAo/NkT4h6BZJ/iDdymMJvmG", - "D6PtM6Q1oJBQDKIGVK53Nh/pk0zk4KQUDk7q+cH9RJjIG4g/7PKGreGexcDo4CUE+CVsmVVBtUk77lx4", - "9kRsmURvB6wd5D1w6lvKCeuN/R3NjlEqIOgo7Dq0PnRug1rzuei7Qu7Qee1FvKAZS0etnLnnm1DEXVWa", - "Je2fZqgzae4YCeoPLcc3Eakc44W10gteBVSwu84S2ptsS0sHHUZx41b0OiiujdPWsTFsvt5Cbzdb027B", - "liom0qTTZFgDFuxzf6Z7k7UBd4l0Gi23ibYUxONgFkBQLIGq4JlYI+xYTLk9ixp5i1UFNxO4Y1OVODUv", - "HYKhL0FC3ssGRnLHYcEuWMHn3D1Hpy/ZTTNoO7PVCDXwbhLdJvG7H8+WScYPsIaqnWxhxtV9+QLgNkzX", - "JoV8clpVSt7ykhpGXPwnywmEtkONdDeOP/hHvfEeB6Uj4MKpaZH0sCNbNighZirxAqYYe8R223JlH8H6", - "yp3H1NboylmGl5ytHKR/vxwNRkUncfFCfSVCghxUue2Ico/cBdBMmK4ZUzc65giGeJphWoor4WCm8sA9", - "juwRZJgjJwIB8yodUXTl83aCFS9n+ZWACeQlF0c5m9ZzwpsJahki4xENlC1lsQRoHq4aorSM5EqgChSs", - "MABQ2hxEy0ma8SeSYoed38JKQzQiaNNcN6vcvNj4HUgehR2qS0tT99B7EZJH40zqe5nYuevJ8mm/MoP0", - "vVOlcW5WXrVBdFfga/7oxSNJIc2EzvR2ZZB0lUHdqANhQdtyaL8nj37lNJXiFlB0di5LY1KGCjrsS8jM", - "CjvQsyYdWIaOotegLYbkrpAiD1Ila1Y4pGyEBW6i45OUFQhib/IN3f4/7P37ciM5ki8Ivwo+Tn9Wkg5J", - "SZlZ1dVKa5tR3qo0nbdJZXbtTFGHBCNAMkpBgB2IkMTK1do+wv6xr7EvdZ5kDe4OBCICQQYpZc/02tg5", - "06Vk4A6Hw93h/vONFLtJxnmnpMqVTKK6jON7yHiQCLzNftlcOWCdkCpxbF3RalewEVfLNQHQUBdhjCPB", - "jJemer/M2d0YVsNQ3ElC8MbnObttzPOWdh0xcdt+91vOD+ZBTuy5RFf93PnfCmLQTuSzO+YxMyNdcInB", - "LkKiYeYjpQKFpwuUZB03zygpZTiTsxtUxUN6QxicwmG6Ea74OlU8JlTSrjS0FNlcjM053SypZ+ImUUWQ", - "93gMsCQdaNeoDnK3kThw6O1cB7so0aRJ1fSxnjZz43qP84zLAi10zX5/Kj/ihIHwDIdFwRoaG5gJ+wDX", - "wZ2GbrcJ7HWyB/w6gH7HbLjtLedq41Z+Rie90EY6MejBm6mibMzz3GjgooOpYAU49ADnv0iiBfvw8hNk", - "0nVNDFt7QX7UtQt89zLNZ0aygksDcFUjNcc4W4/BbezQtDdGq11gF7+sWK7Y0ydMSTHAxA6mApn+IZFC", - "huMou3P3d9M6H/DPrN3UtZHpsY072+R4U+ZQpOe/6hjNEC3edJljHRlvi1tIlI3R31rEYyv5bt8bt81u", - "l0zXTnIGTQcdgCC2UMxEVguUrO2RlQu6k4XdDujLkJ4TLSA7QmUs4X5RDw9LMRaPhERs17a9VaqyTVWK", - "XcWz9tzqm1323ngeeVVxBvVG8CXEy1bVgo435HO3kcLBl94vn94CWnMjyT/wzFyRaNjKnv3oT56L1iua", - "6FZ5dzXqVZC/R0YZxoanFf+ZzrfzVhHC3P7biatU1p3KuUEQ9vJbj7MW/2P0RvIlJKBV61ziJDhafxCS", - "E828pjtMr/6O50nNTeXfVwkDUkxFOK5QUJCEa+pA5VRVVr2qm9Q0q5BWV7t32ySf8CW6WVZok112sWG8", - "TUJ2jHOXBGH7G4AmK0dAYQkr8d2fuVsNL4GL6NvoWJ1IsttzzCdBLOENumQ2H5E2qVQ0g1JtKt8X8Ap1", - "G5TZfuz1FuRyQctizWsGTIngIGD7sElwPOa2dc3IBFfObafF+lc1DQSU72+pYVNBU+BmgGFDEK5bkJJK", - "tbr0HKfyfkyyJ8RmBQCMsRXXnZ28IqPVpWPC/9wkS2FJjB0FT/ypEC7TE930vDoG9sn7l4DA2UQvGM8Z", - "RyM1hJhYInJmdIr1MlP7TU0xEw8hxOTKjqOyor6jP0ahjHk+XiZpmoREV5ncMbFS0YJhEREpSbm/XK/m", - "1vGQnbqspU04v0fXiHuCqgFmXEURE4ZCLxpLCIMzYqK66T6oNoe5l8WySHkFiF3d6tAhtL5RMW3jsLWf", - "IgsZ/d5Yt3QnYS+VNtwjQshsv/W9rXwNnhdg3HATBtnRhyyZw+Ops9tXHCztQftNTYPH+Dc1DV8GAEg3", - "sC+YzqkWeQLsbSk/7m4TazGHlS67GZ0ds86mM6i1l0EJOsU0WY9ztMAMRg12nnmyTDa94opsgN63lrcY", - "ikIcwLBLWLMLKe5a3sIsdrILRUhkgbzQ3F0ETmEjFYpMq8xLfYI3/tZVJn/ErukonZ9tORhNbvXUUvio", - "rsKh2m+TmYjWUSooJpvutZJYfbcryPsH72bI4sGjM4qEiOHX0gXTcuugo4tj/q3Goi9aZIMZjzxGBE9C", - "yBIbVngcOSYNXFJQNO2GP/pEjleZmmdC66rzp84VeJNuHOwO7BStMU4kuIWQN9dMeHt0xKXcoQ8qz6br", - "jqwaiGdMyxJQ5d05qpC5JepcGU2yMIKPpCxfdznj8Q04VDkN4eGcHPJ8voReg9CWbaECbcuE5bsvE8jB", - "LYLfZ3wLUtJG3geEfN1FAszVlhvJeQ6yYrUK3UgBRyCBHhguPe1x26t8TYCme6wil1bWoakcIy8JnGTv", - "unWztCy8pPHqqfK9uR0HqTFFT9KoU3JAmg3Jg623WUiC21WFuMx5lrcm5P9k86K6NMJl2A+l7nYuUE4u", - "Js5bU0zwvG0FCMLsueYDS5ZLESc8F+naiOmafMYynVcvzBJqocgcbVs5VISl7qpoZb2ce/1WpzZfvNJN", - "ZTDsv1be/y4fzEm/xTu1ItQqYJKAN+zfnHS1oa4SxBKmnrqKD/5pbl2Ei+CJDizCA092d7r9RJbBpv7r", - "qDDkgRInEc+Ftvm8SlUdXzYo6MW/dykNcfCCrbvmuJ53OoL+pdH01m7RhTy5iu7SNpXIOpH5VGT1liY5", - "tDtBOkQLPPXoDQkduRcq9IuEmZDIDuGh+zoItgjPnveBOSEIml3x9whN9YGiNC4uuI/B/Cjzny6WzlOn", - "7Zj2dhXMvL3tLJdtnPsOAlu4b09e26WTNnGnA/FKxVIl5yJjCx5Drm6aKrp11TjPLjRed8e3e77TDY+k", - "2eWYg9jlsa2Wa/aFsy25rDZV/sSb1jqbrN9buwR1O5XFqJ03Qz8e9cpjNt3cKkvwPABTZnZRbR7qRBbi", - "H/R+fJjm4VCCp2vGa6s0ZCDpkLbdr0KN4TRQaylBivKF0KKqSVNHidS54HDTIjI/0gwG+CK2DVCGXvFv", - "r+f8Y8kU9RPaJlg8xLAOPK1dlfbyxT34vneycKQyzKhGKQLCESqbzaLfzLC5390uKmL/t7rWnYXA74wE", - "miWsrVnzJV+TxeY/x1BGBni495DdvVGZy2iLLImnqcg0ITFodkpR2jJds4ldlYmDceEyZielC5WN2g6B", - "0f3dROwuBreXVSObhz5RoRY28WxoE7YUnBRKXCamF6pIIcOLTozCuxAl+0bqONCHsEiG8OPCqL4SYA6g", - "/edsYlfNNi5V6Faw693Bunf19xAdv4mEWD+nj32xmi1uXph6yF6RbWLKo2ujo8jYMwfr+iaj2RVMg3i3", - "ehlq07V/q0YqTfkK6NjuFOZF43Tm7C2LI/zPsyU+tnDdQYYu+UHQqtYITdlVzt7ForZJJoe8Ju+8/Da7", - "QFeKqEDNd2UVYcqU4+4nXootNrfMd5pN7gZc5rN0PYDAkgkD9EFKE4EJobEZABpA/gjMB/ynLjE25Ian", - "BXoODdkrEaXcutc6ZzUjfcQqRwtxvrDKgRsRpNnCPlzGICNFc0kPeqAz1abWJ19q9NCCQRAJlRiKDlUO", - "hA7C7jLSTpbkAuJjADh8yGjhdQUVa7pmEy7XH2YTI+VNIPnOJDBMN0e7WonFyvVygj43TTv3STP0eC25", - "BW0x1Bfb+roy3oCxUvJ0/Xsoo8I5fYH2CGT9Lh+oLMHMo7aDsHgHsUXttNeNV1wW05lPyvf9gCwag5tr", - "Km5EypZFCi+2aayZqDqPGumhJFbYYpQkyNmxzzj6X+ZJngqMZbpbwX3CtMrwSFyL9a3KYqZpYN7cy1OI", - "8G9inMgxT4NJ6FsCCRUhxwlGwZ+mPoZpYcykoXQ8XYTwGjT4JuFsWpVuYnFHNmYzC8T7yEGHDKiQIO1n", - "Rci+vAP0gQUMNRKkl/S+1JxsAUusDU3qHRY4Ni3AnnhQg6WrDUFMgEs5fCbq14w6MAcQmpD2JSqzZ94X", - "VRrDuQo6pSJl7LDLkAYa4KZ0xFOe0QYbgpsK5LGJRMvGeFqmaBDxSDo6tMwBmZ054USZfSaLpciS6Bgd", - "qY/pKuuPJG3Zsflvn1lk92Pzx7H5S+d8uUJGmCbyesiIAaDbcX8kkeTGXI/XqgCvyD6bC9Uvk1r32TRV", - "0z5b5Mu0T0j+fUqJlfH1SHohh4arxkkmQN+3E3sOFAhHz1sge9ogLnkkucYzOrSnEcJ6EWCcMgWs+Dwh", - "nF92DpcSqaogCI+ky2aFQZWxiizTR3BySN0QYxBiKCxXZWKnc415J9yBo2vO5n9ApMvDMLoWuXRvTL3s", - "8cjPkN1tkzkAb9oA8IaYJdKakCzOMDjIVy/X3VKZnYdymFXaC/tfu0GG74/Qe9o54Uj68gSmMKOHMtvt", - "d7qc4HAkLQ6cdRJP5MwmuoI34iIlb3XcOzhzFdKgNTa80JeABnRKB+Vkulx6FeHtfvNW1u7H3UQ9qgXH", - "R0lh5QZ3hVZvUMg2xxFSuy79Ddk77+JFPDQfAtC0jhd0LMSKlBGOAPjUpnc3w/7hCck48UwuGWcSvRnh", - "u+v67yXR/Pet/t+3+i63uru0Wi/25sXEOt5L/1+6lTK1cqmDvFej2hWTrcdZIUOzaw5u6XznLQ1mwmH9", - "OaT78qeI64jHIvzuvW3AbUb0OFPWbLFH1qQd5lsPTqB+yzaChgJUFz972Xp3uDjO4U5EzDi6Q3z9MxY5", - "BZjBfTBkthsCm18IyCBjz8JI2lwo6PACEIQIno3VvtOlMoy3dAgdwkvnu2nB7Vje1PRKTCvQDLxN1ZQR", - "7jtMGE8KXCAH4CgLRgiB2UAPhyPpUo4dwT13dMRukzSOeGZW4jXqmmds1DuC5L+jHs5XaDbqgVRLP/fZ", - "qDdV8Zr+OZLhsO08Woxp5i3Rf2+SNBcZm3r7AhcoiXI+GCZmyLFYcOVlYFQEABMF8gkxzPBLkQuU8yLx", - "rB3Hbi47KLSYFSkaOsW0mM8TOT8Mwjrx3Ey22z4BPZnrwLMVefngjo6IfInWQOpZpeaOnIOD4HAkvc1y", - "EFBHR5Udc79jQiHcNvdjzud6uMqSJc/WbVsIkypky7Q+AnHhM7W5+XCCQ3YxY0nuxuHmahrrV5c3cd6V", - "QammtefX9U5r1F8bQ3uvlXU0P47NEt6CdRpmFguLRJMm18IWMhdqeM1CTPl1HIJ3OMdrtM9uRTJf5Jg4", - "nLLvsanIb4WQpUtoM82Oc08Mil0ozIl4LvyAFP+p7kEJvgjk0hLTbqqQO3kwPNdGSGPpNgy0lQfcObgW", - "PzwbWNto4FXbX5DpOn/IWmAWkq2DoERA32oQHUQfyGBuaPIzMtmubZd+r10ozo/TeGyywxMTDOpOcgFI", - "3RLwOG9oQJHSCBWn5IyytJHR/26MjVnsV7wxkQJN9ZKq07UVgLUVexPJfj3pn15VnsBjVWCioABucBPK", - "dJeE20TmjtSoWbceV/ufIEMOr8De1chAqYo8ADVMRSGUO54LbfNorQH/VBX5GftQ5HMF0ENQwIVaSBUL", - "UyiRZwySSZRl6InBlpiqfHHGXiizI7YxyC1fqVXBSsXRJhJEhHzxEARfe0TK/OXhpM4lbhFneiWiZJZE", - "SHZBg5RLHllbUssQSC/zIgSQY0AuvmuxPtCH2FmRqyXPbV8u6bkn5JG98YCeRrGhQ3MQwOKJsF0ZvMMc", - "OBkDS2kjLcKpJpiLPm0RPApNXUJ2c02u0iRKMKcaZ2MoNgzcjbtnz+/OxTDxtErVvKbgz02xBv1+prJm", - "mTVASuelu4LbOiDlPBPijGEQlzMHiQy2ow++BeuIkHyhrzP2XnnNlnl0Dyt0aprt9Wl4exNpV1Bjs04b", - "fKpa0Bo+Q6qJEpIdCcB6Y4QjWeP5Di5LIBiFsISbE0CQkaw8izxNP8x6Z79+7cFrJfy1kV6Umqei1s59", - "f3Olv4osF3c7VvqQpnzJd60EIOp7VPqkilxkO1Z8IeJMRdc71nqpFiLbdQnRjlSvdGWq1WwSDrhArgof", - "826VCULrCQmTr9x3ZpM4JTZ35ZKp2UhOMpFnibjh6TDQx2TI3ovbKsKLc1AptBhJdEQD427Z1IRkhDDY", - "lOGlSxWHssGg34DA93vgN4Bsz7Rl2kZ6gadeC+d3kCz53CgxvIgT1WeG7ao++/jqjT7sj6S4EZIlM4qP", - "Nk0lWn4H3ti49t9pNi2SNB8kkgpkYp7oPFuztciHI0mMfmKWd9K3lj/Iyqi9Zrlm5cTQA0vIWLNpIi24", - "p5D5SNbHK/JoeMjyRaaK+QKez+yLHOMxX6H7HM/LFbATX/EMVF0aUJm7k+4e6wRBqJxeTnKqAW8lRtqD", - "0RLRPx9JNKzLsqXMjkST4ZZBxFPFwq+TFHuPE41JwOYsweV7r3ImIYLVT1XAU0hRYg3r1jOalp6S8U+i", - "NFkNjib0V5Ty1eQQGj06sqrp0dFITiaT37SSI/l1JBkb9ezARz2jueLDzqjXx48wAPyi1VIMZkVeZGJQ", - "7t6AStjy7oOpZMhgJO+hz6B91fa9lb3Tof9oy9/3eyAjfssjXu9g2/nGJ9fdzrftA9/pnMT8DWbhevgW", - "08h4LsbOBXjTTn7iuXhrChL/3uEJxg2lG7Ukcv7J1rCXRT2tkCWnq/urpvG3kMksQWNKXTr3OY9tBI6a", - "JVRtn0M8ZGxINJI7G3GuWFToXC2T3wVbqFsvaJdnYiQhJQ089htdBtyX0RPcde1bnM3G/cxlnIopzzTT", - "a5nzO8OsRhaVit3wDNAlHQtfiHQlMk08wrbFLtc6F0vgFQN2dHQJTR0dnfnt0zRAJ1jk+UqfHR8v3Off", - "9DBSy+N5kcTi+BCbecnBAmnaqdrJnfaB2XoxbR8li6GFn6aCff781gnAZ+x7tkxkkQttW1fAiqutZyIS", - "CIPmbKQW14muhrsc5/7CLsnPuCQw+ZE8HbKjIx1lxfTnfJkeHbEB+wTQJwwp5Vjna9Bz5hWYNxYZ/oZ7", - "Brrqz5/fvTX8kU0mk3KV4JevX1374E4ypivr/t5WgP/ajjWbjIqTk6cRDgD+FhPo3H4wQ7K/m5FR/fM4", - "1kyK2xScH9DXfJqq6Noa9TU7WPVZnNz02eJ0sPihz9KErlw3BCOva7ZKeULTg30Cf0LIvYYQs8CMzPo9", - "Mesn/gYL9/pvBWYCB5aVJZrOkufhp1vX6J+SGTsQf7OYC6MeB5vKqHd4f3+O5pVCi+zr1+NkRitXVvqX", - "a7E295dRqM29dHh/f4l/o4bs14L1HsmnZuBwccPYfxLyL0nOYpWvMrVc2Tsd3XoAm97c1aVE4wSYlulg", - "9SJL/wwCziue8y+fLtzAy89GqhtCmXGRpYECo549feTeBwcPagx/W81HvWAd/dRUkDGw2GNMvUOVVrKt", - "0ixJxdnx8fGK54vjXAU6ocVjzHAN680FYIXgpSLwWAEpTWKe87MJGzA0ajqvU/M7+/LpQjuxBkpCZ8e/", - "rcT8+RQq9IfD4cQS5sQswtnx8YQd498a/jFgv4ip6d8mlXR2jgq+Nb4XJUra1mimpoG3IGqZHzRwRTDq", - "2SM1gXU0xS6f+nlfLYD5AVryzqhgfcGvxdrMgBbs3A3uJQ3uo4u0KNft6OgCRGLD6F6pW5kqHou4zzKh", - "k99FzA6SGcmPh31WuUHcwrqWjNiPDPMu91AU4fkjgxc/SAksY0h7wDWDTXDVDVMz1T9ZWOm82k4B3grv", - "1O9JmnIq5TgD0oiICoD6NTPOVOrow87MwtcoetCbOt2AaVtXC8hyodmBFoJVLWmfBEnth2eWC6apuhUx", - "Wyids9tFkos00Tl9/JglN+YCvPiInBFut1Vm1KJcs8vLT28Yz3MeXWtLeHagzKw+Rjdp75Y6PTl598KW", - "/fnz548sdjPLkyUkByVW/PSEWaAieMKEFp6z30WmjJpgFlhTKBqPDQ+nVoEcWJws8Xm9OYYnJ89+XN31", - "gfgHRCaW6i6FOGP2vKDkP0zUcawifVyRev7JLvdgECnDxkbyGfB2OLWflZLI4+GfSGqJZJ8/fHjP8Ayw", - "g8/qWsjBB+uD9AGESfaePOcPW3ll2QXouEN8Q3McKvyZpULO88U7nl2L7M+YYDuBIJA/P9tWNRawhiL7", - "86g3GuVB7vaL0RYTDTP8ZyJamC0kRgVWEOV9tqjirtNSxIKwuMytseLo1eMc9YgF5oq9ffvOyGaMsYu8", - "VFSPjp6eDH44+f8TAF8myF3o6IhuVjzu8PaNeceXPHEW4EUyX5iGoVlTPhMLohweRUXGo/XQzfIvYs3e", - "CG5G5bHulzg7K2LiMZ+cTWA612I9QPcaCN+xpw5sxLgnbAmbos/YxEgnv/7T06szxpM+uoT1l6kVeD7z", - "aQFuw7hqpnUjkRsVB1bIrZjt5cMqT5bABU3Zt2/fGbVfW0s/5MXQOYe4E6rxjlZG407VxRdYhBdCilmS", - "a58Hv4Uw6vOPF/AcpEkwJMwE2JZCc3iQ5FJT0LUZj6vhWnrDtTn9DhcXmnortMZ2QEugkBpX551RAiIr", - "8LIBe5PkGMdYzQBrZDQYDPIENyN8NPUvY/+0TNiBUdAPz4zEyP4JkKCTO4gsgV2kzL7whj0xuzfxmI3R", - "Vt2ljAduwg4SmR+esQv4JzJmvUI4LbNRpGmidlnhXK4ldyQn7ACVzsMz9oY8CVc847mN9CSagaH6bUkl", - "RR+UpAmd6omtoMs7mMwklLPZLpCVwcFtxcwjL330csX+ikkxL4F+oSA6SZyxf+VSsFcKWXWY2OGTfbfG", - "S4oxgQLxGXtCPwB+9hl79v1JgxO9sgkqJft0/tPZ0ZHjQ7mXvtI7Qx50qsVmx7rw6JYIXZ5/d316tAKt", - "X9LbFMdh2htknuSLYgoSaK6UHGCv8DfV/kmxC7PCS0sJwco8XV0LfZ3I47nCyhX9lHbJKmnuuQtkPG8t", - "a7fJZ9w+I9RavxXzi+Hur3he/RLzHD585nNtPvyT4FHZNji83N9//Wqujfv7Pvv69dgUMDVG8utXT4Wj", - "rTKykhNdSlzp5iARwzw3fUq+xMGVZomzirro2SvGRnU0hT9mSSTO2B++fl2Zv7whvCvVE1goEOY2DsAt", - "T0NhCQzLG4zX6UsvfAupwchqzV4rnRm9DY/Q/f2LtWnc/stpaqVyF/FczFW2BkOmWCbFEhS8//X//F/s", - "I/7bytNe5amK194oj45ey5skUxLI6K88S0DcIgvI5KfX7y7eX4zPP16M//L63424b/i4USrNecLXIHZ+", - "AWU/fHz9/ry1LD7N+AVfnF++Hn/59NZqQqAtlUV9reL844XGqm/fnr87H//84fKzqYaPRNbp2dQnzYmU", - "IrAWG3n37PT02dNnhzjji6VRz8zp/5iJAWbYEDFzpjN7uv5dFWDKSqA8E3eJhlrOCKXZAdg4cMB9hk87", - "gPHAZWkWP+yzaYGg8JhoC7y4hW4YyCY2GzXeJpMhe00/uCoUU4nZiDUEJuZCjqQLs7EvBJUYjMm4HPOE", - "3sYhpFBBjA4BkYO6sVzl6GUwkhMK8lQZm1jb3YQsZpe5WOE6nQ7ZSwJTk/7SoKt6cF5gHXkDzoCSpNM7", - "ofuYZXpBP7CJk+wnYJX4BYIwq/d8ZWZn7Csb9dAUBE2850syB416Z+zX4XCIH10V/DgcDq/Y/WT7cwEc", - "VHwReLdmlvm6BwBr+oACjjVDQofh0JXyBmxKfsV7btRbrse4q2MYO4z4ZHjaZyfDJ+Z/nvaZGagpfm8f", - "FnDIr0QqcsFeiAW/SVRGZ/cLXvj17jatEPjW308wiwK0ydGBHMx7SIg4RtPBhyUlWfT34Ndg41cTlgpu", - "PeDdQaKc2oWENHEixumYgb/k2jGhd8k8Q3MrJs8jnAKqbC6lqeEeQA6BM4prUTu3luJBr3ZzQzsH1Ch5", - "xMrnEYGTTqgJ9gms2vELnkcLL0WEPwY1mxll1pq2EzkXmpIBuOQcX/EVqncG+RsHrvbg6UAveUrJ+Okd", - "qadWQvIETP2NV6OmX8JCBCz5FEvih1PMxTKRSa/fA4PKXa/fU8B2zR/YIf4BmRTAqxifCAH8wSxTr09P", - "a0G/Ysd2zRBLL4T6s49hFN6TpzbcxvnfQB5r3WeQT31SPTZXk2Dqm3B67CY4QQNmxShW/lCQDSUyTiKh", - "yRyNnm0TksUbzktUOJQaOpATvwSECUAnYE+VlnadYj3rNA3OtR1whWn6MIO7fxzYJTatuY+iN5WaQW6U", - "NMnzVAyEjBMucRufPsHkMV7U3v/x7A6VPLIu4FOywxVAMsBpN+K9nw7cNcLTkSSuMfn1dHjSZ0/M/zwd", - "nlxNTDcu+l/ToA9OaSxmuMGhgoPXVs/T1gXTQWKqrRhtCFAWBTuN5CVfCqbFkss8ibQZMkR/YXu4T1bz", - "MbJHZfUIOPg2gdDIBnWuYGxjj0g3uuJW1wUplVYNc/HCkEqq2rZUtv+SsnfovkJBfv/U2tbu66+k1bWo", - "D655NK58jmbU/jt4o6GMbzsE4PxVpTxPUhKGBomMeIbhzoxTi0N2kYPjIJhxlgp93oucSXEDQD5ZnkTJ", - "ilP65UzwOJECMWxq8ZTwlDOeQqyJTn4PeIFfGPbieY/oYkqhowi8EbhKnC8gTKFzSgzI8Dm2vvZNh1HM", - "AErfa53YpAmQSZGWCQHxO2eRsPPA1RB67OCqG6nTtjWBdYvgNByROFkeANy1nhXwFmwljkefnyncmpPy", - "VmXXIhtwGQ+adGez6gzZJ/eCjeHY4AVkoy00mijozd9inbhBhpMbWByioF+8B4qFTwfopwUv3bm3bh4R", - "wub1XcCol9OEIJfN92GbS/1mBLQd/DYc1n8nn40Kv/gIVesMCbevBP6unZbN9BsmzX7g+Ad25arVoTQ8", - "8rOvToJcZWLFaencCMwkimma6AX+45YnOUbt5pDjO4lTsVlgrLu47BjhiBDeMXO+NYNMpQLS8WVJLLSL", - "3sUEIuDWoiTBButrin8fyYoTx5m7lAdo6GTgRIWHZfLp9edPF6//ev52/G9fXn/690mfnilQ45OIomGR", - "zTBvfq3eqw8vv7x7/f7zBEIvtIurghNohz6SC3ULGJ3EicHm4LygBvzWtGq9GAkSifwAE5chsKNjaS0N", - "IXQ4cB78pclVGXlMF9HCCCsTAhqxn8EiPZKoVhGYqHOZR1DWqouhC6lOcmutCC3SJr+zXaYBdVrmAN/Q", - "pE7jr/v87zwBpI5dvObCMZllKVbmJQS+XBI9Ts3TS6frkWxSS8MZFP0zh+xjw/nT0NFIkm3JemDNMiHA", - "Ku6PyjmKwg1Xh1vaGA1ox4uwly8N6fO5uHSwihs5Qd1xnyAKW5EXKOZNFXmklkIzzfNEzyiHbWnEi2gU", - "Dt3gk1ilfI3AY4bXRSTAAVwX+YhUUfqma5aJm0TD4ph6yyQVOleykibDT7gF3W9O03lJLKnuzudl77Tx", - "tGpK7kwobnjXP7AEjBPtnfX+568ngz/xwezq6+kP938I5r03ixGSfHDHiN1g+oV1ibtu0xLXBZ/aSLqI", - "OrGYZzzGIQRyJrSg09rhIdCkl7w4E+ifx8G5HK4pfNpG+L6uo1oInuaLdXhQuPw2x+M2sswhtARujTS1", - "1GlfKQ2RIVoFweeh6W2F7iQIPtiSZNcbBASJwTAoZegOULoIHGyP5oey1QvXKKYbDRk2CCWxfX84CQyA", - "6UYpt7zdikUEp+g5CoAuA5Y3OQbE5cPD7ozsv4so6IBTdlg05GcfseompEy7KlVQHjxVZpGkKk8Wet14", - "a2XugQK0qXwhll0JGahva25JUD11pFYQLIr2HF4BY0cc3wDhm6Kwet1GVFfdcdFaztRWKg9yVjvpkjxL", - "PhfE/PT6s+fe40sbRWoNMnVb2OSXVgft+hOMEWRBHLGWGE0JaVBn0BBviTYStI0hvz24/Pj2/NVrllg/", - "Ovj5sNEWOhi58Dy/WTQH/vziZaXl4MtP6cRFvuM2LXgyK40LIjMdevft//o//28jPt0moOGhm5dFwGhK", - "sJjPt7RuOJD+J8/6rbizJRmvRMZWCl8X0kTnDNpjB9bmJtP1oR+v/MOzfbgFxW5C29tjQ15iMRdE1rMS", - "yHgvblPyGbcfTcojn49yx2b17TRyD2EcAUHa55VaoZfVnUasDsnnzpBs3zwxYWIrFRxU9h3CZ1eZmorD", - "IbuYS2WHURoioXt/q04fsFNxonOjR46XRqKOtq44FX+HpZ21aPteAztohA16+n4YoOWTSAXXZl1Ohk8c", - "pDryaA9u1k/WsRyyd0VewA1SZuQEtQEr+jsMm0OJtivgyAAQInJTlBy8GE+1soirpTKtncBlVGFyqCcj", - "l1NwXY/E1cYRX6G7WiL00HVMbf3ZbA4GuesSyHe6Jk+BVGjNYrFK1RqBYJAY3oIrmE8OO8dcu1zB3XbT", - "lva2kw5LJ5Avm1rMY/lG6jMqV5p4j7YuqNwrR3hMZFVUQiNaJybP9/0AGCeHAcDSQy+A/fDkaI1aQvfR", - "py1XLpLEGy2M48ByB8sXDMt9DsPm5NqMCGo0ayCCwwfhsizHpo+NiHwF8JTBUixVtkYF2/nAIxuzN8Pe", - "C7ZM5NgH6yCqaOaWQX5GUDgsX2RCL1RaYX90Ewtp1H/duL22vB12H7On7na8i+BEfCyreUJvgLvsGCsI", - "zM99vXHwiSjM2POCMTleVKF3tBxPc/h9eIQSwjFBSAeb3AYgk7yslDXWSC9w6NuV5ISx4ozXQmZJtADA", - "eoxXTBMQQShQMXbBz4ZtY0pwvUhWZASUqFl7jXRgb+VaDlACBrNB3jZ8248o4bWCw98ZfkhveilxtlGd", - "Z4IvrYBruMYcDuSQvebRws/iGKkMLgEufUOLPQZLAXKewy6DO+LAVTeT7Vvjz7VYHw7ZeVpeWLDjDngT", - "QOvhwFPjesUjwcqjhw7J8OuQvfe5FnquINfqI2n2Ha/tMxIG68QIrxuh/fnmd2Un9d8mgYBzjUoh4tnd", - "XWB1IyAvE0n/Oq2ZAPq9QiZ/KwR9xvPdmZZgxXe6QxvaEClAhzUNiHjG7mrPQ26A3ANxrM6gGZeKEXlW", - "u8f3WLjM97s/h+xSCLYlzpUC9sGLCa8QMtM5R6bezyJNVd8LTYQcU6Pev6qFBLdV8wfPF1x+/SpSLe7v", - "v359D27A5Lz6/2P/rgqQb75+PZ+L+3u2FmbaKmQz3mFp1Wp8XVMImxcJupKXIC6Z0EWaUzbmvMgkiij0", - "5NOqE+6vaNxvQKJ2bG1Q2j++ttyav9YVhqt7r6Ulif0DJ/b3zn791bFmTzy96vs/o0hX/c3RbPVnq9lW", - "f60PqvIxOJOrq6ax5DLneeACMT8nGvxk6hHs1rpsRTXLrA/DLhLWjcOh5uz+rNtrhtu/gcR26laKbOCe", - "BWx/5Ru2TXK/EDzLp4LnG0zkcOXd8ARsl6GcXp3NllyvZTS2SOnbIAr3RU6b8uh6lqTpGJ+iN0jcTmcx", - "M4TCACk3LZIU0xNk8OYCf6IgAn8ayc5KKsXqQczYjRWuwfGGDFYI/QTFWJndlFw4bDNdbJt7DM6l4Goq", - "WhSB4aW9qjlVexZ1azROfke6okeGPgGe0/2CTzc19bJqirb5311XRkrhgG16OjxxxE0DulXZtdlhlx8O", - "U1+lfM0S54UUBgzcV2NxKwd510KvqmQuB/wZcqWipNzsZVVCN0IhwtYcQI40S50YjonIc/SkbFt6bud3", - "a1Z8ybNrzeKEz6WCtuCTS9Y1LXLruw+fIkE5Y5O55OlDrkM4JONi5R3EvQ9K2RbqV2Mt/lYISpv+iDTv", - "+nHeNg+ePcL3fbMBL0R0DT+OcWfHOU9S19u4BUnOs0qj3XaQqjmztbRLoihcUvuyIwKHgqSLUgh7moaP", - "y32s+XmvVy58spVajNEkOSY7yriTExV4iL+HipdY7yO5SHW25ELf5JM0bn1/tDcRFD8mEd/KJtpfYv/J", - "nxzQKM6QaSF8FzaIrHvInYRjJ2L6xset0te3PSjYFYXFrDL1G4JVbN8c0MgXEBBO9m2wzAxou8qmwEqd", - "4wMS3MpFkgLIbSYiJaMkFfGQ/RvtGia7ZJHKMhHlRkee8TS1lznGKwMSuO8D7ygD7ds2f+n+aTnKh4fr", - "McRaByTe5HfnJWllQ/SJftzzHqtoO7Nyifiso2tpErFmyJtEQ5wfIdfCkJ+T3aPUV5d8bRFdSzB328Sq", - "dNPxsz0+4mRFPPdY82M27Oxl46yQ4IK6zWPU1fiEFUoOKrJMBaJ9Xpuf2VJoCM5PZiC4aBaBuELZD8kn", - "zAcl3eMBBB1MxvNMFSv9yCsFKcC/TdOLaTQGAK5vpeKYDuj9+Ft14SmArR7e1om7TO+twZrd9LJyQi2e", - "O8DXAG9FlTFhEa0guAQzXKLvaLbKRA6ZpCgcUj9EHEX7ZdhL81Vi/mXUlNxLdo3zQOq+4VnCkQ9YV+RS", - "R3kIqHXplNfpVeNdWdxLp/RNyFiq+NswKe/29cTXkKhRM1Am8wWIOAGRlZHvjE2IbsVW73ou+3pkbh6e", - "Txdvyo/e6CreN+5oca1VlMDDkwuCaIrkOzlUPnRipbfXxgl5TmHdJ/HNd6UtWfqrTfRCgHlniEjYr9iI", - "rAdWHyW9FU+ysbWdDh+29hiGtWNWQ/vg5OpbjZ8sAHkS6eEDUnt4mkgnPQp9fLzH2FK+IP1IxOOK8PeY", - "NOB6+HbMzHVhna2/XQdK5TCRR+4B4wBIcB5vEMRfOdteGbznydsW+eZxz7Eza3W6Hz/Z0iWZlae1Xckr", - "5ea+r2vbQw3GNLIjoopHauOQlWuCTuhoSrO6hRMjbH4zeHVyJlFryXTRAB6SxrUQK1TxHqzfIVvquH6m", - "qL94wNE2+L/vOoxxonXxbc5is4Ox0HmytNGfDx25jrisD7/uImcKMiij2QyckgEWAlNy0AUxMA0xHkEj", - "FsDL5hyn6+fRTxHOoFguebYeg0X5oUvydzAQUS/OmOq/Sj506N/W3mRf376NbJ4JrdLiAeJB2cAjywff", - "6pIic8YY9PaHbb/fkr1Y9hXTbGMU+fgoAyuzp+07KnQeMAItWTwDEu/HnWxpdLeHLWniLhJlOjdruX9k", - "DkZP913fBTZH9UECXJ6Kb3M6ibN8I4HQSA/f5PqEgJQxiSubzLBEMceVwBwn6jzutuOgDEPR2xMiQbHK", - "YOqes48+MrMbetNioe8ZyHze0Hw/sHbX2YcPspDXUt3KcSaWKv9G5H7L0/GUR9epmm98afTiX2TNu9Mu", - "zC/nbx9z/k08ImsCDMZLueG0hUpdSIDZst5v3vhjMUtkgsbOsh3tOT0IDymsfMlxiF3oFINxunxtc8wR", - "S83ULfDnlWEpqtDpOtCYdWVtC1cyHDDlqxYYFEaf0XMZfimnV41xOXloNFILLgyOAzDA/w6DENn4Nx2y", - "YV3aDYutu2wgNC44uL19BdC83Q4hgE4xXGuRV0H3wQkNfn6koXSOOnEBJ6VzvXuwiZMQS3xtA4bLF81q", - "5FfI7fvxNr0lVoRiTGc2ZMRoRmVqzHqGWucmXZ66FV+nij/I1jcr0hQyd+Plu4OzMVOScSLGACn0GSDp", - "ecwCsgskkl5HcyMifqdLgOMiTQdU5KGOxteJjLs/Pv7FlG5NDX4JAw3xPHBSD0J/4alpO+SAAIopx3R5", - "3m2l6oHHMFPH9ENUajEtAOISwOvLmDSLmOAad8b9hcjEcxZlAn7gqU1/QyeLsB8o/B5yXwfJDOBdPkB0", - "0Y6+7a3RMNVzS0CYtYuGknOWd2CkpC6WEI9Z5472wd18Jyp1d+DzliX1qyCFuyoP8tJudYB3YCch9/fy", - "7KyK/CH9kwsKBG9sGINGkgdolGOI9xhgFEgtO37APaQKesMgKl9ka+JcLp+gS54MaA6zGSXhsWQKHunT", - "EhkQYzj/ViQ3PIVwos+KRWo5NQJR1T/TQ1sG70IgH3ySwJZLeF4IdfFmVBvrkL22ljNwA13SP5Yg50dc", - "krvBMrkT8fPyOqGUWOktX2vEFeMNfJTHitQMpXwl7rdZyPwLMcha5B2xmQCvM236D9BwKHr9HpynCh5W", - "EOuqxdFjN6grqovOqu6l3D7meW67Hl9AJLgNUIHRtZBx095x7+DEbLaWcUiueJXoVcrXyMy/yOTOJXcx", - "WsUySdOEcsI8B8w3ZWiSID0w4xtnSyVVbhg2i1IVXXfFraDhwbQJ8wxdozrC+wXqN7FXd6u/5HcPHoPO", - "OZgudx4IIQy3b+SqjucG0Gz9npGgEjmnBoxufS1k8jvhvOGFCH/by7qB+hYieduphZ7bGX6x3oDDKelU", - "eR/T+C4uvZ0aU7GIxrEw/x0TvJDuXllqWPobgVh6bc8eL8uCjhfgqSdPMO1YBb0cMsQ1cVgBt4mM1W3j", - "3D190g6L+UBUTVedqH23yhiwsWfPWHnXfqVZ1hYEKnDV240yZjzn6XiPig63bRxxGYl039rEi3erDliS", - "e/PbRm23gcGb5ReepgO4EFhUww1NJF41/vXiaHypNAQM4wu566OOKdr1lmmMeqc7olF75xui0YIMafcp", - "X1FIErkwSSa5VH+PtXH8ufuqSKNtI0/jeXDzYXsBI9XteGWzHQyY5WiW8UGrbCrwRV+kydyIwkP2HyJT", - "Njgpx1KJnHedKNlM97wGXMjYmBiu3qFmR2e9/VsLu8rtghDYxVNt//FZhd0MVCc6F6E70LmSoP6y4FmM", - "mdCTKQgv6JGeaJt9FR+IdakGef5mvu+cjFt8A4dtucA3ur01k05b9MF97gJLxeFbyahN4+l6vOB6sVu7", - "5Bgzdo9+u9TNeZqOLfREKWr2+j2UScbkNmsUQCNvjkn6rAQPqxuRxYWoCJnux5CwCd2G3s0sYdhXD3Ak", - "WnDNpLJCksrgB3xHNYpqrksz0LGL1bN6TXjn9/FncM9YyN5lZ7ZAK1kiXzbHQ0U8Mac9wMbogPMM3GUa", - "iqMXppmuWVZISWyz3mUd35oEpubKBGTzVga75UR1ZpAdeNQO7HEXTtVVowjeElXRso1bhKTJdvUhcPt6", - "nKROWw1CKo9a7azXtMyaVtqv2xuC1oWARLFJj2xTEdvl3Fb5OcguWzhhm17Xptq0a0xhlaRNy9lgsdhg", - "DNlip9hmgmgVgFu1gY1C7zYtICzvNvhk0LZnA6bqoNBxF0svZPYXZZ5bOEwMUgHPMNBwnvElJrcG8JQq", - "Gyxvopa4rbdizqN1PZsu9gKZxSogK+eSPlGgV6gj+6kZqRHqwytCUH8WLItiNmxm7+CkiFvMcnN/h5A5", - "ivkcU6DSayHJ4bGASP+q8F4Ruq2ZYQO0ii8mtV9kloUhOLCN8Yd3DEgsQeEPqsgZZCmzRYnZdrnSYGeD", - "pFfJXdCwztRerxP0YyCwXgJwsCgCAEczHMnzGB5EXWqE87mQ+Sccax8CWdfuXy55WdnkAKCi0hIOkh79", - "QnmCqFbQT+1tonMzvLIMPNMWsit61Gtb8T1fihA+NAoe4zwr8sW2xn6Csp+h6H2/91sRb3dF+wmveOVB", - "beJbQqehU57hXhi3p9ubk99OCyNCDd2jiCnlAGxs1jaIo78Qip1p71/+whDMVEMKeDS6/8t1n2Ui4mlq", - "/pJxNP+Xa8A24nd0AE9Oth3HFTd3rYUlrIIZDr+vD+kywpQtPoqhaeHY3JwsFuSQT1JYA7mwHFjQF8HL", - "hkZXPbGZyrCeNhbqMz1G1BY+kTaHvd/1021rct/CFeiMBh/VZcxTVWUEHngr+/jh8jM7Nh9L3jySX7SY", - "FQSqLyhFYnk2LYcjOZ1x5CuhU0/Jt0Ne9BQ9XPqMHdvC/Uaeu3K69YP932ylA1vp9zBPd3OJfnJPkFgC", - "EFUJ350dWGYDhFDmAKlBlNayjQS4T5bMIRYGvh/D87rfUZssAPQxTuLA1l680gis1qAidlAZrOVNhwGq", - "KnvbmOLQI7Grh/HnT3Dxtmasqh1SKNtIrEN3fFBAQl9ZrxHfUusLR2GjiuGgnQjrEkuaOhhs0qkSFW3l", - "Ypeu/wBn10xlcy7BhWe6dsmsG+tTtQeGX707HnroF2FmAkfGrKoNa8eVYwcznuSLWZFKobW5/1IBSaP6", - "TOTR8LAXmLej02812k+1g+DGirdzn7krG6/p1rG27lpJAbU4lPk8E3PDR3SJqEfJRXiaely56TVAsPQw", - "1kDLlDcHPrc2uT2xaVtOl9Kf2b85Fty+aPbapJXdGqMaOyTLqPvCey3irdZVYCgvtcDiEg5g2Tr4oeoz", - "zLdcJyh9dGSICdgl8+/DIdF/bjj44dlIDkgiPGOfSDJkAzbz4OVsBcPMdcnbTU1HpWfsYyljNup7twEt", - "MiZPwnZNQ4bGz9h7QxvoD/gq0WBwETF7WSyLFPF/fuKJNMWXWXbG3gkuzZiTVaYinrJPXF7DR76ij5Yi", - "3dhwrd6+fTfgegBXfWi58APa/uwKEc84Yxfa3sluXShIZ/3P7OBWZdeawT3hX8ssV+rQNOSzIr8t3CFw", - "lGIkb/2zKW/tI1j+lRKuBmaY5HEM9mnsH2ss0MblN38r0nSACbuKTMRQUPOZyNd+KfPLMV+ZQ58lPBdQ", - "bCHSVWDABYii1CMgKdVLzHhE6QeowD+zg0ILTemLEPAaFiVK8F9jYtxn7DwT7ldIl28uTDMezwcLqRaY", - "FO1ur98zdGRk9ywDCX4F5k3aO0zp4pbfS+zi/klL1+v3cHUg44ubPxRxczX/qg087P1VvQoCIrcGB2BE", - "6cfcDuUhbzDhpch5zHO+Y/ShK1k27eX3M+0FeJJhhZtfDzyhBvkmOjc77TD0WlK+D202Xd2tUi6djQTN", - "NSR8NZ+Awmv7vlgKd7uyg5PB6eEDtMwg277jlxDy56Wv7AQs+4uYVire9+vGS75KxtdiHXIL5uz84wW7", - "FghNqUXOXv9v5+Pzjxfjv7z+dybkDbvhWVAXEDIee+gPQW/ci8sPhiTEMby+F6uVy/IE2rutDWXYLElz", - "QW6bzd7Qd3ocqyVPgr4LWMBBMVv3fy2YrdNdQQDspM39fZCQ/vBRO5XFckxNdclCiUUxwZtL9ukT4ZOt", - "JhjKWVkGrZCho8eLXPUaJhjEs16qWGCYBZkZzKxf33GPpVJ1KYoMXlKuxfpWZXHbo2uW70hKfsKw3Uip", - "efQaAMxNW6s5JziXYxcKcCumBPA9NMKAKZJoZtTDJcgdkGWjWbjPFsl8kSbzRa77BGRrpZoVn4uRtEBZ", - "0MKn858wo9/cPqbOUnWrhyh/XIq8WJ0dHY3k6ZBdJnPJihXjucNnF3d8yJORfDJkP4ncHXXYsJjrxVTx", - "LMamXqlInx0duapGwhra+r0wv4ryS5Xl7sVkhxybUyMY5WPML2DuvTZWTlEjWJ658gzLlw485ZdbrkcS", - "yw/iLLkRcsj+ItWtpKTv9tCO5ATCdQAJcpzyXCB8CmT6wCdAPenbUhBbNNYqy8cRlzE4o4/R4REKQd6L", - "MU9T73Ok0lSYtpNlktdKYa/BpjRiDmgxBwxAAKywnw0tjOQkTjT6pIh4vBSZ0aQWPIttMTTUle9A22aw", - "wwMU7Qe9a6Uc08fbY19Iyu8qYpqh6c+c+nIwLWUe/DRFlorqCxhmvDQdmMPpdV3CdaBNK2ih2kiSjbe+", - "OnFaskOoUTOGkaR0gu7imBRyyeFtGLZn0mcTqSSsCDzQ068jOfGXDTbRVYDACdpeMwkojv+UClcXsUJM", - "YeSM8AEmbnqBComEGItxVGRaZWOeJTlUqP3uOrC/Q/N5sTLNsAnI/HeA1uJRGb57Q7XyR2IB1l8GZ4JQ", - "eLJIU0oTBJVgSYzgNZ4KSBzuWizHh9/hiGNHAB1jVKaxymKRiXhs2KupAgFeMQSV4fDjRIObSWOhS/rE", - "o8cCJ88r3zh4oV0ICp341TLDWOQ8SVspr9AiG8w4JPIsicujP6w/ZBeSUu0RdDKmFDfCsATEYlNKkwZN", - "eX7NsUjVXPdZnvFI0BVlsW9GMhbTYs50kc3M10eab2tYZ5ZbxKUA+pzH8uElms8E5gtbKcwRWo6L8uGP", - "ed5hNFvOfXDFsc5zBmqp8wYkrwC6q3J+p6RariFZOP42Ge63fM4TkBjvsydPfB777MmTwNv4lgSpyPBL", - "rusYYNsCtZFty/a6YQefvmuRyLsFNL1XcuBErUbopSFpJcugOOaSmhoJPRfSqPTlW38tk35eSMEW6tZh", - "h6H/TezDfIEnQDC01ffzrElAVc/3mpJdslFG0ryNWYYaiC0Njl4skTA7aK+81crpBH2THyXXZc2RvR4D", - "h8PGdBe7jfdRRrfkd2P7aASsf8MgP756A8K3Zjz28o+acZKSM+C5WiaRF0XeusDbHFH2zvED5yTH6MgW", - "NyUntTW1no4+KLaDCxfQGVBFeb4YAL8VtjggW/DoGuCkXdU6pun54D/44PeTwZ/Gw8HV/wh6gbsBvINk", - "bIEp2gIu1HkzxEFzPpDT3Ex43Cm+3bbwAerYIHdq4SFjuJUiG1srnBfjbnXwr/ctBqpV9y4vobSptuAr", - "0WnCr3jOL01pO1WsamfaHBB8vhGZxUlq2jlAlm1poE6a1e21861uWXX5N9Ix7torEaUhpz5wyu268g+g", - "lgeQCSzvg5eVlq3retm4ZqvYAYXiSGy/OGxKAoY7ZrNP9vo9dxc6ZIIewXf3fIT7HsHg3QBfXEarca5U", - "ii6zYJIHu0fP4pvOChlRB3yVjK1Di/nnEz62ZXmRL8Yu37u1mYBhzN3345zra7yUwapGDtqQqOwuH3PJ", - "0/XvMCj4N8WyisyvwmO+yuGnSEkJkAB2ks5B+WoTi7u0R7nmnLULnUEblsyqFFF/HfBTNJs+IO+EzQjW", - "gXw20k05Do9sorTQuELQjwvOEPE4nraujVFZLtRLXFPiKXXzUXQtQkbSF/gBZYrItVBLET5T2XAnw6wj", - "tKZnR6FztXSuVezLp7e4wGV3QVfYhdKh8f9sfjZ36ZRrYRprTgWjEoxItdscVpmYJXfNLv8i1gy/dRr4", - "KlO5ilTahUBxIz/aGnWick21EVatfsDmT/rtxQejrkpt1HBmm/UNU/qp4UkRJidMhV7rXCx7/d4iz1dt", - "dEgPzy9TrnUy8/CyNz2WVek05dOWWPr229S+P21zbwhjVmCPLQsanNEl8vHGGVusVypfCJ3ocRVuxV6S", - "nwE6BnVOc8C+3g8bLwbv316wsqEqKAsQthl27FTjIfuiBZt8vZ8YJTlfiJF0BqMBTI1BerOFSmORDZlR", - "/EwXlCNcZZolkOMfG8eszr75ulxnaA0m2v0IQZaZsdvUDvhO+MUmBc0XAqz/Rp/IOEBw4jBYsTIq0gSy", - "j07YAShI65G0+WQJDQZT01JjGKOIi7IUAnw4J+Q8OHTvppPK9L2301YCDGRA7beoTTR4HJB98oBnZ9qs", - "qEJpQ3YBmxOPJHCaibeeE5sEqW+4UCZCU8FQTJUSZbhutchxlltca9uOi95yXl6DCh/QRCqAJ5WA5rZj", - "3/l007NYuH3rcbv5zsZRUOnNM7yQ5DwacO7FW28Tq3+54Pk7tBq9pBr3/V4SRoPp6nXQdAw2Ilj1wG65", - "f2hyn03FrU6gdq6bFwql490ugeoh2GcGtSsoKKDkie1vx8aJukNv4HFLsHG690w+WQ0h0J3zKtI70sb9", - "5i0r/am779ks5flYiiwcFWudA0BUim2gbns5veJSh4tkgsci22UBTXnP8dupTzs0gTW8RipBGB0Ejw3L", - "XR3gTotuVLoiCzPOdlQiq44FP4ab2zaDdA9RbxfO7rDBdzs0r60W4gLFH9jGetXBctAOtdve+E4rR9HO", - "DjYzgDcT772JpXTbFrlJUKAytni9JNVESuo844lkluE5LU8P2bkt7RD4uMW7a5j6y/3ehFVX2dRtBWnn", - "NhbbZx9dmFF1BhDRsQ+/R8ECrfEXWPX0yY8gqdl/BsRse9a3rELXSJnGVQCCmKWLblWJjuqLasPSaYFc", - "u9vWucqB26kTXN8HGFBjs4mByxBRZCziYmUz3TQpr/wsxpVr28n2lQvVu5eWiRyXMATeLVeG6D0kyM40", - "bwf0TTrQyTJJwYmhNcLwx+8fFCaY8eh6bC4fNCh2WlXzy3hZ6HwMnkAd6mzjcnqlpA7YEK2IveN5/dAa", - "h9d+ASsnE7usiq69oG3FZandTcbzz52LzrIHEOa7+dSVV0F3b+dLkYoo1/ASWM7KCwyf8WWSriHdrA0B", - "ZyhR23tkydcjySMAVJXr8ujS4jOtvAum7CLi0mr6Sa5HcsWzPImSFcdgTZL3LeY+uBtkRA3BOM1HVkIs", - "P+yoinS6z8o2HkPFaB9hF0Wj871g22p01007cfXfWB8YJQU5fdeXaTehyhzF7jtQXshPmgtmxQ1nTc0z", - "cOcgD9s00fDsAsWvunjchpeg2wGtKWrgpfGI+4j7gCviN3S6jSXRSDbzILRFNN8z1F3IEumJwKX89Mzf", - "rGeBzepkHGo1Cr2BCLgPKy/IspuLQau9QPltOU/RlRbg9RmLVOSiE9n0e29E7sdHbHPb/vLprUu1MDNV", - "EzkHz+kvWli49+90tQhBcsfqVhKkGEhc5L0C6QIv0d9PQ0DXz58/fzw2/3OJzzXgnqZFVBjZw+JWQ8CY", - "KfvuLfmZHBCr18z5jwI0+U3C2VyRUym4IkNg08dXbxg8XHgVTXn4eLHEJvG60IxriMNhXz5dYFjUxxTy", - "GJr2oRHz2+VTHO+BU2D007EHY39o1unSzmMmODBM67gKUz9PU3UrYrZQOme3iyQXhhNAh1lyw3PBLj6y", - "aaqia7OKB5eXn95A9hOEGYeRvaJlxowh4CgNo7OwCtbiG7jROPY+bnlJ+8WOh6kZ495IJV8SlK1HEhcz", - "JparfN2HMFNokuDHseKBuIvEKmcrNzF9OASn/+UqFWfs11HPOsybH4aRWo56fTbq8VVS/fFqJHd6toP1", - "G1O/42QVEt9rb6AAdmrRv9D1ym1HxiXQSn0zhiMJ9UR8xk6f/HF4Yv7f8Y99dnri/f3HJ8PTH+Bfp0/6", - "7PRP5p8/4r9/CL0zdHB1ghcScteyOVRSuqi8iX5/cnJy0vYOYQ8w1oPIxwU3h0Rkmh3kWSEjxI6fOXi+", - "w2BYLvqMIUlCghvfQc+BpTz78fs//tA+mrhC04kkl70DauGMnZ6cvHsRHkD1DG41vHtFd0YucUfP4qlX", - "MEu6BBrDVfmSr4hRBfzEIAsrufBGrqBN/o1xKDbrBsyJLflqBdjzhumqKXklx2vJl64l+4QQQKO3bhtN", - "j0r6Ag+OhHtylx+XSfbI37OPj+DwyB61+Eb0e86vULeYl2xGWTstjAe3SYDwSd+tR7c+BTomtnh3uEyA", - "wvovwlK52eI4Bnbl6S7wZhwnM4C0y8vEOamaQ5Mrni+CQ2rx236pZJSJXBAaFY7ENEKuJ1LdyiH7iH6B", - "GNVsN9g+EGPmELVMclis9t7Hzr2wsQXY5MC9OYPKzah8tzVHXx5wZ4YgghYf9TjJrJ+JLHPysLI2gwAJ", - "G61kX6ZZqdOJShgN1xG9pAf1Z29UK6WTcBb3/xCZGky5FjGzhR44Ot+hvAVRdEk4fXCGW1JhhfelUqnb", - "5gSiRUL+OQjsTIWPTWGHvOd7iHsOA6HOwqfuHQ6aGYGxPi1To9tEzNHYgY6h+B7EXLdVVZapZM42w4wu", - "ooURJW1oSJ/VuF7fztn5hyCGHDHtMX0dtgIJAahnQHQDWDrKD1MGbAHAILpCIXdBKr3hWcIhlQ147WMt", - "I7r5l8xI0i1DKcLWK2gaJwgBfpM+m1CQqvkz5rkw96L5ey4USCsu+o6CZrker1WBEVHPzcHCLE9S5UyL", - "Fc8AGYXYv5rPUxsu42Q/e9xd6jAMBgG7Fshp4Eep0KNRWYiBIEtomDgMc0iTmYjWUSoA6Da46xDJ4BQ0", - "njJXx8sO4wXRFVoM2SsRpTwzFbNqYnRc+DJXj1HDkkznlI7dD50ptHjOXASarb7gN359iVgdNl3sc8aj", - "SKQUY+H1WLYDqlrJtyzH83z/6myvJXwR+C9O0/p0wl8wFvjLdQrgp25c4RBniufbBIHQJO1EMzsGZluw", - "M3K5nkbSE6zAGjpk5yxSaVxWds01t2YkcXfhehtP17gxUZpABDLFt0ghAEZypdI6hRS6zcHIcv9NAiye", - "ys+mZPC5qsoovHWscLPwVdByCK7aBFni5Z+Dl9Ybj2+YzV4RR8KQJxGzyd2Aw2QGGJeJ4UE11j2SVrwd", - "sn+9/PB+QFKZysxyi5jxNOGa+JcsIWxy5YwVIwnAIXqlEIic8k3D0M4IOeiYbmRTTyJuRZ+ZnYFsYLhD", - "/ZF00fQ650vwP7Ncr8/mQo3RoTVXzDJAjAU0n8ATnOVqJOdCwT+GFUQVYmaLfOkH/wPAgRkN/jUV6C5t", - "o2JKpXGKPuB2OPRnDzUcGC1yRuv87UaLf1s/dTfSSl4tUKmnYLaEzFtNbh48wkABlw7oKhge6ECwpkaf", - "4swhNTi5og6DNW9Jo+oQsCoqpGHGcN9NUzEgZRfZ4DCc+4dncSJ5WDnzY8xkPdMxhcof3CSc/bxeieyt", - "mr9V88POcuAdhuK2h1yV+KJEFMeE3IDToTfCtkYIHnh7I8ARWjIelchdkJsXLHcgH9Lsm2Lh1qmH1eQ0", - "F9nlSkSBTSCoCkcqaLzMFQhza/MH0qdN3t4LGsDbGBaof7mynaigJyxe/6Fg/zdUjQqA0U/87Yy9/lvB", - "U7DTSXHG3qucCffLPD+e5+KM/QQht2YBuQRsGSgCRr80P05NkbdC69D3SMmcJ1KfsZf0F9PFFAdMWGGz", - "5O6MXeY8y3HfAFrLiExn7IXIb4XRZG5VScZAJ3qhijRmU8FATmK/LhPZZ0t+dwXdJvKM/RUKJpKB5TBc", - "77DC6sTfAOfEsBtgP8CnAFoohT/tZHrOA78U7pLw+2gL0dNu4JgOsHifjm7f8XVIIQvTA5hM09NxIt0e", - "6sPww0XPowM7guBVqdJU3Rary1ysuhviLTov8Ec2gzYGxYppBPwGbK6KUQgcfFUFVhRddBYgdsiYZ7G1", - "HJXNOetBGKnWMQFrE+u3MoSyTbCiwgDhJsSoJt9R4Nk2jB0v6VcsVpkAW2TYdPuSsm6SJnab5NECNBum", - "c7FikKoVGvOUddwb821lVDWZP7d2E6plLnYEOGBJG0B5YJ8zkczlS5UWy8AjUFgdxtLIdUhOnWE7ZZyR", - "h82xxGjxpq5dpGkpLW/xsXcAYhF2DhZ7zd5/efuWjv9wo3ganAE8n5gSEJCAju0zd/zoRQcJcr0yXYss", - "SpQcjuRLtVwaBg6AimBm6zOihT6bGnrO+wycTbwj68SaPiuKJO6z37SS0xCKy9ZALfJrb3X2on29dC5p", - "9UNi5r/RI4lWyOXKL9OCVfeaXcxs+to+1QLBdiSNNjuIE026FJ74SSLxdk0A7MPIxUOqNSkVSQ+HupNb", - "QJWMA4pyrEMmF7P95BgIxHzgxV/hwh82Zm1oZsrB+8I+ELLJH75qYVS6s2uxBsvt/YRlgnKHEIIlefCW", - "dldrEYeamI7aMEMm5E2SKQl2ZSsGNiA4XI+r+TjWMhjIulI6n2dCj1uU0vdmzoSUh1qnETwTcVs/1R+x", - "oct/e+um7zAs6wOLIFJOZLq3waGxBodqDheNw19ml1n4pUspBFbkkZvZqGcYotPmq9qJLVTFJXK/dnKk", - "RMppLGXwyBVp+lnc5ecUCHshY3FXHsAdcC5MKwVPG0m8KcM2aKS8ngC+j4Itl0xVfV4HKYCdlkmIAgqK", - "jd1tA0ORRCle/vrcDrKSIZyGOmTnftZ3zZaFhuzPpHeYoRKNuez1WzxXWgRgx7DQ+KHBkasULxLthvad", - "ZpmIVBbXkm73y2EMUnEj0tKOksiFyJJcxM/xIp4qs8KZKKvSGUbDEULC2AT9sDXVVd/Fj9ftySZaAxor", - "5bPwrrY8JV1iBBfxv0jJGyETw6/MGVwO2buC0FgBk1EnN4KcHqCCfu5bLqxLGuZophJD9sn6HKA1zlmw", - "EqGHbnRU+s9GUMLU4ZoQd5DSwcycpZhUbZWqNcDDP1ZK8E2vW+gtgsRA9k+O/hhD9gsdtpLwyZ6OeQPJ", - "nxA4GjRAFC94tCjrID0+d4kYlBR9luSuOi9xSFA3ZZcwO6vwAGKgueIARB9njlYlGu9z+yDSB40XLTtS", - "SeQb1Aq9VzKpmGFz4PvyAURLRQA6DyDrHYBcxBJwvDYYTxWQ11IsVUY5xAnxai+XBOqX6C9AAWFWhxxO", - "O5LI1RyG95xCNWu7a8iZSz+fvce5lwB+MmTnVV6NJGOhPglpasGdXbvCp1BA4R1Z7zc/lN3ktQ13ZcUL", - "7odnm4II+j2cFH1GVWtP1B1TkUy7S+J8A8f5eme//uoIpV/jrFdXdY58mXP0VaoxZL2WEQbAUB7OHfCY", - "Td1FpqQqLOllziocJ3wuFRglh739U1tMeXQ9S9J0jEneNgM5I2cwxA2F0zXLxLRI0hi4DeYuhD9VxiKO", - "HlisWA0fclTdADHZncuBuMnmVyYUsQc2LoCH2lzANOwuuX33GKlNNhiw85RcgOeUKh8VTi+BNTeXgMPG", - "B6HmZHhiOOHp8MTwDQCPo82waWrNJ386sSrwMaUa3bDHbFoeGD94L4tmQIk0jAFfFsv3ZR6v+zbXUN9l", - "YOuzWMwzHhtpyqjf8M7SfEzuPlggtnGx8qh4b4Ir29onifIe/WA6zIbD4B4t7ZNidoduOqUp3XQs0Qw4", - "SNWc2VqaQc5AFDcoQ7eXaZkuuSRN8aUSOx4+7sGNE309LsJ4tJfJ705PJfZHTzaPPAYVdVhAx9cAzjgV", - "mLTaU6gecUQinu+S2XmHhsPww6/9dJYsmQEv0SwCA71UIM84PO0H8ArMLCHi8TxTxUo/8uRmmdCLb9N0", - "IiOeSR72Cfuw4kbJtqCUlmRjoRMnQjKvhSF7SY4AS455/kGFoGx3NjnStMhRljTrD6NZZSKH/NcWyvwh", - "bBul0RZfssT8C9L+qcxZ5HAeSBnWPcgz/5TuNlcPUEqSVOhcye0xJiD4vSuLe55r34QCpIq/zZHsmCS7", - "iWmFiDdBzs6sDZjSx1ruXvblMfpH5l2dE3WHUjG40QXT+gfxips3Vx0e82TwJz6YXX09/eH+Dw84MVuT", - "lG+aUFlsh0l8810pEY9rDGATvZCD0hmL8JnF10MqsmUmVjzJxtbKNnzY2i9VF/SImhZnVXNXn6SYx1Lh", - "yvQambhJLFLnI+6bE+878cNPtvQlbiw0YHeno34ZOeO/V/UBSiRSQcfhm6L+2IGALFE9SLWgthKti2/D", - "x5sdjIXOwRHoUUYOCStqw69DUZqCDMpoNoMsLtO1Q5mn8zgwDTEeQSNGMXf+OFUr4+MvDaWSIAD2By6J", - "0YG6X5J06t39SDUrMrw17zz+5M1YnaJYPjq0J0uHwYLwB9VYsWJTMUMAPu+oYoo5OrDDBx5SVCobSmyD", - "xCrr2Fg8HHUGdvdcGb3BG97jrirmkvgmwh68HRcPuG/KBh75wqFlHoOu87Az5Ldkb5h972fbGPnKPMrA", - "SnyffUelc56Kb0MhdFK+0a2fi2y53SThXkREttQUq2KzkD7ycQO0ZkjMstuB8IyupvJjW9AhndTYhhF0", - "tEqXTOtx16iQEII3zsRS5d+E7GpP1p4KH360/v13JzpyuSb4i0YCkzKvXRCbp3S5+1rNQ9f0sWykfPu8", - "EGxmh2HtIujGwl5yCb6f0jpyGZVhBE2PekM7gfW/2dzodWgHpbfitL2AQv6bc9PNwV+kzU9otqADffYC", - "qjcs6vffb1lWOO1doCWyZXCfy8Tau3mPmr3w1FKdi9WQAfow+Ddap0z7qi55OpLW8QCfRkJp5AttZEpM", - "JMS1koD0oG3O3KAH6YInMvQczBNILDq3yfzBVzTP1pDrJAOk4janie9cpe86P5JCh28TeR2G3/kv7m3a", - "91K4j2m1W3HV50USc3I98WkgB48IjL5h8OiNLgw6XyPMvHPregEuLFGiCeNPRAuZRByyY2EOtkjFwuJY", - "a0IfthmZg8GLbsu2bdRPtqDLzNpD5PGxUfNX7dPGUgxLwdy51LfCX4JeJ8CU+ggqSWVLpKGNk1Bqnorm", - "VDbX+qvIcnG3a60PacqXfNda6Gu1c18rIc8vGrWumulzfYTRTtttU7WCaYDnAjNAbqv+iefirSn40qai", - "6Aie0YTVx96vmvfcOStkMjMKZdRgs9xR141g5xfMtlNz5PwK0T0WjfnJybMfHThcb77KB8+Gpz0flbWn", - "VkLyBICol/AeXGSid3Yy/OO9T6AfvUWujtkVcSPSLDFjWWLQ3HRdQvl4LMJTkq00MBfmfuv1ezdAnb1+", - "TwHBYfADDhJdPYJxGT8J9UIVEHf3Qt25+74+WjXP+GqRRJie1lwrU3VH7uPIXikcFsMaMJYOEsfpiKc8", - "YynPk9xwJsOwUiXn+C9oDq4WDP4us7BT45jRcTgXajydqrshe+vqEuwLPONkSuNVyWGBsiROjISzphik", - "NVyE7FboXGTyeJlIbwzw1JpoNvejeyAelVNxflcWf27Dh81u9dkykeNUSXb6xz8Nv4fJGUoyvwzwJ5iX", - "0AxBVpkUPGP/43hw+uMJ2GuF0GXbQ/bWLhMtjPVnonSPeL9Cnzxno+Lk5Gn0Z+yRB6/3PSW1UNwVJMPK", - "FzaSyaxoGVHZiErwIA1wdAEFQWX5QmSypA2YtJErcNKVDbGvc0AruABhvw8n/P3JzxQ9+FMQshP3KvAm", - "i1tfp9Qhe8fXZmipC/OyBIAXWkl8g9sMkqCag1LDmmgO9fTHyljhn0F80eBCXqoiuJAPXx6cW8BYJlqW", - "52JW3TVaYHQ7NiwDTqpontWHL1BLBJhdtXI2JUmWBBAW7R1r/KjS9VzJb6wOrbAXPFLdYSJ/EupjFf66", - "JR1Arf2WOb9KdG7E00eZbEyNYZI7K7+enlwvN2LvNDObqKjlme9XIK+U51e+qrHNHazid/lkI1RkPQOF", - "HYk3uZaV/Ghxw+sJdPKOQ0xrtq22kmEpWV3aLGt1eeknof718sN7uqaxDmmfCYB+G1EERapMkE7UgsYc", - "KQXx2HkVHPW+HXCzUyCK3+zVhtn9JNRS5CE6zTzEeysrgYSpRQSo1l4QK/qrh1Eu7BpuOYG41vXZ2Eh9", - "N5hNc/nG3GXuLVWXybilrU/KNRScTKqmPL3MeS5mRQpT8hDYSz1t0wiCtZsqTOdALxe9tWNUSlv4U0AD", - "0TRkNof5WzPbZxgDYOoT0om24PoAVwLmh0wVuQCbG6qlrzGdXLabMQnrGh3n4CdQCQ6Zw4fw1Z6RPP94", - "wa7FGmA5J3yVjK/FemLd8DM2+en1u4v3F+Pzjxfjv7z+90kwRA8aOjoirEj2zuhK+uzoiKE6MnBdD05O", - "Th1MYJ89PfnjExYnS4TiPDp6pSKotsjzlT47PubJcA4TGcbi5pga46vkOFaRPnat6qryRpPonfXWqshM", - "+YH5p+HQS8zk1zszPXtKXWCYVQ2P9Kr7UEIP6OtrwNxqNwFXGM2sPNXwFKcFIqFWlze8ugFnjdssycUH", - "CBlx3ptuchVQRDPP5tBcaXs2Suq4gTSP7OCPP/zYZ6ffP/0BHEhgqzIRqeVSyFjEh2E8jvab2VuQl6kq", - "jNpHUBAHYjgf9tl3hR5EQuYZT0+/O3TBGhjlgcYWdn7RL2P8zO9I3GaJw9qGhXQvc7q17HVzsNLjHOXq", - "QIsUjlMN9m5pOARSBoldk7jDIlFhdvGKHbRMvF/ykMpKHbZCpFkjRM1oELrwyKzMtydDtNP+ZGuUFkFK", - "UdOc6pdPb+0Ke2fFZZp0M+6j8cDGYSUzhCBDQBGixfJlJ0u2Bri6VbAkEry8gmbB3Rhw2N7kmHJISd+F", - "pXTiDI93JtsU+jJxWRgJpwTfwZI+0ESYj7Qf3CfD7wezlOvF1kPrrf3WU+u3+lgn9jHOX8WgGIrThGSB", - "GZexWkrKPe9ZCA9OhieDJ8OT6glpZAJ5si0TSJkwsTJ3tRpcM22WEsQKnvGlyDEneuBpTa3Gq9DDcJSK", - "Qrc0s3/2kj2ZzjdiJRlfLc7n80zMeS7+atFvNgbG13PCB4PUz9Nbvsa0kkP2GnDuuO2GiTsRFZTDI0kN", - "behc8BjgoABIHe2ggCbICRPKMyabRq9Cr1st6D2vRJQseUrBuEwrBm4TRsZdcrCxap0Sjf4rv+GXUDfg", - "IRzMm19bdhyDRZPcvub6k9DAR3aCI3hJiJQEEOmWVuOjK2cRlwoe2xiYxNm7888vfy71ixpnd9X3zeYQ", - "oqP7fjPdMUZCA4mbGmW6dV3CBJA/ehVsryQeysxSC3fmd/5gKUR1YxIJL7iUVmM9MFx60C0/VW38QAc2", - "e3gQryEOBkzg7mRAA2joqKARlxsT1PptTOvWgcIJRErDQNiWJPVeh7b5q/1dbxqUbnZuR/7ySLT50px4", - "N5QNtNmgS8mXdQI0y6j/q5BdHaMiRDJ15pPOxTTjSfQx5RD66T1Rd+dBDuaD2+bYitork1aA8wDQeJ7x", - "G5Fpng4Dzz+FUd3GrkinVXiBtT7bSlauv2+fdsJ1jRB2I0bnbrfTFqEJNpFRvh1WC4ZHXB34xgBivcDI", - "SSEuidD2SrXtGq1dIxwB4zb/BDiYApDItIjnIodnP3Ph4vPcBLZlbJsYY6GxzdIw2XYrw1q0YJpVs/sW", - "ctNFaDYFgpWRd+7CGkzVffh05bxAI+3Dk9FCZYj/Z1EaIA7htY2S3EVWsnWcgA1bwKGPMb0k26QwlJUz", - "xPcpCLMlRXBO2MhlPzCYoMRUBvdQ0WdPnlz1m65qW1J6UzM2drQcoT+e9jUmj+iX6JG0FzN6VQFl0WzJ", - "Y1F6uoJeB1yIUIC9bAf5IlPFfMEm41hFQxDfJoF0jioalwmv9kuudr9tAV7Hc0FYyHstwju+0jU4GzNS", - "wDSVTMRzBCvus1uRzBcEKkweEe2ZPbqmDa+OBV5ubFW33CxSqwSMyrnCkZZACyKGIQ7ZpZA6AfX42t7K", - "yqbWMNJtBE+3JcBR0zu4IwY2LP5nGpoTWnFx9qm7dYPfq/hbbTCSt1SxDTIG6SWwm00DhnXM9ARP/6c7", - "owLxNGwG6JQLea+1+ogQSNleC3UOsECEopSxQhNCWwkONZjy6FrEtG4uQD+YhX/c8ihoFoc02e3p7qjg", - "R4RI97SFMvWIFnlwlS1Y2BZuY6c7/k3vGpTyS5bkAuGb3KJV3NSesySnUHR4j2ReriQLRiTFDUQmGVmh", - "EsrpRN+mDbBbQu4gYSA2kSf1BcA4Sa2hXq660lyl6T0vIswcHKA8t8JDhi5LBGIEcCyQ7rSKMPa85J5l", - "SRmLjHH2M5dxKqY8065Qk4rrOSStz4mtsRkJeedE15swjCur/YBVPq/BMFZukUQ6fmguFCMtG5Zp0407", - "zqkx6x2G+J3+wN4lL4CUT/snJyfm/8rssHDhM61Yaq4lAAjvU6xkn+lVmtBlmgnADCVR3Owl6TaMxwA9", - "gHHrkEawRAWr8qJyk2Fhd8ME4+BB3QUUrB1ucgub8RzHO5/ZqmhH6CU7tdCUje5LOdW/zcpEvi7NtY8Z", - "Hv5a/gp70CJ1Q76+MQCvdBYvzLgxywbiQ+idZt0UGChp0QbUzUypHH0g0aRmYy2QulZ8nSoel6k02eQP", - "kz6LVT6wqXNidn758uLCy2PmZez5w9BlCMasPBV4U8Qa5ymb/Hp0NQnWMh9qttWD0egP//to9IfRaPjr", - "+eA/+OD3k8Gfxlf/46D+w+HRwWj062h0NBpdHf7z4T9vN8duhOxEM0ICFkaz0rsaQKS1v3piV762pkkr", - "rnIyvk6xoyF7qeRvhQQkLPujOZ1pMoWHp3Rtk7+Iu5XSorScmCOcL/rMenT1rbfrGcsXpqTFoKRcSIjE", - "Z9oTs0TSO+w0M/qmiG0GK03uwKbUSkhw+CgN9BhDE1KHUD7cLaW34Y1lIBDGtlBa07ELDyqZGQTh2VUV", - "dwmkBsWAl1UizYS05Cu9UPlzpvKFyG4T7XSFoFYQfL8kW4odFb1fblPtwm49H27BTIImX3TtB1/SAf5C", - "2/28VGjIDow5JuISrL3afXlYfv2fbDTKR6NsNJJX+6BlBAMXzLpsOyH7vVSc1zIQ2PsQw+OYmnnHxR2G", - "Do8Y7ELqlYgQaUQPy0yjuXI9QmI9CJcXkKQE7/Jbrpm4W/BCmxPYpO1Ht+LbaYUdGdTtDj60ph/chk/q", - "tuoeeoppWpsZ17u+EXR4HoDBPtbDQElWezwLuDXdaemqVtkHQJpaF0a9If8oWaJyxRbrGDwKfJzXENOz", - "AX3KfoeUyGWaoe6pixvtb7c8/4yD9MI0Zxj3Q4dTKjmAFD3ulFJooiDe3MKah+ycxVzO4Q2dtG2HgrZM", - "INUZ9kgY4Di6kh8nmnF8TbacHSN5QIuCHNMTkOMn7s4o3ToCvMXcdHi5RkLE7BQk/SH7XDO2u/LmMqUN", - "BIna+fyqzMrtnmT93AY/UVIjUKktaj5dAnZlSFBHKckljfbM7jpJCWLfTBzzoXtGTbduwVBPF/jmpU8+", - "8f0UkF9sSqBSYwIBPla/KcJPMjveGO45KeW3qEDnikVqOU08far2qMTKpHRNjp7yW1//LYuOtVgmcIau", - "tgmSpo3WeTcflUISBryEHBwdAvwu/A2PD4dIz+4tB7mmlXeAgECQMGXJc6HPlhwyu7tKru1EQpoU0P0w", - "AybI6LkLyvOLApPQgEM+EHc5OFCWwrzh9s8rI7WOEzAonw9kFJ+Gg0y04xZmMzoGu8IdpG4bL7Vd1KXA", - "y979ld2dV7RIL+CF6zU9cO3zgFPzZa3hIeJO2AdYxHUhowPJG6V3g0v86QsK7i2ufOnrlY+HiPRLadFD", - "okTL81LbC19YybXsoc2bEIIt8H0xEklagkP5k7Tz7oQwsuVBy61U2IfecgJXjN2CKiSqD6T+rpTjRAgC", - "nA2uTtA3LhNLESctgyDBsNAiG8x4BLecHzqfibiAX2sjgt1s6e6/6hOevxu+Z3dJN9XFamWYVkh6oVQK", - "RtAXCuG+djiMYTPIpzcv2Q9/OjmFLJ/sIwqeFsMLjHPxwIkWzrOsTdv6n8e9TbbRLU/fMMKt5tDGWuBr", - "864CsVLp2IXVbOWY4Q1o3Phlo1uH/4rn4hOXc9FpK2tXfs5SwTVk28C0GTzLIZbWpjkgR3Jzlwk5U1mE", - "rBTTO1DGl9JqAUICN1riCwxQNkKcoYynT5/+CSQsTjg/Lny9kDqZG7n1i0zuBpJLpUWkZMxiteSJ7KNO", - "evqnP54MTk4HJ6efT07O4P//h3tBfvL9988GJ38cPDn9/OTp2dNnZ0+fDv948qfvvz/94fT7/8DrVhuV", - "E+Vcl7mGQtf6NG0H6EsSqlkDBOFWs5kWuUvy4u4PqWphS/RD7Xxnee/q3tyngc9CxubjVSOQRKAyXAb3", - "8VwMKDXsnrYHUk/MUoyF1bWrqsnuwGDNpnHGfuNok9q/7f88juPm8pj7EOJW3Y/5XlzKjHqMOTh34VJV", - "3lIfuNfo1uGXo66bqYzEqiOFLsK1XVtlIk4icF3GqDeQqLn36D5kFzkhSRA2BEAOebc9tIgmddcc2l4r", - "pl8AKYJE+pKn69+FUzLLDDVRyrEnsq4DdkOfrRYZN4rzskjzZEw/mvYxkTdSx5ABYpY/BPvOxGUJA+og", - "r4YM1t0vX2hhc/7iqvfB7G//VhkrN4QhvECmcSTIrnEtKJuq0e7jOIOcIfRQGj5QiWQTQ6KTIQNxf5Cr", - "Aaob3timIlVybsrCrr8zizAE/WRHRcTRi1ktIppOqoit+Flky33qfQS76D41P4m5uFvtU/OXJI0jnsX7", - "1H2PlOCzhJ3Xae/KdWa0U2UgjvM03bvueyX36vgi1vtUq8uHO1XGOi+c2rBzVfdopfaq/irRZfWrMFe2", - "o9tNdrys8Wq610uuMGQTI1KheXCCXpYTeignsZI46nM2wYTTE2TlYJwsB24EPaEXKo2fY1um2bFU+YRJ", - "ir5IIMEclRJxpTbXIN4C5uOQfV4UmvGyjeEykRMj+773roDSCAoWF4TvtLLye4wW4jk2+Z12FwPwfhIy", - "as7o9Qzm9jbccTtBaS90vkfVKiH17Pz3aKlCU/0ebt1D27nvJkH4s9jVhxxr7vh0EVj/js8XTYdsO4CO", - "0pK/Prvay/Tfea7wMRDwiVZu72XADQ2PEChdeG4w0eIMjLTyBp0+wN8dXFgG2nmn+odbiyWXeRLp5xBB", - "nysGGcGSJbzgSwIOS+R8gIfU8h589/gC7lwlP+lTBBQFebVxFPC5m8PDRIW51K1uVbTRH55tBBvdV30o", - "93o7WXlC1W4EtScU63+a3vafBgLbt8ancsW2bkspl+y2KUnNKT5kVK4+Uu/21lqHNI47zKUm3O02ITir", - "Y56mW+vVBlEbadlOt/F6AuU+A5ZKikcZMTS0dci+5P8INr8lIPayJb97mMWvxbZmmq9b1pb8rqv5zNwq", - "rcazJb8LG89KY5Qp8o3sXO7CeyQrFw31EVJVuhYT+cgt/mdx9L1MZgEVebfjXbG27CRBNc5ow0++0vTW", - "qVSsE/8ozzV4CW+/OKlcv+O2Vuwt/yhrkcGgt68Fleu6Fp616x9lJXYXpjqtwn+x6/C/r7eW623vtysa", - "z971/6Gurrp1drfDXT4L7HRpVY9R6ER2va5qhu1/FNZ0S8Pezp5cyW0syoWhtOZxLAOmIEh4yP6KwQQW", - "2xvdyr58fjP4kbgQZG/IFeM5Wyqdsx+esb8kL56zyZLf4WwnhnfhwnDDj2JCuYDED3g0nBcRWFTLfP/8", - "blDksx8H4O40YRHPsoQAmdGR5jbJBHzFKC6C47Cr/MP33z/9YbMTv4fEUe2NavtL9wvEBH+ypLwDX7+w", - "jIytRDaA5cUAYwdLX+H8wLh9ju9cGOw+zBKZ5BgfAU524NV0I56jdbi74oNAujgISMSf3CRxATBJ8DwH", - "oXP4PqmEhtqRkjrPOKVHKhFRINgICY486rcZnlt1jHborMfQNx6h9Vb7cANiZy9EJ8yyw3MIltA2OYQf", - "XmDzAOApoPT86G2sChtcilAdWKPpD2tDdMfOZNOWE2uVJUuerW3fGAZpq7MDs/HajKFEnTkM+/y1uh2f", - "bPVMq422lcV5zv2NGX3RIhvYpOy4nPh2rDIIcIvtDAuwxJI85Xurm3bZCoK32c0pO/giE+B/p98PT4Yn", - "h0PmYwaZA4j7g/wSOY52EZrmMFkvHnPEP4BHOWQtwmB7isDTKx5RPBdPU3Vrfd/x6bw81ZBxYHI06bOp", - "mCeSAF7+MOmzBb8Rhr0AiJbKmDm+wG6x6b6L5za8BDu1M/tlkeRifGnK9dFlGUva7y8jDGVVqfn6csZw", - "1zGpD4K3D2n+cFeYXqsxt9+fPsFc/TXmffrkxzrrJrHUerYe9a7uN/DzcssGuGX0GEGL+FLF4v9l792b", - "2zayvOGv0sWaqkjzkJRkO05i19SWbMsZ7fiikeRkd4d+yCbQFDECuxE0IInJ+P3sb/U5py+4kaAkZ2af", - "3X8SiwD6fjnX3w9ubjjQ9eDF3/42MMtyMBwcHh69HXwemr+/ewt/Hx/S38dv8O83+PdzfP/54bf099Fr", - "+PvoNf39Bt43/4O/vzuE8sz/Pg//9v0Ph8Pvfzgy/zo8HH5/eGjeOXr+vSkT/gd/f394Yv42/zN/P8E2", - "PjnEMgZPDp98D38/sX9/+xb+fv6M/n7+HP/G50/xe/if+fvN9/D3m7dv8fnbk7fme/gf/P327Q/m77dv", - "X2F7jg5fmU7h/+mX1/TLa/rl6bOn0Imnz55iKUevXh/DL69eHz/FX94cfffU/PLm6Ltj+OXkEEFp8f/0", - "y5ND/OW7t4PPn2GBfCoW37/CK/rboyfAwSnyGxGfIAghrAv/KxorYH4HfzAPSly5P4kcY7gHuHmRlQZ/", - "OvpSPU6K9QcViwtIpFW7G8ZdGHdHSkCYOAq4Th8pLxCDileCS+0J8Xx2YH9f2hkvlieEIfk1bPGuh93n", - "sozFXX+Q2gCQgxGeQB32ifJDphZurF9KdQfo2ZdhJfl9ZxiKak47JIa3TPe7REOqkpOvtSN2T2pQG7rv", - "/FqZ3reg4Qvld5CurqeZyKdh0u42PFwXuGYvTRQHKzgLmch9gAJGqFK6Q5aLSMTIoixjn+8Vi7jM0gSx", - "fMfsv0SuMNICdASAKmCaL0SxxmuC7XmmcYfKYO5aibnKUHEAzAd0cykGn1NAvyaR+lbl14wjA5O6EXlK", - "QFDU4hWXyUJoB+sXCCtbxJUdGYPNSbNzvid81j3N9wBRqYKn2CJaFu5HonSKlqW8Nhc96L2AVUPgG9pm", - "XMRXYpQm15U09FImSlpMDuARCAF3HAIGoQxYHQfPHYaEF6hsmL6yuYjAeY6/BJ86PRNn0y64MTvheZq4", - "hWk2nURFCaIks9RjqgRLG4syxcJ29aUF+Zs7YHKANtcHkqP/muiaxgektO5AmQwc7cmv4j4sjA50psnD", - "3sSyqXBPms3qaiZ4HeIL/QQo7LXEWx6UYykL9HgiL8zEkgnGvOQQ4jW20iPpVNn3Br/9ViRFKr58mcjf", - "fiNoqC9f7hN/ba/STmjQFXGWjhxnKQipdpd6RKPPlVvWKZ6NFKVQoQx24IbLFYFQWoCdbXYmGYyczXkp", - "eFosEWOCkjRteuUm8M8K6GetMYWZikLEvRiSvwwHC56mcx5d930fM0H7vm03cf/3jUo5dagv27/50uaf", - "v2+SPddrGSGKJMlIm5A67luLGe5FkqZTDsafFhI2gmBwjNoQ8g0vp2uWi3mZpGb3DYMTGP40KicFTTEM", - "aro3XbxrJJyw0yxXEQScbzJ9oMBj+dPjEtYxiSO22Y9LDu5ameXqKqcgp5r1Getl9g1zfnEGciMmKB2O", - "Ie7sCLSaR3C415sGWYtt4EOUFshTgDFDMm942aUpmN/XQ5aXIH0PgRl6Df+KxVXOYxEPrTmrgt6y47E6", - "HMCqmZZZsCTvvXJ8WVmWJiKeavFLKYgq7hHn3tWTLbkW93Iw1UtCbMav1uCliK7hx2kuspSvpwVPUlfb", - "1EEld9Ltizy5EfEoVVfMfqVJjTBnRUx+CV8Rmj51kaQpkwJZ31K+Hj/uLoxV1KPxDiQjMsqEz+uuwUk+", - "YrtAu9zasAD2jnndiU6xr9Eip+92QbP3uSnbMLDdEWzUTKc2t4E6ubz3moXF/MwojZklC0QKYpEq0xjM", - "pnPh2ekfcNyIuwzEgikE3+pH3meLXOjl1yk6kRHPZUda+8eMGxXMQ7UgM0ksNMQh0E3uSxiz12kCOwJR", - "QleZUf8okBoM1KaUeRnkt0JrslwUkDZBArV+yMmPalnR7udsRW7CfuDKACUB4VQqCApNoIT+LVolqdCF", - "kttx/0CAf+9fh4+1TuTV15l9I5b2PU9AhP3q54mn9JgGt0vbxVtTHJOrpdBF641SP5/treLrCi6Y36U/", - "aOubLhJ5ZVZ/0jb+Z0HrKgZStx251ipKIPTf2hFbbswWDhs+Wnz+7ej5lz88YJu1d8zTO23sUMAC1b8T", - "X31WPIpGe3RE63ph+NULFhmpclhRZioyLSLFTp1P/2Fjv1KbU3HaoBePA/Bh/J6kJxYn/EoqsA60QTb3", - "blc5TxO9FPE0FzeJxch5xHlzakWvg/Tcvn2BEwsF2NnpqaT68y749AFaKK6Cns03r4ZthwVkF9WDVBoq", - "K9G6DC6AR52pegVToYvEqPeP0nIdcVlvfl1NNi8yeEezBTgA5muf3o4FjUxBjEdQiAcySnSNXOrxhwbt", - "mOspqMQPHRKje31l7ZRqcaqldwE+vOlfV03NRabyryacg0enfMBp7At45OPY+ppIOd+tedbj1NKmwIDs", - "zXUOiOberUVsuikoOw9bVGFJ9ra4711rC6NYukdpmPfV3bdVWMI0dFr3n9ofA4h570KrTSXJNQ+ZUV3w", - "VHydTUcHxlcSMyC8+GvcioUqeIre+DbG0oKngfvdGp+9Telx76FSXkt1K6e5WKniq8xTPVDEa+adgSJ/", - "uVias1oXZ7xY7ho/+TaRMSszVig2u56xVKkMXKwYogi28Zn57wzeKNTMji7h/4nYRm+DrSlOchIAGMZ0", - "Onf2L6UwsoQiv3idpK0W0cOytNTtJSNcLfAx2bhAW+hc6GK0SHJdWL3VsWNWwmzr2fZUcp/AkTfu5S2R", - "K8c+upt6H8SwwKKGYNNKDPhuzuyQIqHqwO4BwwxN34GyqR7NvUNYcRfi8xZUZwIUqKE617zVu8E7Pwi3", - "w+yCe4WO3QNU+rgoeLREyirHDaFrNACcdunfPo9r1ADFZtT/CrS0K/6h6NKtaMrXFLXu4oI2Iydj6BXQ", - "NdQgmLuAGFoLcZu993x9dF/AlXOPea4d3LBaoCgzCD3P7r+WIt8VzhPjERr0M958Nf7jZHIx/mNbwsz1", - "VFP108zeHVu7XbtuWm+sQUvZ3WMgHhIsei3WnWGiwINH9xURSQiz5ok1onJ4bOGteNz4T2h054AAqMGO", - "9/gM2TBnFBEGkV4QqW8pHCytSi0qFsJ9fASYpZsZsxPIkAR6EA9PnZsTCQCAEbozdJtZOhRSoSDDqXHN", - "oxAahIYdSzZLYj2jjBow+/EKWgtQUgKOk1pU3tVDHwQJ9XyjXeKFN/hGkRVnMPdHiFgzqRjClGLyAMTU", - "XWyAGszV7SgVNyJlvCyWKrcCv00DUhRtB3OAnhrIIJCQY0SmMTCp8ysz1jZTIuLST1AVvMZzaEk2mybx", - "DKWckLElhGH22SyWoXYTPWoLUye+fg86CCea95dcYIEbiWKL6BJyP92bwpogQuJ7sVebhe81CpdO7JHC", - "/iUI1K3stSEMVeWwoETMUrEoRqosRO6ojAgaEjpaYHy2BZUUFAgKfJlmqGxgc6GYKQ+f6CEJaTBoiIy/", - "mzBrabjw7Nu6LACbslfBP5s3T+6yXACNXJPpCte9XWl2PW8+nE/inRMaLxy+KSoC4i7jgALuj2VStPC0", - "pdRYULnglzFzugcjwQjemqmymL0EeNFZIkFPy8WNyOmACNISzdk6m6tiCS/BGjZTXErUgCAyhNjGlknm", - "UuNc7LlDeLX8gMdskdwZwTKRV6kYLVVWLSGjlBLNsuVaAyORFuliZLRMf2HMjQpa3KoAdZ5Wo00P5itR", - "5Qobs58oMnWEGFHBcMKdYN4e6cScpSgkv2Cc7cEtMTRiwb6/IRINa7pAcFrQxpSMXBS1kjRXsR1FU94Q", - "UkIzErijVGlIFJNIGcxTsFBTCD3deTitkJkYIMZG6ygVIS95nszLghALgpBnHy+K4ZQaSeNFPnKHkp11", - "G7QbjImlnLHXPAyENhdqKTGPD3Q2RKAX1m/NJYV7YLynaSVsowo1FwSEw9R7YjTt+xcsDRQT4IpqCgY4", - "OrwgnQeFtQCN4VFV9t9J7+2rMDbolaZLlVW1xKOd9J9VIh9aRE8dqNr0DlPIySozamXu2Nvg6BGaUDic", - "XSSUbXjDfPKvYTnpVPM2Xxj3oEl8I6KUm32NubVwrEDq7cL1/ejJ9y6ttJI+ipYTR6bXZMpjVLqNNKrS", - "7dENBACPkD6CdEJVVcG0o3LchzKAPZyc/c0eTv5Mgu/rrIrNrf4gm01ftkHwXtIt62AsH5FfsDv9HVaH", - "CwN+l6ySBzHt8Kgo2+RApHxBIkDHRaKZLtEBas2lpE305Z/pYM5BNDxXzRRy0u7Jn+NS7No7YNS7rID7", - "C27qHTuwiUDnX5lOxtPG0IRvPn362ZUaGaaObdU5LqMlJAFWNVWrEgtLyW1ObNjtY3apMlKcKWSUp7Sn", - "UP6zCjQxt+pIZUSJZct4ybKysAU7JI3C6dwgF+LqmK9JMUOZvuqCcLgPDa35ZSBcIRd7yuci1YyzjOdF", - "wtMgWRKKGbML8gRCRVgBZvXhUtRVxroRGWAzbg0l0dIIhbGVcKk9o9skFmymjeQlYtLg9QyTMB1t0EtW", - "5OYMFbIIjlNMGsjFiidS20wiog/wtLRzksftAMSCx2kiAdGAy0ikeJwPA/HTs3E5Lu5QXgWB3QnTWhQW", - "d0ErT+JtZponKQjKInbiaJK7nsPMx2wlVipfUw4qwnUQ6VTzZniI5XNlTWz9zAd0HhDfZo/UU3i1yzSK", - "lbsSO/du3SpaUyjhSSXStNSOMCtdW80F5PAhrX9NxQHoxI7cDnU7bS84+1Y4gF5fEl+rWFQ//RyOzv24", - "dV+bGjXs0JxOiAZzrmdIRN7cr093i/aHtsvR2cBqzm/XRCoS3nsZeGbRiYuHE+jtSqJ+BdQrO4nMONDW", - "jPb70uday8zj8Od+UMWJ0TvPrIl3RxErFYueiAdGngTRTsZwgPRWOqtf1hk1Tf22tA1nR3ECPsAzexzu", - "uEWcvdAiZaEb3lP6gCBgLTXW4mGtg+B4TDQqJNZ8CtIbMP44aCxnb2RS3NrSmpvtcW3M/TH/t9gCq0bL", - "3Vnw262yeExYhYjZShCknoSZG56WnORezPEH7wDa1EoNmGPSXA/WIGUUOjs5zuDoTEckV+ehxVdV6h2z", - "E3NwYPo4UZji/Uz0pw5U0yL2+wIcezvc+9LcS0SzFF5VQaG5KDgQ74OxCIslH1TFPk3ACKrMLHuANTz/", - "k5bPv46/ojFMbjY2+S6+uqfi8ez2W7bmGaFQ7gIlK10gUZ1BH9yHmGuxJW4JQpLQyo2lz8285FwC77CO", - "VC42rM925w08fon/+1vy2comgsQWWHLwu7L/Zv+HHX2+B+BRL5ecZ2Xoy43RLcPYLjYAnYBX3JwF1mZo", - "hCiUcuk2+QrQTs+/3bK7HxhY4j6fOirUmki3rDLiImCd2bor0K/nax/IhgsCqaVhdKzVd0hImVNdrtwD", - "/9PQ8b3n/BbRZwhZ01LDLxBA2X5DP49bM+O7oSHNbneVtkjK5YqphWsDtdlIEkpeYdQS+FmSALOIkim9", - "UZG2VSW4b6c21mHdKz5At9TDia/0qjmlGw+ke7gNjZiAaDqyop+YwRk7j6GMyVGYiwDq1cVaOkVB0LLX", - "Q5C+SukPMuehg9quxVoPmVZes7eWhVLy1Ty5KlWpLS4UeZNAy7dZqy72Jk4WCwHgD/jaA5w5lXGseHXu", - "HZu3EgWPecF3g/m4b4yYLWdHF4V3VzVid5PCp/4F22jcsn9tSG+ZQ0yPpa8k0KlEsr8dDvHSeMgOCuPe", - "/MTSV64rW3fJm3BN1HIinSs5iA5WeSJkYdnjUwy7sdBZcHub177RdsfYjTBG93mAGEiactWPDXumLRr6", - "Jfrc/edQl72g45qHxBaOBZkNa4qBAglX+FqIzAZfp0pr8P0if5113eO9JyQc/Eb2y0WUZLmKwBDgK9Oh", - "cUGVRpxLZKt9oblSdw7B6xFB1+WLsXIpWk7sgZR7T7k5ykLvVC0SoOGvWT/UW3NfKi8zDhtX9sdQfKgN", - "g32ErrZCweUWiJ/6xUSO3B3/ggE1mwVdq2VAjO2r/qqqfaDx9iVQ7BAQu3IbY0GNo+QFA7eMLcweMBan", - "kT4e0hkDNknQ4fy+IImjUGwu/NmDwGG0ZJ0TG/3ZlVu32aTNC/teZkJz9ZJvo8ChMfOwIDWVQKIBV7gZ", - "Q024einXBROpWJH0Arc2CBl4uRbIRXcjchdlQzo9y3jO01SkuNxBAm25OgNoyv7JTBjp6Ftdix5vjcZ3", - "QmOzrxtjzUOffjNSfPd0aVLrel28PckbdrQfmpIxIuQNUNzfx03b4Tg17bPrGvnzp/hmq8t0g+/yoXpK", - "1olIgI5kG9o9Qgh7+zp67tzVJGJKDAdYkyKpCyntRDsYhNRaOVnXQw3pqkxiLiNBxJM5iA2kQODhqXJv", - "wv8nOXmHg5tEpZs7teLRMpFilAsek2uCa/Jx0ml5yz0iZsVzQEf3NAxuqoQ6Tflc3YipkoIYL8w5OlU3", - "Il+k6rZled3PLe3XTVVj8r2vTvDGy/JxfTtec6qkJZjVUc9UsA7q38HnU8+MCMi3bD7GbnYNGrO6i+Yh", - "DpoGQUOHlwZb/Fhemr/iHDzIkophDw0rKh5hdROfDyqhjCc8km2wAqAbQDqzubWtiR2A/XmFgsoWM2bH", - "zsuO0E7ubRu35qscGuHgdplES/cctzyFX9XjXsYMTMEW05dg2EVcNwwT88K/TqR6MC8vftvF+RzEsfTy", - "HpNzVOzwSZjXtMNnbSlc3ksNf5/xnFvCkCwXESJ6tIlmb9xzWn9a8BxiJb0Rx0hicHYZEXEljA44bgOk", - "NXLrcvWQlHlXyDRzPfgaYKge31x42Nd7Qwb8z8vn3Qm7DoR7d73ce5R3y+zcAfktTADtG7C8W/EkGNQ7", - "8Khd8Da7x0OT9Wrw45YLkHb94mw/qFj48FoSLlcU3LzpQ681vQfX5pdHkA/W534i7y0keImpJiUoORIQ", - "uF6XFKxceS3WInYu7Pm6JjF4rcjcmf8P3NgdPHj+xqpKwGDHmq/ZO3HFo7UvJpR/rUFoMBxIszjmKgcR", - "MpTH23KJvS3vARiTvkmfJEFbG0He6fOV7uBhAdwX5p/bUyw3B/jAh7iRLAwCAJidkH6F5dv4yinGQ/oQ", - "6o1R8PTRK/imGkw+7BehvVUQ2xSwPtxmzNimwzSNK7ZI0OOnpdRlhvBU28X/T/7lalm3Kr/eaVx/Vvl1", - "25h6ete10SwGL0hJ/rJTcOWmWeslhnY3r6fq2Bz2Xp9uWcq9ymifqV6fbl2ONVG8UcujGO6aC7NNp18I", - "XpR5yxn6Y1UDZfTi0GHAz2DDziCB056Zs6GPZv+7TbKhsPbG9+AKns7XsyEaeWeuMuAanf2BPp21Wsk2", - "mhq32Qpr6jXaCCm3ECPThb1EoUe+LZSZVAmO9xkAHfY8rncxsvnisF1ZLm4AjRndToDOGfkA+vDuwqTQ", - "qfm8EFfrqVTF1M9/oIZMV6UupnMxRSdcQ9lxz829NKhIVe5R4AzRBc+LqQ3kblTrcAE3P3cGHHO7Qi3g", - "tvcmQnO7hm+JO6RYnEKMmfMO2MuvXk3tdQJfq79FWWFTm7KKl0b9rRQkiGlVIW4pC5bJlMgVG8+D3YkS", - "6+fhf6+kn9C6ao8St+I7Tak1GqzdrKk/OTHVRYEQxCEHhCL0VHLpIPFGcx5du03v8Pyrh2jEZZzEZv1z", - "KacujaMxGf49kE8aOGuDbUFnvgBcNeHBjRRFUTG9FmtooVgkd4PhgMt2JznhT1fRmlsIWzc2aFFqAFBW", - "q3kiRa05UiV6PcUFwCFPxCyz1osEi8nyBC+jloANp9duCdEKi9qoXPYppchLXTywQddiPQ3JrprZECBk", - "dTyYCstK1eOVaZys9O6ryoOPVsAkG5VBsGc+/TtdSM3nTUDKbe9MvQeCsroHQCkXXEv2byCeg/W8bl1D", - "LhyjFTUNrwSXJeWqq9j53ElYj6IzM2RraPazfQRrM7/pPCOvUM0+F+jp5G7Jghjoim4+Zq/Qi69tXpQn", - "Yh4SRXgsDsDIarV9o7m7eHnd4vIBI+wmndHXsVmgdq9RV4cDCo3f/J3tkvuKLKkbPglTqJzLaYuaoL3B", - "pKb2wNrcTeuptbmfwlEfn15fhV3trRK5Dz5XVx81e0cXBn1Fof3u6Jdlmn7+8jmwWwxeDP7Y8sWgnoi1", - "Y3h74EIIEsbGzOeRwQogDcFlkQ0hqjTieU6J4g5LC4Q1goI5a4TG4JapeSZZInUheEs2ZwD6VyGpUJkN", - "cbZZKyAAQ9spowQaFDjoIOgPxFmK/SOMmFUiS81Qwu0bur4TAdL9Y3F2CsH5p0TeiJsktkDj/bt3xiHc", - "y37MhLwRqcowXANhZSB6w2ZcS8LYcXG5928wRSXWrJV2nFHq2xSyuHtQUhsEYTMGO0gWaVnSQxtgVZA7", - "CRazefKNZosyTdfMx2z7IMgzWvqAk7dAwvogfzWxwDsV/w/CrP4uuRO7DeZ0S0KMKaYS1g7IQi2jWRsY", - "GAI4EVqH6KskzOxEUULbYRN9aHPTLFQaV6iC8fytn+ME6MZbiUhB6+gJpLsL5Hg/+BXeiO99uTnvx0Gy", - "/FNifId0WT00xqU9w31HWDlEAcARoix2UOYqzqoxm5EMCea+G54mMW0JsAYS5WouFhZA1JaCkSf4/GVb", - "EIK97NHg5FAdijI3l5HKXag9tRE+zNUymSeFb1VTHph7GWtHFCpnLGhHkskF5KNEYC/VFkWv1BZw2g8L", - "2lZrbwPDDUGfNEnPt8Fq+YFuNnD2BzQjU6tmNKQYSbyAmNucy2sRU5Q7vTdmsz+gdmN/mJSHh08jeAm8", - "f/C38OVVV0hYmDkv67OuXaqIm62NC8FNbgJA08GzSocqywJCx3m09AsHIqVbdy35N6HxIyz4YT7O2g4P", - "pmiLEnqubh/B9WyViS//Uh7fMOBv9wjMVDg+jIBAmztAmMrZ1Nj6dhFM3YXURbNn4X7t1jTv+zW0hyeO", - "ylmubvV+OyhU5/5ttf0UeSkjS8hUh+MoBZ4btpcW2ReHQmUZUBQsFKARAaxNcuNwlKpUMF2mldrQhA3a", - "sFgtcE0df9/8bvPuYZzqxhBvKMHEZvcAdELLts4c2/qOiDJe/Ue8nF3Vf/zKK+fqFjhQ3Ss7Ovccfcn2", - "nUs1XYIzpTFLWFD3hFS/bkwMErliyoxD2cItQ5NVgdt1yAkONgElvVIizmJcg3Jw6N2yTFMEcNA139Yf", - "u9NpwmjH+3CNmLN/LnRBcvj9CUb+l8fjf3k8/pvweFTypv470nj8v8fK0Xk4N6PAfz8+jmrc24sdY9Y7", - "8eaqxXb2/JJ8Lz9ZbAou13ST11u6Pfr0y+em/TlNCpHDDoCcUJW7/FMjJyUR5qITBcSfuYxTMee5ZtYp", - "FMAeAc6A2Vqegs17nQuxGvteWU1lx8vqVS54XCyJTsrrO2BDqoSk+IAOI+gBX6aRojYhxaz7gLk7pHYb", - "8VPPJuc6xG63cO3mfEG5IEgMN9fE/96Y/3tj3uvGvMfFd0oNJ6XeeZu+0Y37b/e7jrb517joGvkR9W42", - "PwnMTe5ePNzVLtRxwQ47bFj+OIKPxuxNcIYoKdhSAaXftRAZW3A5MgeDxQIdD3a6tsGa3mvlVIE6m0FO", - "+QadqJqv9ftduS4C/UX/pDKedl61rrjOjgKu2TFGcewC7I1f9D8jG/hp90dBNHVv7k9Qz24+cTca/WKc", - "zdsWv3PHT+B0CYwF1bJ2mwypKFyy57aoA442YadseZuH2ffjHu11H/ZqcAU7tLW9WF53gzvi0XdruQO9", - "bl6ir8jK5uCpHVS0RbP3ubjVsN3UXEBTh/Vlf7CgX5UfpvN1geCzVVBuQNPxlClTgN/WGLEJRhj8hT5v", - "s6Z0hJK35iXsBJmPkZ5lHgSERyJJPVN7OFR29B4DMd/mgW0Obl+Bl9tGphOofB4SQDw8xvx2qbRFQfPo", - "577XIQY8jvXjg1HkIi4jL66Yef0nQlA8POQYZncY7MiQhmA7qoPZrZd52RYpgQ9ZYZ6ymBcc1TtU9rCm", - "WhbGXSYiRPvSreu/EHdFWAhg9M+NEHS75CBZYxHe1qgzJbUYT+RZrm6SWOjqZL579579vYQwgz0xvhoP", - "2WRwsVRlGjMjXprVleUJTHeRiFxPBvuIZtQy3am44bKYJm2agYtPOX2jGXBNE0aS/Yzt4dqyZAor869I", - "7++gDbRxkfw5ETnPo+X6WEZCb3RDY5hPTTqx8amdwbl9o6MaLUvi1vIouqeJmxEGUfgAWtCZWj7oNRbb", - "r9uml646Pp69ftPl6yo+y9XfHXJeb22zlEnxVavYPFhbefXbfS7LUl5PcZIr90/bKbbDYNhAaXu5unBo", - "Gy7t4sRbEfe7lvGOE2kHBelCBjRJ06RjKbYEUUM/PvcZ+AvXtP93h//xR+/1MknjM56LnVEPsRlbQA+B", - "+yYcCFpAWxGX8EM4/ra3PxdyZxIDalft6jSlMSTsMZekpaYds08yKSoBxDzWtUx2cVfkHI6UiVza9rGc", - "A9xoseQS7ZfmGouIExdQ8DDL3nLEAOFZQiwyZI6Jq8iAcNC1gybZeey1O8O5b0LFwc92BnvNAYQnvlq3", - "wzq73AHKKUS8fnOZayOISuAZ8sPmhnrMLjIRJYv1RM6m7vk4UzrB9E2uI0H4+sQqzLNMyFgTbbB7Tvxt", - "ExkL9BZBaIaRV0bzXPBrog/AEAPN5qpYBuiwYMlbJsU3plyt8mKGk+Kkjl20OqGjNuPaVunXXeN2KbSM", - "yPa9haW0TWkAmrUFCbwx+ReQYXa8IDNrHeSU/1IKZpsI42xW8SiJWVFmAFZGQSskkWa5iATMWnXMN8p5", - "vvlPwuY/2dT8Mtcqvyh4AHywU1JBy5JlERRq6U8hQVClZrnj6Tey4iLLxU2iW535PNqodUUp0F/hW6Ru", - "Ab82aLk2EDhScpEmCF1uV0wuMIHWtXvq2r1ZTe+XUAyvAxljWKmvDIdmqs2Ab4N1rGVClCsug4ruspRL", - "VHcBPKwyAZheHSlZJBJjvFsUERwI8hu1eykgmxwY0QsFlnMbVAzfuhB1wdBNZtZ02G9Mu5xy2BVb820b", - "4Un52kI5AgUwLSvqnC6jSIj4pW1MGGLvk0zuoVAf/vC4CjWPHPRhdcTD7m+8YE625l60C3c2IW5npQuS", - "XV2VbiwWqeLFJqAh+w4Yj3oLi7GIEmvja0pQKuLptEMLJN175/71TeFseW/apXna1OiND6fdaZf3Sgdt", - "bctGPe3HXJXZq/XjyIuuVJIZLXa1jTMTcCznMdn5vGkWUQjguovF3XgiQb4MGI6coDiimGdzZRJLJJJ+", - "VkAtZuFJMxtOpMrdbxhyab3qXmCI6Oae1aSwIR00o4xfJZIXIp7IsiL+VsVREumH3XIp8R33Fkzf0/v3", - "TVLoIbNCFWQp29kVFZhFNvUHfHl/TooWg8qX4SBc6Dsp0xW9aqf638FXTlGYxirq3M70ykaNc8vw/jnZ", - "dWgB8ak9A+ic37Ib5G60L9kshUQbIZFF5o7EkI50bXNBYiG1GNGHuCFeslTditzK9ma3zUVRiLziA9h4", - "1qOCPnCtffAV0DrE1cpO35gRnkJWfZu0YBVLeGHIpOlISqy0bJlcLYMuYyQAIOyD4LJzz7EV7d2GVm4x", - "xmzAy7Q1OJORO7F2Oz/s5q4fDmao3ThuPyPErspdV9DLW/g9jGqxWqXEMF8kJkMejMyZJ5EPjnTnuYh4", - "qcVEwk1Bt4IjnLN3TaRWK8jiWfIbERDQ0DiN2SctgFpilRXriUT+vEKRDkYuKRdeYqrwSo5NTK2EW4nY", - "BuVUVeKtYTnN2JGnXSEfrpfmsZAxl4XZ9bqCxALDCWNTJ06aSLh2HR0zZv1A/WNGE80WKk3VLXoIFwvi", - "o8A34a4FbAQsiVj3zCKyZAf4h1ow06o54h432oyECeQ4mkhQEbVdAdYNY0q/LZZj9iNx+6W0FkrgmpSM", - "xzzD5AmL8DKRt4mM1e3QNEXWkomwscBWmHFwGkMUH2daZDznhUjXLmkDujucSK0Ypw/h5YUwZybd5MTZ", - "sFB5BLlyV6ma8zTM6tARl2N2SRzIE+m9kFyzW5GmDMDcK/0dMrusHUU2SjqO7//wcCL9HFpxScbsaHh4", - "eBiEzNt2WoamkK/bomPwK4HrtWn47Y3r2mpa0ZuPlcD18LVOFmcBWK7jHAJG/Q72gWLNk2Yi3VFj7lZh", - "uc6tyGMPI8pLUEGxWJMpBvXTieRRBIcIpMjQ2QJpGXap+ZaEBxKrnUehMdUdSrXTpx4buNMx1H8KT88g", - "BNLFaNUyNpXSW6W5V/CSUTGTuF1V2uCqq+TYmO9bG7niV+LT+bvmOoEn7NP5O2aEKHOIfzo/bZqeyrxF", - "06l9xfbMv14kpsSDTF69nHMtnj8bjsdjyDMTd9zsNiMjtb6X/PTq4/nt4V9+vFLjcYt5ptZZ06RNfYW7", - "XhZnFKnX1u0IXzGX7sdMyOPTUcDKgSdAC8W6+XRKA7JpWt2gBxxnVj/yhWy1zBJLWNsXQaelPcreiIIn", - "qW7DXshSYKsDs5jZwFcgAFST8ILMO4cTACI1GnPMZp8MElfbZNCWqtiON/hzW70vLJsSCL+Sp1MIiIcw", - "hhfsz0nhjJYUvtPxLl7btrBCXQvz/bt375kqi6xEohCXGGjek2paKJXqF+yDYvAvditywfgNT1Kfhm8a", - "m0QQoWO+ilKeJwsKbHdYgC/gBseOSSFic/+VWuTMmnHs8IJKIp0Rskru1N4zonbCLgGEEbZ7MBy0N2b7", - "ktqAjwcwpR4Xrz2KijsLM5fMhnn2i3B8W6bppbgrwnq2BS2eWMeo3uUrTOPd4QOHJ1H56HObo8YNSSdz", - "W/CzpeiEgQLJxAiDWZlnSos2K7SQeRItg6jyahR5mgBZtORXkE9r32WxgNQR2A1eOyCqY7QpsRNfNBJj", - "QkIGiU4I42c0CLIqaBeSBr5qPWTgqx7FIk9uwisWvJOQpOKc2JrI0D0OItNFLviqchX8zbrbdfKrGLz4", - "9uiJu+wGoKlZt7nzlls122JtYsumN0dNVLVp/V1opnkVoKYg2knEhHj33fPvu6r2jvmO6qdgUejXBt9e", - "s7R6xS77SXMr+b7oIzaKoBkwWFmlrVHhPWg6YefYLBTygldUucMONM3qHknM4lhZ8spqJP/heHBvQKoO", - "TD5TWOdZiIdPiBVau1bt4gAUmiQWubut8JhEfYWkZ+uOjnKBKeNGD1rxBHhwNMq7bgPRnkWRHI+LkQbH", - "dxIxXc61KFguQP8DH3bAhNvaFP3S3D+xMrqaovNCgMYXKUQggY8mEuBOMo4RsBvh+6A3nQcqPK0N4HAw", - "F3GuoutOaRgfNz6LFPKkt3/1Gp42ProSRlHrvCKUukqbH6k05Sve9dFHeNr8KBOSd9aE4mXjoxuRF92w", - "+D/B09pHTYxBO2/9cQZbu7DtemztwraP2idz21etk7n1vm+bzG0fdQzxFkGhbV1DFgWeGNascS944Q92", - "78VBlDYSYBAJOV78eFJCGuuSy3jMPsp0zdzOJWyzvJRFsrKmK7zz7YHUAkJs7qYeAEOuh2fQLpBZQozZ", - "nb790nXyvk9SoQslxYVzR+9gGJmnKroWeYsE5cp1h+mwGbNgPx+zEzBAONNHLtCymIA6IXZLOaSP23Qj", - "QSFhiWYr20BTieZFohdJkMEx1yK/cSsgkRHPJe+NlYINGPrx+bx1/PXOmCGoHPaSFuqz/GU4AHOj9Qnv", - "XkCty760oW9Zd6dLFDpeU5jMvQKASCliKyrMB91gIuYMAiKQZcAaz6f25akqi0itxLSUEOw+A/OtL2vF", - "14ynZqGSMT9SK8KFM7pFxNNU5Brp+Git2MOi4IWwmqjRTEFsubVLT7FEa4DUBJBMV2PznGik6YAa4Xtg", - "uzvwHs2pNbxH6/Dx1t7vTPNaiZ7Zshs2JXpsXSEXIr9JIvFJOoPB/YLFXJX1RbPgSQqMFSGuTZyYrs7L", - "UE0bUbJXlqtCRSplC8AVBciAQqwylfM8Sdes9E0FewbIiXPRY36hZdOIZ3yepEmxnpbZVc5jMc0wjBIy", - "v8wrUOA0qOhh80f1m4P28/CrzOYZRJ7y4EKqIIlnubqCLE7gbK3huPpP2Vws+U1CHKfca+cNYX4WlDhj", - "K34NvhLNF8JoBUsRXaPKEkMc341gHlyfuZPMWZOMus9TCiFkEOzHr0QFjo/NeKFWSTQjHn3OpLgNSw1X", - "hZEWUu8VAUxE7qIzoUWN5MHqEGFlrbMO430ueJxIofX5jqwkRP17u1w7oxPgGwsCf8rMbuApGCEwZjps", - "ZKp4PKUNNQgtLMGPJCtVVq/jBsGT1KZe6SXP4/C3qTeJwl5wd3KwQ8gykPllA0s044lZtXMeXS+SNIVb", - "CucRH6eg4IdfbR9ccxW26PnvkoWI1lEq6CqwqY+x0GAsakoU7NgGoNt3sMFE3woJryhpusWJuMyg7Fac", - "iZiFxMyQmaMsqIXlYoXqrx33lxTMSteYZYiz2fqBhESHga6sSDfkrlFTWh5IDhKbEcV+9RzM8iEyUJeo", - "Fw68H4wl97vXb2mAlAwOHB5FKoc7wdzbhQ5N5KAtEHZvK5yDraszKNv51HqskNd2pjgis3E0c0OmISC9", - "FGs2L4kfWKqCgUUmy0Vh212Y71vjcWkmp7j5dddpsUF+t8dBmdsjpCLS00YAFyaP170hV1rPshaBH6ZM", - "L0U8tWHdbQGDV0uhfeA3cx8B0A3B+65H5qkXE3JB0O6BkTc4kPrmLldk7e6F6psU3BwRl/bm4lLfijwE", - "jW6JKobzrw3C2mbUUhINjKmHMwpWdt2U3MT4rpPuwGAJbYSoml0Oojd2m3BMJ6sfDF/amcTFbmsIj+wv", - "ARnVtvUS8ayAUcGBPcC7wi+jhg/Aj+3OC6UlsluE52uoWzW3baf0dQ6XiT9gm6ipZrLDdHJ/cXHJeFmo", - "FS+SyO8JnuRjhj5QYN/kV1Ih2iqv+vZsNASPV4lkx2enGDeFqcNduRfex9a5W7iknHuV45GXC12uxBDl", - "/CEEZNF6BlklzhVGBXMw0bjw39YtBKeanva5XGC+4ap2/G1dNw6MBdoZXHKBu4HcVbuxRb2OEd+kqMxz", - "IQuQCoLGbRARMHQTThnzaONB0+WAfuNWg5Um7QLyYLR1dHwQY6pzH5p+mtHo7ZIXSbXUfXhpzE7J2Yt/", - "j+gOY9mSa2Fz/QqM5od4GWQHzJQWMVuKXFRTeOZlkpLYc8uTAv+V8VKDuadT2unY2fX13jLTzeXYvc9R", - "tH5tFchE7Iqg/EZkqVobiZ28lbjjo6BEJzNGoCBoADMgTUmX81VSFLiqQi27batbk0XnhWWX9KJM01Eh", - "7oqh1/SQKyngehMkGrklHylpToQWfytda9qh0YoE6tGJvEqFVfPhYEmLxP7tGC5VvjLCzS9lcsNTIH1R", - "rN4ZhGD5kzsI23dQ+1fbLrRq1rV2s72mu60OoVQf5856O9dV7UJ+mE7pZwHH1WuX7TolWjxoY3XplFa9", - "a1cut+iRpDF2Pu+lFLYLLTumNwR5CZ3AOrVV7HEdB63Ugb2uMD8VyPPijN9OHwKhVEmChrZOfQYj2764", - "v4pSQZqSa/KjKRntS3yTzNlUg7eqvB07czB098F9pDuHlN8Y3goCvg8h2uhqttE5/cJ3sO7AL6V7RhfZ", - "D816m2ImUI9IJvsVnPvbI5Lw9aYrF4+UggIo7hFRZQveKaCq30fN1u8STkXfbFwreCzV7SjWh7rRYQzh", - "FDGUhKDB5lzVW466PrvPTlSj1b7wvoU0APEpCMClyVZa3TlSlxSH04RTMO9XAmnC28qv58qmsEt2GGyw", - "9quEOF+On83PhYZE2PX7VnCzd4rHI/A65/bF0IGNBg2LAPyjWK04e8aePH81On72iv312fSQvRcFT80h", - "+/rTm2Prwx6zjzL48OL99z/A8yFI8ZaFV2OQONRbOAANIxuC2FzK1Oi/SaED2rWkqJgPTXEwLeYWIxpl", - "LG/LyMSrBPAe7+X3P8tVJLQeQW6vGShxhRBgVrzjtnimBUiyLaJrYtuyfV3Si5TQXm97h1fe9rSME/V6", - "WcrrrqgoeIPZeIKa5cWsAtfS0L310/Eb4mVo6Zu6EXnKs2lMxUzbaFA+4kvMvsQSaTTYNMEYJ83morgV", - "RqfzDdRsjxw+L9jhfgUJ9vvnzw77UKDc8HjbiP90/AaDQ+E4wcSczX35Gd7p7soCwMLvRDzC0sI+mdH0", - "vXpq+nCfnm1cBa94dC1kDOxISbE+kUXegvNCbyG9EhA0mvcwlAV8kKkyos0VJnzM5vg60vrTHy9icZNE", - "YjacyFuj9zL8uwoSbnbtbDyRE0k1aggtn0legKttxD7Av9jrs0+MCoYXlJR35vHHDx/+g5GiWnlhZU4j", - "88ZxlqWCDqfwhaiMOVTw0+mb02M8tMLndyk8Pvv388uD/3h3DLYVME+FL92K+VVWYr9vuV6ZL37menXw", - "s5j/6JtsVoH5mYG6r01/38BoUHdhGExby0KNYlGgDmyOD2f6sutiv7PxP559gmeFadCIYWQVuzz7xPYg", - "m32+Zv/x7pi+x3feKiOnm7FVMl1DXLvHGv6/ezgL/zBj/Q8Yz3+Yev9xl/J/YMf/Yfq8v/diz3QAHxZZ", - "+Y8oK/f3/+0Pg03nLs03TV0btHh1UrWfAM8P6M4jsD6hYbgmiZQx32wkDyfeSPtmioqQgxDbEcY/dyJo", - "wyBtrq6yEHvW185VPRzg/GyuTzb2zwMrNathc5Vte/KhI2sW2uZafz6+eP9Ytd2lWxaNOxUepcItB3YR", - "LV383bnIVFtq1EerYlsWTBfcMESkCW90A/NtnHRwDy14mppOecqyzZeoe9/bb2WZoqmkuoZCVDyzJqdz", - "n5W9uQ56v2eLcvF3TBHoIF07JW41fMvaGvFa88Gb83UQSDFkHj8FWT2s5DXuQbJPWZZ926+BBqTf241A", - "wWpdjaGujWWtssbYDevL4fPGlZpInq8pg69FpoDHDOB1XRIfZiSDJ8tHydv4shaeD17wNmlFi+fPRkJG", - "KjYzihVBhuMeMoT+fPzTkJ19+HHIRBGN90Pv2XxddOTzxFMw/rWtIZCUXzAS4cwBAIddTdjbjsUALE58", - "JaaxSPm6vSqZoLPjBYsTDc5CeLmlspbdCYU7WoHOkuE95gAiOrgT+o4HoVjdb0Q2HoavecYjI7a2Y5kd", - "w6FGVuICSOckQDmiDktRdWsvN4wiKtBaBvWGuLr74KbdHxUNigle6YA+a7edv6+3yvXTg6R7izgNxtS+", - "BMFQ9kd31HXDniFYUevCeI+HF01H68KtHKEdXLOd2GrHCIAC0UsYxVfkXOpEyMLPrQPF9r2+f3AiDHgV", - "XLw2CBsPyddLXrz3a6BhNyt64H4GZdjT1jRcpTtYEMzLwEWm0mnEUwtMVrsq31jLlHkPopQp2B1rR+u4", - "GVwb1LTnYjpMc/5kPttvzUuz9basmUtbl2YrHgsbQs+1BpSeol6Je7Df10ZvanjN03QrhACM6Zb5LOU1", - "mrpTsvv2swC7Ei7FXeGnsedH1bv2y2fzYRuqb51j+5cS1CXpiGchbMD0ge0dDtnRkD0ZsvF4vL8xX88j", - "bJgTzVwKRat98/3p+xMwcL5ghbgrDiCXfYgmj4NbfjNkDk8Ab+at+AEA+eNrbE5NC1sbdk8trNAxZmbI", - "rRkJovFdiUE7x8157p9i7ew5JJKBuYPSCN3ZOp7I12o1T6TQ4PoSMVRvrWkoHPm7yslHOH6h3W3SpBaA", - "l3qfBw3jIFE54SB1c5D7hUTDWSjnImeZyD00SGjHenb4w/NtcjMI5NW4brCbNSK6L+0KNsONYrylQT9B", - "T/w38OE3iI4HaaoMEvNHc65F7D4mvj4sQhqBCOJX4G99AG+JXB/8Zh59OaiCY9imNQ86gqXbeByZE8AU", - "H9gai2Uu9FKlccdqA0xL5t7yprlap9je4fhwdDQ+3AdbHZiBQEGHlYZ9A4u7WU5oi8QfW5HEPJxO6/T1", - "k+FolXXjfNBDiA5nusyMyostb2yr/jzNwTELtW47at8bPaXywedGL5DWs3r699pueGu0HNrtkvqZhVuG", - "AzvR9kiD6uvUH6j4wH02HnTTlzZB7bd6mB28GzZzh8OXkJrwvfufv+T3uK/DsfUwh+DqrA02+NT87DTV", - "goz0Y3aBK1Kz4laR7op2XOgWjt0LNpsMIAc80YyzVMkrfxSOx5PBzHwQLK8XbPbbBMZzMnjBJgNQkyeD", - "IZuA0os/TsrDw6cRIu/AvwW+4YYPX3P362TwZVs9ZryxEPyX+Q0aaL6sbK7aqWv6asek5eDbcSbCXdZg", - "pDOzs10IO6f1j7w6dDD/Noh4tBTTZRIQPIIR4W+/gYofLXlO6GBGYjq0i/vFYUWyGfh1OfA7yOFboHqM", - "ZR3aQ9/Nv4eVxhU0HiOWha/+B6r+yFV/dN/qf2ivn+ACfAM+uyvWXV6u3DTRZpuXVkvx+EIW0cb0MsvV", - "KivcL0dPjFhf8DT45cuXJpSzm44uAysdFrdcMzJqwj0MX7bGBrVbhY4BfcxKJnT26P5hPC0HfUsATyCm", - "VPknKgIJj4oSokHBE7O34mtCc8SueSS2ZMGsyY2pCMIP4/1+koY/1uumYbglzPtDCxg6gSmeDMJkKzPn", - "bSq+WwW9hutHEv4+wVedNwhMmR09W8cwWB2b97v31e9279qQVg8ZUBOanbGjt+C83Qf/udZ06TED226x", - "MhfxFAami5f4PQq5SEOMfjDC0U0qKUUL1T9VgtolYijcosE0YD5V1JGDRO7G16kqY2ZfY3sI1bDfbgoD", - "OXNTpM89Wt7W6pbRg6B2J1baMPYEoEEtduV8bdWJdSbG7C9iDfHdE5mlZU6IvM1hs9C8exOK2AESNXPB", - "kl6k4O+JnAxyAZDg9rnVMOivlGudLBL3Qy5u86QQwdfclw1WryhP5u4XIrGB2vZZlpZm10OO+GTgtQWN", - "IKi2899oZs4Bc3Adn5161BlqzBoGi+vrMTtTWZkiKpuPuQ/ZCAnBEm+fCU32ZIDIpeA6CWnl/CYHMIwu", - "esRcXCW6yDEeIREaj08gJMd1z4olL2xYEbOuOIesYzfKbgAQhKzZaiKrrHt6cfOyd8gvfY8YIvLL7aY0", - "g9C2BbEN+JTtEYhLextakSHPcdRiJmQM+cyAMInJFBnwwldDzzu0BNe/bUd4twJ47KRuUAEXNnh+pWIg", - "xTBy+R6G8CcWChO094Rw1ozwvP/1dMM2tMqH6pP45EJEZZ4U6y5Lk3ufrUR+JarHPaLHIoIbODRGFEBn", - "NAYzMGP258vLs72L/SFbJKnArIeLp260aRp9yi5PU3VrTgT90iGGasr8xYcEQUw6aNAanfwqAkrS2j2L", - "n06XShcdqJdQPzWXmfd8Y6qxaCxWtzJVPNZAN4/RdpD5Ln1HEJAWPgY6qFjINZDRQwvG7OekWGKk+DTL", - "kxteiGmSaSbAmRQP2ekZSwHdkawlGLjO4zgXWtvzJrYJT76pImZvPlxALWBRguSyuSCc5XTNclUWmDxN", - "4yc9WjhRxxeKcXYjECwEa6zC8Q1+eDo++v7Z+MnR8/HTZ4PPu5xtdiYCnvjWmTDPR1kuFsld50SYNfXi", - "4AAGSD81//p0/u5es2JKsosTWjZmbyFwDisotWB8Dh55gc8tmenBJy1yfWDW6sE+foQtMZ/My+haFAfU", - "C/pitR7R72UGq+hgvza6YZlGZm18sNuIN1ZZxciJIRP1gxlulVSpzCgGQ0YfD1mayGsMPh2yiOd5IvIR", - "YHywD8eXELUAChRudDhCI64LFgtzx1MqFTvuWKyWvwTX9pDWPGlniTazUYgwVRXjhbGynVYzuxAFA+0c", - "E/i4riwTlRXA+w93eybyFUBx4hjgYstElCB+p+9Xu6pIZwW4slVZTK1jugJdf9iFXW/OI3feVII8rTOz", - "Anv49PAlO2Rxos0Ot6mcPIajeGvgyIrfTV1zzWlKFNxhS48On33/7XfPDzsbDKewWrg2izjET7aU4G6x", - "B+VtaFzVE4TZx9RgBDzeQHbuGoYEanh93yZxAdzZS5FcLfFYcQiM/nzBl+H612EEsnU4HERLXgwZBLsE", - "Ke9D5A7BbL9hKEmgAD5kswMjTM+sNDE7IOF55tZyYkFSIT85N9vx+OxUj9mfUQpnyHCOibgJARvT/tkY", - "aPQS68S9ZkTtXJg5i8fsVa0bKITRMgIaMfhytIQWBJHm7RWN2QclR0HEZJTzjMK8dbkyfSBWIiGBDoAS", - "ozBsfbuTvtQinwKYcgvmuBb56BiAlqm1ZoYrm6m2c75BX9mbVwdH48NvWLKAlmlRjNmFWtl4T3fs/5xc", - "JxkIfTTe3u9mAeY5862o+m7Cqlql2w1SbC5i5FpqyYIxK1jr6bVYt2oOxz9fMHyFXYs1O30TWJavxRoA", - "8Zley4LfEa9TlAu4Ba7LbMzegoMcDEWFYsc/X0yPX78+ubiY/uXkP6enb5iQN0muJCD93vA8QbAqP46V", - "IVirMh9hY0bXYj1K4o6QKNALWhzLT0MQdqc/0Ox8o5+O+Yr/qiS/1eNIrb4xe/0buLfMZfPih8PDQ3TP", - "vU/k6ceqwav+Ma7ETVy1OFJTP/7tg08D6ufgoRNwcfL6/OQymId7TAJWEsxFa766gJ2OhtYNeNbYSzwV", - "4N1aHFTkl+9OfW9rNtQywha1WhHFVOt0q6xzguFbFxfvDi7fXUDdqKCQJBHmfbion+OfL4YkP5g/MRvH", - "LaWdwm0hyy7wMzVAT2qEAy4ryxk+ql778Lqhm3cESq24K+iObAQ52ttTb5haQsWHzCJzxy8gWsxdc5An", - "NmbvSd3gxNh6g+gEVyJnUrGU51eW0RjQCvHGcD0yBaMMHXBQkcsYgoIsMr/l80o0e/dklItR8P7ee17k", - "a6WX15xpseKySCI99Iw6EDEOw8p8x43ez1eiEPm+ubdCLzCsxoznWgSO6uZ1BMGgibyaWh92JZCA3Nn1", - "pYefkE/P5b0FeHAYeTTBAiYD36z9MGkPSu+kYJ1mLZB1RlufLrhutuq1kkWuUs2W6rY+w/6WQ2umEY5H", - "aB5x8YwTOXOFz1hGKoFu4c7gaTpS+QgjD234uPk+EznE/84cpTJQlmq9KNMqWh6bGS1phnj5gKMQT6SR", - "2LkUsjhwsXmuefAN1Khn7qo2D2lpTMx3SS5QWKDuVlkLw6GzLW0d/b5uXujReCLtgQi+XU54DuTfhZ8k", - "MdyoBf2q6z/TIOtinYqKTUvXjVpuGDd6XS+wDYV3NLe6XjscYcFnehcDaGeBzbMNO7eD0yGwn9Wq/Wyn", - "LAguy3IBB0773fHGPWd2WWPCA08TTsn+XF9P0Z5PpJoAjXKA3GArnpkFcH5yeX568tPxu+lfP52c/+dL", - "+6p14B84CoXGB28+vv70/uTD5UtrMnfQbzJmUVrqAqFw6EPULMiG7NrGhEMD0S0kxNBWiN2vtMoiO5kT", - "xP9UbQb8YBvRsUl0kZcdnNWXXF9XSDLQ/uO+GPFbo8b484kui72/3gr5dOQy2PeHLBcyFohppJNYeMS0", - "sDx2m/MsEznbmwxO6ecX7LfgjS+TiQQWpxe/wVr5Mhnsj5m7JU37arM5ZLfLJFoCYh8Fe43UrQSwRDiN", - "X3oMLvM5wupKJUdu1s1EofvtpVfv4AIO30BDIY2APdrCztHl0e0b6/L8gSURY9wQcM2Nt9cWW2PR7ALb", - "IFf4slwvWWmOPrtKqy+AEQABzFFS+PdEcnbzLZ41/m4YEhwezjWMjsWHxFm3pkUZt+wmZukS/Ajj+1hn", - "bXXRmA/rM+9LgRe+oZStUSLZrZiPclHkibgh1m2apGFbaxLtjQE5vx0GFmFcLX7oaClV7VlB+eH2rjV3", - "MBw06x4MB3/9dHJxefrxw/T4w8XPJ+enH34cDAdvj19fTn86OT99e/r62DwdDAevP745mTYLff3u+OKi", - "8t67TxeXtqCLk/fHHy5PX08vTt+fvjs+P738z+2BYdaN3yNUh+RrH6qzi4BNAW7EeZipdL1SebZMIjZz", - "r82I5t0LxF5gRDFV90wrepfowtxyvgX3jSFxS7M7jgRloDbpBKSoNskOyQMy64i1hPaohYSiJoIioWQ0", - "vn/jTwuxwsSb/nEweGBB1MsuJ9XXimXR5WrF8/XufQdj3AV9vXNQjCvmPlEx27dTo4EtEbWlJJkTFlIh", - "Vi0bS7dkgwL2kJcRw5QwowOIuOsxhIK1Paoz5cF7YXEtkEebeu6XZTNu1m0a31kLAQXLMdgkf7I7ZMNw", - "RK3QKV2ZYLYm+KpV1GoN9v2YJ1eJdH5uRL4D14dDLWtxFvRMK7Nt8hlNXyOli1KxPEh+Q/GreT86Uob7", - "5YD5uMEid1CYaCA307nia0aLqx3htmgduLMkA585g+c4AS4mB/1bITQ5oXeZa8YsYFFES8f/GAtaAhCY", - "6/80MtLUo7/Y18Nf8IvwF4wM84FqQa5e+5HXgdNqFGpSTIklsqaxOGw0UKi35h5bODDqm19gOL7VjLmN", - "cEkdl2ZLlAip5JWTrEyLZgpnyHNT1xmNiAAmG2vF8pKDFSRhNVdkCBt7761RkG7oSBp0RYl3F+72dOCG", - "6t3rgrmAJvwETbOac+vhAuKEh5vKwKvq5Fl0B+WU0qcZQCptSiDoczt7y0Z4RfspeXjaQdui+dR+HF6C", - "FbzE5EWJ09Fmg63FNjei0Xz+HLxhczBQAB52XYed5V2ap7YsAn/dst+qLaxVsHmM7GX5SHnOj5rm/JhZ", - "zj4o0V57kK/qj6F/lRToHiASu91+VgvYePf1IGAEn0p7VvTGJfY2kYlediG7nnsE18C9TVjOob2rUNlg", - "OEjR1VdJGh7aXOkpRqEA9qNE+GPzxmbQt7f06htHvtp2MdhnZptzZosnSYBLNhfI2+RCiK3cvgP17HGd", - "c/bWlA5L1tYXq2qMxuBHQfzKCE7MbgWHFZBIyDSKOrWqdh7RyyUacOyt4DuqoH+Vuq9EMaX6WmNbrdtm", - "Q1h3m+X23y8+fmAXcKvZmxVfmNsV7VoVVDEM83yqg94dqA7pomAUljFivA6ZGF+N2QWX7G3OZZToSA3Z", - "6+Otka6ukra90II9mSdR1RHViin4s6f2spEY+KnvegBNs509DmZ94261ORsINdR+O+ym+qCC6W6IMFhn", - "9vrjh8uTD5fTy48fp++Oz388mRFn91UurnhB0bsjDB2t4HLsTkn1yGpMHS2kHhWzsxpTXQdbJrIhWW9j", - "ymqb3nb0T4w18uhH5iQj+5Jey2iZK6lKzQhGCFHuLd1vFV/bvL758K00pzOjc0Xt3CnpyPfQAy/1TyTZ", - "2MJTowRhkNspFnb05HuYffvnNigJ25z+cxVW3RiguYrXO/fIDveX4SAqdaFWrZFJr4GNcKRLCDiLCRN8", - "kYiciWipwGljU44hFpTsiptPS1/hEBu/y0B4e3G75fZBE2w6Yee3YZe1e6J34a2Aba1p1zZucQz76lEM", - "lpWeOXvlFnujrWSnCbFj1rwpwoXVHgJxv+4423OHbvtfIleEfuCTkIy662hjiFACMiKKaNkeOZIHi23H", - "zUUfblr5PXTY1jlsDPO/sGnWduD1kheXYpWlvBB/ueX51a78IZfrTMQODiXjWnt6LevDi0Bgplpa0Lsg", - "nmxaLBNHktwRX2PL0Cjhk3eWcclmtUJmLpYPogC1YODQhHAVW4pppsqATg8JHw4WCVJlkRMrWnIpRUrR", - "xOka0iRgbfoYHSiRchCp3G+09VOHIToPU+76zKRKopbjdwF6XoD92GvDVLTDXrYqgLwwTbA3D2WoN9d2", - "qq6yXM11G8I5sP3PLQpmYP5hezKMMXPGKyPP7Q865To/YIEgutORYZHAuoypXtarjnTP7UfQVHWZPRd8", - "VfXBWQioi4sTBtROkO2138I1bibhASIV4j3gamq5bSOkBGg/tjpuFOd53OBF9EAnvBh7NINxF/DJcKAz", - "EZWp0yF3zDO33b3wpVjeuc/N5dRzpwZex3s2h9yPTZiWsliqPCkQKxgjhXkUqVJC7D/xXyO2/tRiTlGC", - "7xQaNbP8WsOJ1G6BOU8aE6uELM0QQanILAdHIkzCkIGqZn9O9BJ/n0ijp1NKg3k4+9ubjx9OPs+YNkKp", - "FOlLssjACl5S6hAkls1ouc4IDQfOyXsOfhsOmZOj7LL1rlu7Ufpv1K4jNhYpirg7zfQb+OrL8LFP6J5O", - "IGx0r86/sf2rm9/SwsPEgpvFLqsWt2wH5Cxa+20h2KjhdnxiHKxEXk07Sz63r1RLHzruXBQ68eq37/Sq", - "+0EojpvQFLGFujqWHthx96gWi6Lo1lpD/90++/+ulUS7n894rk5uu9GSbIXwsO30hscPtUEOtljx2qx2", - "qiyysmB6qco0JsMdgFWmqbpFJQTbtmNeQl1i2IQd2jR4etY6i9xpl+2eWZHkKHGrCAKGkFFx/5E2DAYC", - "uRfhQiiQBIy7XE7/mKTjf+X9VEUnRSkd8wNo6HBUKwcw3JKu8H8GXmndEtS0KyGife8xg4klNgcQ4QDC", - "J0IWWGCjNOJPcxw9riVRFPzlJ4R4Yv5bBt9S4mNMEgJEfqYpu0k4Oy6IapO9p3yS8UReiFREBTD0l3Mt", - "IIbvGsBbZAy8QUCmTzAtppSP7y6Mumb1LsZdqYGKdTg+fML+xL49vIMG5pjwNGSH4yP2J3Z0eGf++S37", - "E3tyN2YfyjQ9ULTE/8SkCjo1nsjLpfB0R5DZgPmMAd8kxhVrCDyGLUrthdjH2acPF5/Ozj6eX568mb49", - "Ob78dH4yezmRSLFiWZSDcUwQxmUtAi2HsijuorTUyY14b+3YuJHuC/1oF0DcHmfcPeu/lFwWya8u9mTF", - "izF7p25xxiPLjhyXkdBsJVYqX5MfH3jK+XoikYw3VGuIytzllgLr1EjTAkFbkZlmYnHYWxw9h/vxx7NP", - "Q7Z4+gT+eH32ab+WcnL03KhkT5+AQaf43vyRfY9/PGtVJ4ziMbU6/PTaGUN2VJ4a9hQiZUxSEU+Rg7rH", - "mDsmFSQ8xe/GbAZ0++YjEc/MJuBg7HdvW2oJSNAVaXKVzFMxkViI+1aP2ex2qVIxgiNx5nOUeINDJ6Dr", - "TdCnqm5JMyjTdALY7a6ZeskzUZ2HoMGD4SCotXUO4pwvimlHoGpjkJwCeCMYRGzBrWWT9sbsjGPKOh78", - "gYkcnUZ6BUZ0BpXiSxU7jW/WIkdU5vU0E5KnRTUzrJHZfwwCzY1g9LZfxZThaYsj8nQIxN9L1VVSsNGf", - "WKM29kcGit7+RrzXJ8GmHz1p2fVXOV+teN5jYOlNwKnJk1iQeqlFMUQNseDXAuSPSCDKLSDazKwySRl9", - "s/FE/ohFjSJllE6eYBYHzVWig+MU6x6F665jOpIrqXIxFUpvd9U2+jYXMlqueA7Qf5Dyg9bFVMSj4FgK", - "lhH0HY7c4NyaSIe8pQuVWXXZmjyFjEdqMdIWzZsUdqoukeaeMKp7yrjEhARTykRGSqIwPGS6jJaMazZb", - "8TsKGpoNiew0WorYWyOBNTXVIjBLYnYnO/l4wZZcxileu63Blb74ymg++fZ5F2IFBUEFONKhnx8+9Kvy", - "6Nl3z75/+vzZd9vQCUjsbeWrlzci1zgx9jW210xEwGW3fw8ASY/43xY4n8hptnnLv0/kKGPaDAGGQxDo", - "85i9heAXbQcNidYyhjixDIpmf2RmErIxuwCXbc54CgzmZOthhcqm2UPQnofOWbt952dNL/NM8CuRz9i1", - "EBkl+SDABhEoAcAPXhKZ0trMxXgiZ/YumZn3Qiowuo9SLiWAertLwvF821PAcSXhXQfYvJX7BVo28Fds", - "67XScaF8CEJqHPIf3RYBrvg0TvIDjwx4UI22gZSrgyuxWvHR09HRfJQU7ZByRuGIxD1vEIj0pW235IWR", - "twWg0XMA3Wq/SOp1smSBFwmuPsEOH3ijUIAjwr5uX1z4us0js5oEnL7WSzNfgwtJ23fxJRfn7LOLP0rk", - "aRuCLZEkHXrduhZ1wdfansywUFjGr0Q88toDLNxE6kLwmKnFRBrpc8WLJALoLxRDzY4OJJe6uAWXhEPh", - "mpl3XjC/9hM9kS4F8XaZpCLsHF2DRgm4EV1HdDjO7WAZvcYacgAzHglE0jgP8Vd8RP61WBs5zzS61MKj", - "AGCJFmDsLz/ZgYWPJJwT5zYrEwKjIB6n2tWXptQG6oupEbB1BChYeLvBUKz4nUURgWulxayRCTw6WjfW", - "UROny77vtxZAcDoNFrXMm0SVOl0HVhnafXtH40P2JwsYFQ9pK5lfocDkV6Hpx8j8KGSkytxcWPs7qXKb", - "zvKamHVvL/hb/LzqSplGPE3meUdoXj8ZPCgC4/desJlUUsyGbJblam7+oXI2s4gXRlAMoYXoJdj3xLyJ", - "QrpbPBo8zLhsVoLrkvKUaR8znaQoVbqJQi+FKRxagmVD7B5yg6rFYla9Wsx7aMiew/+psdt8URXwiN3H", - "Dr9+Qf0e2jYOw0a+aRC0EgZoqMswf146S0KViRlKHgwHarHY3KkbMb1+BH3Mh+RjM2lLZbnKAAMzEzm7", - "EbnPbwGiZncSkKDzfJsgiZbz7arB6YKkehA/eOoMr2SFv03SlM0FeLGMIO48r7ojSSn0ve28KdHnG9KL", - "iFVmnpS52CwsXDip038BnCLmnBKFyM1w6SKJhmyZXBld409spXLBci5jtapAOR2Ov+srETROph28pGhg", - "de60/r4M75ZWqs30axOC4THbsyHJej+wAEdcgnUYpAhzyYf0JQEYgDcD00EbYKfiotzdGdOmXhjp/nrz", - "FF+qbHTt1Ys9L0q77YUkoom0SD3HZ6f7g22K2EbgwO2TmG1Tiz6UUSpK7VseRHfcX525r0fYen6dptnT", - "DN8/Ex4inXzwgvOub4rSaF/AQSH0LtvzthIKfjnaXdWthQttDvCoQQTK5A4oI3XBVxnboxDsfZ/DEzT6", - "lmvmfe4evfK7w2dHT54+C4S5RvhIPf2ilMkvpajb75AVrTLcFcXMPItWWTri8+joydOdETxcQvz90uAv", - "l8KmSdSS7WoBLpWUu9qz/4ZhL4/AQNE3cKNX9n2HzNvwqf1dKzn1PumdutHwkLtd1RLYVUYFCKvkhV5h", - "xoBdAEALM8TWuCEI27Y1FbODdK9lSJpLoIktDe6ammw3crJdbJ0+TkEPAreaZ15VsWga/6m09ockx+9C", - "Yu0F8S1w+PjesNLCoD39RrMiv3Wg93jrDURYpWsjwaPgKOLQK+bCsprDWAnq6pWrZK3XEOHFKaQLPh9B", - "GzAua1MA13gij9OUDP7o/AdmsIjn+RpiMI16Yr2pUHS7LaNXAMWnjvAJMAxMt+T9nqHBgXSLkJ2oxUZi", - "TRk14qLgSmrhVeqbaVy5mJol3zN/GT97jARmtkfGmf8TNHV/97Tm5hDtkupsiRpagostb4RZW5baoLkf", - "Wokyal+xPfOvF47K9CUStA3H43EVSn7Q/l7y06uP57eHf/nxSo3H2/NoTJN6dXojxQb2P8AC73J4tJwR", - "gFJBI9OHJiO8slzKtSuk36UzbP2ipffvxE2dNzSRC9XA1Xynrq7MYXMj8rnSSbFmKXzpr8tYzMsrTAQ3", - "n9/yHIRAm/3sp5Ve6E55a3B/tMS6A9bIvI0jfw9AQAicnBjsH06NP+5Dgt+H25faF/L5XiWLBrVvOxFN", - "fVlAz3svCYuXtIEPOJyFMFKpGwHPeofMAW6JLZznJlUckcByj7Tl7PQTSfYy78oCaqKWhI3ZXcpnzAai", - "mDfP/v388uA/3h27+gE9kK8dKQuHV1iWllcJ+RZuEj6Rs+MPl2/f/ef09MPbk/OTD69Ppv/x7nh69u7T", - "j6cfZkPWfGyKcc8nchb8PT07vvwzmQPD10yLb8X8Kiurjf6Z69XBz2L+49kn1+5Ews8ArhfrlyzjSc4S", - "xEWeSPJgTJxPbTKYMSWr4omnTLaDTHXUYk0qY980QeJNDEgsErMpMJ8pKmOzbO5S81/sFuxvvdqcuQoL", - "6G2AJ9y6fkzjFhyxYVa8qK8ju4IIezLMnb26Khe+tZovRCGkRsC65XqeJ3GP9p2aw6iNN8glu/A5ekbI", - "kOsys5NCQ4YzWFCSNvir8GG3cSF4i9jioSJt8cX3rCgHqj2E6hNViyNEG7IZBEqBIogxbGhIj1Q6F3nh", - "VyFJW+/53UVCwQNMRyonzoL+tCxIK9HdLQTLAbDbRteQq0IHYRVG1TKbDw5B2lBwTM52aNSXrafZX5K2", - "o8zG2hJX2nUiKynazttsfiOKOsgmx8EHJbi0/3IsdPAGMtDBPzl+FRDPgahDpHM9Fupfg4C/DdupEhfY", - "fy/98mx6PRgOfsHYvKPnPRp0biSZrrG01ZbEoQ2Uq2VmWgRcMA7rdWQbFYWk8M37+qFxtju658Kz68tw", - "cJ3sWjUsNYikWal8PUV4hulqvtlGeyZy8msfP3tlwzaFvBGpyiCZ733yasz+S+Sqcq/Yu8TenHtPDp99", - "b971+qwiH73ZWkfPn37/DB67qFob/Algba8/vTnefwn/c4G1XAYNKZTCiD1YVgujclJJ0XrMPkIMFXbj", - "SqiVQO5BYhqBcwAwRysiVZnIAgJEvdkZyXW2Yf20JzxUkX8twWFAvea2u7Vf4niKGPb/kJUaGWgTyWbo", - "QVa3UuToYT7AX3KRKfxhFkj/XtDlwJQSzykiRTyZt8l0v9Q2dv81VjkS0BeNczDdCQ7i+Nn83H6JaBA1", - "8RFW/7AHMgpyhXbDDwATmmNp2s7kZbNEojQxTXdpIkt1S+vLkUtFarXiMu7gsaINondAJYAPiAAXA4eJ", - "b3QTA2v/uQO5o0G2enzDkxSxUyqMyJrtkc3a8YROkNp4Mthvy7tpV29alNaUWxe1Y6jzsuVBHZq8IiNs", - "yhXqwvfxQBGW7PX3Gc4Gxno9psy156BtOD077O/UXFufm/7QF/oNPU6U/MbLj+vWheBj5X6flrv6Dt69", - "e9812NUAvkabOwETG8vXwQME7p1xH2RjYgP+fcYEKzv4+Pq8a0CoOa2j4aT836uxpjq/U9qbRNzKv0+T", - "LsQvTy7EL8xW2z2K1KzWcQw5n3+fdgc1HuhMiGg5KtQIAkM7ehC2saUTW4Bp6tdrcO8F11e4osJzuHJW", - "VI68cML91qmN6EbB4CwXcRIVnVl0HVwziLOlFmwhMJKGEOnHRoFlR4eHh4csV7fVa+n+6LEhWpUpuzds", - "O3VP5ShxOgAdG+NS8FQhJ6CPobR4WAEzW8aLQuSmwP/7t+PRf/HRr4ejH6ajz//nD4NHpBRwc9ElpXUj", - "RGT4abtXzeguubpl/iXEEdBjhn1kwqgYVUqJmSxXU3S/6tlE7h2BBJKLK0oYZAdkhB3iq6BgCz3zfEDw", - "y34tBugBa6DxN9fXO3CP00K4NF91TRIUWR3NPjOm8nZj1WXO7UnG3r+z42/GRxTc2q5rQDS4n6Yd7PGe", - "EIb2HfLbJpLMOiqPRb6bvchMHpW20cFma4xUWq6kZuIuIz7QNbncqHfbKTaDpbWpRnL+U8KoTbTOACHU", - "PMjV7fa6EEF3usj5StyqvA1HBjF23RtoGoEIZxWXkaj2jp2ej7s4aR5pNdIqrExMddD6LcpLalIL9xH4", - "O5BHal6mPPf9qxA1+e1urizY7tMGEZPf7OYOQhFls5XKtXCDRtolap7fV7D0PXywjFHd9g05o2vbAyMl", - "bpisci+N+wsUQS+2LAK1yorXPFqKLgr+D5TM5HJ6yNEd5K9vNPwRp/z24AqiwuzKvnD1YSpFoOh651SH", - "9WDF75BMum6/+/boSSPKUsVidIu0TTkgsqqcpb4dI2yEkGAUGzPIbafMrTi5SYxqnxQTyaNcaY2+mygx", - "+o6VyLEbllc/iQpNjFNCF8mKUCvw4Mc4+1RdJRFH7v4DzHk3owE9GrNX1n+WAj6vWT/oRit4ImGwOOQt", - "WzDXIcb7J9pmyXO25Hls9FAGJmcF/LBESZ3aRC4wZdbDx7/79ocnR98/P3z27NnRt1sJvhPZlhv5/Nmw", - "A6qWJj4l4cPmJS0aa7BO2NxevUvas35yZKZfcr1seMvPamvOob/iImcz/+3MMbTuI+OW+XFEVO8inkgw", - "r5r5Mp9oVspY2A74LCQ0DHzcO9rH9xzt7sycEneAN8XlRKoMCLwyfiVGPE2uJAEfYGUMXmZFLog1Si95", - "LmKbZwQWUL1MMrP2JjIVfIFBS+/OP+FSRIbuoppM7A3LCziJYFeahQkG6Ym0SsqYndg5QlO182yaDRvk", - "jHnEiZeeJcw5SGXgG03AYObtBdCpahrUmM00TE/oh04TKXg+0pR7VUCu9KiSwAWcnYTPpcsEkZ0BLAgM", - "47RJAa1/XkLsRNXpipWaCy9cRjABrRdLUaQOmNkR76OWUvNdxkb1TFbCKJtw8ixaTl13AJ2LRS70El0E", - "yMoGb0zkMilgs0MX6X2WikXBSukicuEkcMz+4i5LclHb5j88efL06XdPDp8+//7bZ9999/xwC1/+Zkfe", - "Wa7MKZlftoZZuNdMh+E9L4TwgBe56uAzjRkMBzciLwCPS6UpB8whlQnJE/pHrkr05s1FnKvoGsKtlgIY", - "bbgslrnKkgj/vUjXECQbXW8WUc4Fjz1fy47hw+ZjRJYF5pO6Rt2LLWmFzB6xuBtv4S7xwhqPH4GE5HN9", - "GLptA6v2THQIl6KoBx4jg6LRvD01NzDRvqGwM/0C2XZbA89m5s1P5+8024O8XORK0CIqcyRETdWt3p9U", - "otr/RuFu/cLUvnzeNTsgDAfzhglz3W1A0f7aEAJHh0+ebQUP2JzhjfajjtTuwBDq/WirxAhDalEcFLmK", - "cgAtHmW5qTJuz++GsMwNbOBGOaETsUJECperLrgMfR5mbVzmH1+fvwBubQrXNErqx9fn+7DGlCwL9vrj", - "+ZsXbDJA56CeRiqPRzdP0EM4GZBlIxJJBtwEmniR8eM3Kvrpr8fh57GKbn7h9DX9BlskURJ//XnJ4aoF", - "cnFV8PTf8L2D+ov2cy71rfVhTgam8repQmH5ia/64+vzSps/vj4fuoevj88uTz9+qLwQ8YwAA0yRZ8nd", - "kwvinp0MoI1wBKuFH+cE40H+DRvxXikZ54KvzAdvkMhCkIy5gvDlQXXrbSm1v7VstT31iE7ZLYDqDbjD", - "2KK9EwnmkCkpAqsCdmznlKHgwviq6Otb9vCqmpZjdm07ufZXo6Z8zCSXBxBKBndwC9qzSOONZoCWtIuO", - "DBUsy/oFUbvfeGqxPThXhsEO3x9P5Fssh+fo+i2EUQBAKo5VYdQ5T2YnIVmZadsIbXM0LV7fwrsnIZPC", - "Yjn6VoPxsbp1fxushCzHcgVYMouFQI4OWY6zHDJQB394Ov72cOBg1wd/OHrydPzs21aSmFxctRujTyUo", - "06U57I1gQC9iV+eqxKGaqztSpHMRqSuZ/CpQkBhP5Jkjip2v0dI8QmrDwI23d1Hmaz5kZzyOU2Fug3vl", - "ooq74hza12p/Fnct2+fELQQn9yCj2grxr/1ReSpvAAkcUg5eMDuaW0OGTbVblj5cMseW7csbgDZC1td8", - "DfxuGilJyuI0JCHxas6TLsnFfxkA2jitESJ9AJoTTBJoa9F8JSYSjL2Mp1oxnfxKNNe3IrlaQqClyq8Z", - "EeqgBmzWAOpGdMSaqy5I/JUxZOG9YIJHSxeFGympy5XRAmUAhJKqApYcL0CF1gU8LqXRthCSxL46V/Ea", - "2heCXUHrUsXjIWrhKMLP14U4QDwZxJ3imOIE+dXYDMwzL5ZcugphSBwMCOHRmE8Ko9ZAChLigIi7SACL", - "Z6LRUG6bKu4yIXVyIwIwkRU3w7TkpS78mAZcc0qK3OidGAZIPRj5mAbmCORYCO3PhEuAsy5OEY9Mx1kk", - "Esho5gU7es7eJ68m0ty0pos01Kw60nARF7cK94ses2NmISV8S83ymMgo5atM48RCvCr18xsdNgNnwbaj", - "UNQOTNGdSD9wBGEmEgxsog8SbUdMxEPA2wSYS48FmIsAuCZZQdx/IVKy+3x7+JRdiBwse58kdw5ooIcX", - "Rb4eHS8Kkb9gRy9NWyzEiyqYtW2Y1cy4ZIkcWQveL6Uwzb0QhenPofkPJf6xuSqWtu166OdrInHIHRY6", - "AkhBAhaN6igWeXJjFoQZV9yTNmwx0ezpk9p9YTa/0z6ePfnh2Q/Pv3vyw7ePkzf/ZfP5lnN5/d6c/BDG", - "/Iau241YwpsO/ADZjJJbuvKsnaISIbUQ2TLiQNR1CdjbiYSors+7dLZTG7ctaLlz7QCB044MilzjBQXW", - "skoOVcbze1DDd05Kf451J8O6QuoxLy1aKQVMVPXSmyRWuTl2019uhXwyujkaH7ZHeIo2lnFz7zN4RkJ0", - "Km64uU4oIH9QTZrCaxykIhaXwghXoremg00YBtPXYzl4y9RutiXSrHc0K5kax6brHag+8KRF1QpHLYAn", - "w0a0AvXVYgA2m65szT3GayM720aLyL2W36vj49OD+ZUY2eejmyej1dNui0hrhq753OgP4elitixZ1mBW", - "4JTGIFxEMgJ12HlT7NVthCIUpk2BkJRuS0T95cCTBlGGszngEKQx0U6Mql0BfxucygKc5JZnkwgUzaWe", - "y0ReYR7oGyEy9xNblDLm4HJJtXn+eZdohY4deyF4Hi1779l6MxHXDD3aeueta+ew10Lc1V5hvqId1GGw", - "8PvpPid2t81iV0uDdf3/T7Q1QBDefU9loi83l86OJzNU2+6UMgdF23oynxRC0kniufFNG/zKmgu+Gu+W", - "4bXxpMbm9BnEzZGIrQYFO3p0MEIxGLBpLpjqgXVpfra8Zj+KfMXlC/ZnkaZqyJbqFsTvtSr/bbdTactW", - "aQ+T3Rwl6w+rdPXL1cEi5XJUfDsCH+ZI/1LyePTL1W7Rh30nYNcjKlxSHYfUfU6ncE89wvFkivsfejyp", - "Nr5186vF9bWYfhrkAyTgDpJNCEHK40sHnS81aDpc60QXHLhM9FoXYkU851vcrBhg08WZ0yctKctFhFBc", - "bYlK78QVj9bMRhogYx7IPA7SHIFrMZ8p5no5VzyPdZDLNJHNZCZ2WljtGI4dUzcxEnEZ89Ss/lhkqVqD", - "UP9yIldcQohE8CtbJrpQuYPRLVgCKj3owGP2hmodUa0Ta0Ag3j/NOCanjyBISYsCYpEssArZ3/4rubJx", - "TEMGdhMK63LSHQ7HkmsmFdlyujB2eZa0Q+sen50CQi1sOMiHtgIkl4yXxdJoxTBP1nB3DM74wDx4fHba", - "Gllp6uyGL8EF3CjMpRUduHwtjXaYytm6LIrsxcFBqiKeLpUuXnx/+P1hJfsxT9raRJEx0yxPVJ5YPN37", - "LkUS44MJC4ymFIJms0q5C8sJ7g8XVzb0YTc2oZeWhR4SsD6hMofZ7S9bFgUknbWsid1OcYphO6NhOpFF", - "3hrLXXfs9welxw8v6DsPLXYtRDblaXJTiw37dtWICfuzumWpQuug+cw6EdCCCGTqmO+LvD0p1+BrZnsf", - "IRolSHvaH08kplqCIFGFyS7JJOkIAsP4HEjMB3zrpFiPJ/KTFuxH5Z/jgnzBJoNvV5MB2/uWrRJZFkLv", - "D9lkcLQ0vx2xpSrzfUjSnwwOJ4MaCC5+6wyH8EpoPEziVOBR4kPGaCgg3EoXhPUKj0VMEW+5wBhrcD/p", - "0igqFdL/Fb+bYs+nzm9f2YQwJY0tlqqr3quAfhj56d/ovti8T9+457W9ytMEaecmcuYsq2O3Vccddc7G", - "7IO4rUbT6omklFmzkAixKhJajwC8JzC00/Yds9MFWnd1JuBg1RNp1pglNx8i1POqhG17IzzqOFq3H268", - "pVGtTGUVR/ywO5Cl4GltW1EcLU8DhAdTo2Z7LvVpGCBpuPBBZX63SVMWk4f4RyBCcJUUAQPIEH8VXBej", - "XEQQ+TgqIYDTLPhQFudy7duSaLfOwbBxLViuIIjJ2t3NdV9KqA+wbsO9dnRYH/Gu8SQcAhvA/Hh3iL05", - "IHbVXQvmlhu6O2Roipon0gzTX37CK0JHOWR3oZ9NO8BwsBQDqYwOsQIm7cBtpmvgqJjq5FexQ99+VCPb", - "cvje74CuDjqOGfSZTKS4Az9Ju+fRnuW79SY1Glo7LBXRfJhCzTXGE7Aodeb+6HJuP3Bx5rODVXpwc3Tw", - "x9lEGkkK2L3iREfKiNxhBgGs09lv2J4vB79JvhJfDiiLAzfl+O9aydl4Iqsr8v87GGP844Ebl4NVWj+L", - "29/qxIWd6sKoLFdWXu8Ip6jzkKT81/VgOAB3fwcbSRf8hkUloYrXjn4Jcg8zi2SDeVFIsZ7kja/8PU14", - "0AiCwzN4kynJYmFE7jHzcoRVGDSVaU6PifRXpZO/g9uVrlYNL7trFZje3Kq1gprKWYay+wom72cEq8f+", - "GEnMdXmCAzkZhC2HmMpTOPnsCVs4dJdimavyKuCvsA9ueb4qM0Dh8t+ja9nKK0PAc0eh3o8Gxpkdsr0s", - "kTKUkvZN2y9NozA4k62SO0iCgDBxSHvgvzoI8SSgx8iUShtBKc6WjpYPIUc3R+NvBy/cWlolebE8iJZK", - "Xq9Hq9Vc5AW9C34kc1WWPB0dDV7gsmtzLXojzC5b3MJBVfdzQxWpbeYgwRimbPabr/1LgHYwYyN2UgdH", - "2Pv44cN/7Dc/s9chfPW6jlDR8ZH3Xpivzmvp5Z2f+fxjbKMsIPkkF4iSS5eIjzUK4gewmLaWWMuXKbEr", - "q9w2qMe5Rh8Ylfom4ewmyUS+DzIUcJEJMHWma1CfIe8CIDs8todZvdZZ3+uIrGXkh/C6Kt31CrynimhP", - "lIlEOGiWuVMzSzIBwIwZQP5TVFKi7aH1koUKYFJssAZUwFnh5OoA+dHEEuNgpcxZExxKY/ajE+hMHxhn", - "RSLXFfo4F5ujKD3lABNQMHpFG6HFdKHMcNVdi1yaiq1QjTFFKMiIGLEGuSP8QF3HV1cFSLpOZOxKmsi5", - "aYYP4IC+QHloQucFwpM1ItEdPJZD7XO2BYLJQwirCpoZQie1wxRVUYkQH2z3GHYHFNaijNcJq3rmRNYT", - "DsmAKXQxNQOtLI7AvaTAA7sSqKhuaZCEOgrtgc3zEiABIXAnTXRhBpqYjIBc1zEzwxkAjoNM5b6qdn5F", - "/XQa5QKiPXi6A4te8E3dxmstXRutuRcZz7X4CS6cNiIADc8JkIHyqnjO0/T/Z+/fl9vIsX1B+FUQ3H3C", - "kj+SlmxXdZccFd+Rb1Xu9q0l166YaXpIMBMks5QEsoFMSSyHJs5DnCc8TzKBtRaQyEykSEqq3T0x54+9", - "2yUiccfCwrr8foCRmIrrJxRjBxAAQ2Y8qwf8yrhJhExj9PSZTON0DEhP/+61aQMsXKoENdMNO6B2fPUN", - "VgZ/JhzYWAS3ubVLMS08xoOo0Q0O9yYJCvSG82RFg3SgTS62LN6ZHaEJ2sj8NE2+i7cvZ7nJWyYqJKMB", - "WRGH+qV4XsLlNLaGMXvkPnuEof8qVxrCZhMljcrFkD2yD4NHjjffXc9/Pf/0ccge5Wq5WJePag69kVgs", - "sgQiCy7E5kfcNAXP7Jv7kVSqoJqAQ2rcQOf33bcNDsCUs1gDy7lSRRNyOCjc7yX4Iq7LV7208OK69LFL", - "sN2TFYcQXM3UYmFEaWK5y+nUFotsH/8x7g0lrY6qnK8WfFxIT4mA8MLeEgeesuxwECdb4rp8oAahLhM/", - "ItGIZBCU8GkZTNVuccaNrg/rabt1SwfLFQeJbiyZxwvBiC/vLmwuWM/Qgop2gmVuBU/fjsq8PdI6CBDv", - "8s/PVcTp/zKIcGf/uD4ess3xkF0/HbLN06924YvsGsgTlU4zyUthIl74gLDUp581ss+eRy3sC0ASjIZq", - "QYw98uj4chR1cnA0Oj6MCcGcz2MO2HP7cC6zhMHveCPkfANQvZLnG5PZ8zJejoewH4eszEornSBP93DQ", - "E8vQ22mXGNCGkIwR0cR3OCzU7cusVB67akul8po+6xDVwC6FVhfkhb7YWWl4Sx+8FosM16mfPcUed5eA", - "hR2sE77hzTPx7U8G9qHjmb0OQ2Z418VdD0vkg56ZfMXz/LXIY7EE7+zLBUPDsO92+pCfgG4twdcx1WTv", - "GXUdqWfWdqgn6veXDrfSAcwkvBu1KZELLwymScV1dCv3hAK9A6WEfJn1wGk74xnu14m27gCoLNbl3Vd8", - "+5I2Z/LWpXVN7by8XC/7YpvDen0xUGoYdT+yDHFoWjcCxArrm6+958YT+HX4utGauFJXgdAAaz0Q840Z", - "0eGAmWgCYPCTwQlDBN1UJABJ4iIXSoWzyenQO+gK/FYqKepvgbNRqpK+kBtsEIu6s10XB58SFg3TNew3", - "9hMc9wl7C94+DiRI2SJL6mUuFZtjRKlI4TGlpKB8+jbSPbKJevkSsQx3Vm2nZh9IYsBaun0SUzAeUm5+", - "jW+lt0HPm2OKb+svRGrs5Es4P3Q93d7T7fDCXzze4J4xkBi8B4Dyt8U+5lwuK0/k1HTWlph6o4DwLhEp", - "84XDd4bYwkvnFUQaSdHLLNejifoZaCf+weAafemGG7JSpXyzPT06Gle569L0RlZiD7dRruAigbA++PX0", - "P4fsw+dnQ/b2/ekr8sHuwsPSv5B4jP0hdgXxUg0XhR24pKEhAdWnohDS3s2HOyx5T9igZyYPT0oAo9kT", - "stmABQ2ovY1y5P9cpizNbPvzqgzZ0D32ufd/1Nj/fgwIc/LkapWZQuhRmcnNoA1OuVsEKC7yzltlvxjQ", - "L40FaubX7x/62ZEm947+bJ7q/+9FgMKMZkKWryjZ7w2QP3WfNHAxENpwSd4uTRwwwJysNNeBc2vkczIX", - "PMshF71ra4m3dY7ATJSKMdKCp4iwbYuzBBkfu8cXI1W71f1crbls1xIWiWNNmxhnx4d2n4Ic2QrZwD39", - "l5uJqSuDRFv0Rx/H04N1XerNFMLBPIJUDC8N1yIVOQeFdJ3leUa8slvxL+FbJIDsrDbuZx+6yvGc2H1S", - "r2tAzktjtuW/RkkLw92Kq14vmJ/tsEudKbh1F7eivHZ3JtTbFQI9WU521CbAYcOjUEkERot4FlhW1u4y", - "QU+2ggK3Zrlazhw2JUXOdI9E7rjddhILyAQHZkSyFe/0GVqWex4opuR5/sbRYveqBjjM6W/+mDgb9beb", - "2IFK9WaqKxmDhOzGEi81pxfdTtfDKw+nH7sPTKKKrVPjx3sOpW8AWKyHTLW1nbH+r/1z+YHLbHHbHGIn", - "dp5Kuj13nh4/NryyXoskSqOuHW/GrvVShH5ffTAvZuptSPv3F9bCkfJ0ql/xYo/OvuYlP7efxLsbW9Ow", - "77csr0h9h+99SmDji3TapiF7gGOQud5OeTkVhUpW066D7fvng9v4enZatXd1mvvNcFDw5IIvxTTNlnQE", - "unEQVOQBWuk/tHeWA8ZTO9cXO8wkGpCsIgBJ2sSvbLWuIuWUT5RqVRSUYmpvvamtTWy3AMBUtGamO8rO", - "7LpB+l7Htu1fVSZf5bwycctTfe3Bzf+bwogmNMVhKKxCsxJFwDjkPpZJKfQQ0CYx5kJny1UJNRCIj496", - "rwPWMHnB0xu2SMVVJj03523LZscE4JJ2swIG0tR+uztAYzApN1+HfdAOBK9kq66dU6MrvsFBjicSLCpB", - "KTsayJ1Wzv+AD6BFIxHHFsXx93ByewV/Z0L57eYrGLKSqXcawHJNawSs1iscUahK5eiMajsGrnNJUM7B", - "OMcT6eKq8PUu0iGEdRM0VrQGroVrIm2iXyGWqwO/an8Hz/N2880UTzpHYs0zsG1Z6bFXNqebotwRqWyb", - "4LdU1H9bxhXuj6FtAYlIFO4gO9bmyz+pTKnWROfRkXJ0sKarbDu6ie3hOX3QkUFhf2E/9QoTv4li8gR/", - "wuPCS4DvBf4PNhfllRASh2timvCCdmPPZqy3DwDcHhQ6W3O9OawncI1cJDI+f9MsikSJRxbfpdsm75Mr", - "2zw+WzuM+/bALrBIt3Y41s/WYgWT1exJ36K9rfdwu6fwg+0N0uzbf1APCQCC7oQIRQV8O8Vz2q36b2Lj", - "OOOxpCfyw7tkFzMhNeHBHm5VUPnV36HczXAAuRix9zymoUhPLQF7syvlMlN3chfs48buaGh+4p+d6J1X", - "al1wnRnlLgw0WaMASNwZwphg8c/ZCXvzz4q3PDkzCb98VCVytsDf8nJ2wt4LA2h/kv4kwr8xpYPyS1v+", - "JwgR1fUnS/gk/HPwVWDxgJFJ+P95Cf/PLuGyhP8XByOw0/RZq0UWE4r2x8AqanWazJRZEhEWLhsuaiz5", - "AlmShb1pPRQcbDuY35bZpLEL+1ViOHB2r0xNwqUU6W10JbCnqFxLanV3fn+beK7v12hwY+7aqm0PBNMO", - "7UGcAxWup9hv4X2arCTVM7UTdVvLMJHQvL0sifkZJOkBMDmIRfkEEPlBSzu8Wydg4m4dP8xsfzfg9zv0", - "w9/olRHp3jd6VDD5EhHTK+nliBHUPiwoguZa8TThxgqLl+7fhKHvrzL7H2YF+fJ2ibheCv9kYOylIOLK", - "NFvjg4v0AIaQxez46MPLMft0ALWiFnLoErQKobFqgHmegcdriuQJsxP2ixFsDtN+ITbEqWAg1h8KAiZq", - "0AF6gFyK8KGCJUmHvxDAh6WuXPvqUuiV4NS8WVWLRW4l5M/crEY1czvkZNKo5htfV7N5mJhRMD30lHAO", - "I4Mvr7W6hFCGJh+BXwjn96NZsA9A7FWv0K2h+N29BM+3ztX0haJFoPelsnNv9+wJzbwUenbCAGJWi7LS", - "sj4AJAYgkSeYCryExMJunjP8BBI97WcYEebF4pB9/OX9ewS2VXJUq4/+pEFl8J99tQXyrq86Lz4ac+um", - "A4QPyd7obP5NbM64jDkerL6j7U8uhReToUCZoKD1rgql1Tr+FFFxU2TngKNl+yfNi5VD2vu705Zuo3/t", - "TXVeaCFGdtHZ0lbqdDdkhK9Zvi6Pxk9b+SnORzQe3N7PjyoV53AOUVm6FYx2jyRt6BJ22lD1Y/aLrMlN", - "iO/HnnAS1Zjoy5MVyjF3T7ZCFsD45HWPlruYJyWTKnVRYSXkfn0ijjJybQkuESAVccl8OzsZ+mDOPvNy", - "9YbA14hqp4dgYAergZ1HQEbYREZErhKAfVALHBrKxC+7Tt+OL+zde7qzOm97C/o8bgASCmAemGqxeMEy", - "ecnzjGwKMBKlw5Ub3AWqlHppGydrwbY1tUcAn17ECk4djEW3uuwuGBWg22F5GGdlBIMMrVS48LA/WaVj", - "WoprWyMUnAzYCEraX5DdtFkTfeixUkzny7XQS3vEMJkk+vXCaiyRJoG1CBjmLwHIM/wGTqv7ZozKAJyR", - "keRr4SkMsCrOAGWFjjgeJfw0kpiz67Ld3MHqRusdiLS/12iEO4srHDtOpfBXxBg0Gisr8G2YKTlCtri6", - "Ifb6/H3sOe5MeQ9+/uh1PG1Ay7YcefvXJtKluG9NVgtynoS7bYDhoOCa7yiGYfo/Y/mbgA52R5/NZyx/", - "XoriHquBqtcUUEjEH7PgmOwBUmXbmPoudqDCtBfG/avZwSVQrw46BuIxlsF++fog5x6JJT564rm7Hn64", - "ZT1VOnpPJKsC3QWXFDJNI/YQUZSrSEwkILElZNWCNYWW7nPJpVRnt7VfHZC8LxI8eFNVoWW5mT2yR7sB", - "tvfueu1b+/72ELcH2YJ1RNmP9rsuMf/uPfNirGMMlxiRCgUQNsGB9DgtGq8yfHkrvbNK+CZdinucbnFZ", - "JwDtPpefubZXufuYCXkpclV48rpLIWGzQboPkV+mS+Ehye4xxVGkOffcsWrcna9/kOGxo1Mf1fAUgsZI", - "XidM+naWtvp0IdhIZtxRe3DhbBue9uy6N7DXgD/S9Q571tepLdfV6p5brd4XsbCzpZhrniWR3bNQOUKu", - "uX6jojfEY0S6Ep4jLhn3NTUP1B8x+z0uvU9X4DivSSY5A1yrEbkrVSpeMEXPQjRKpMLvJavMZu69eB9d", - "Nrz07Ll4mGsOUbVrFqWHerF3VGC6AJ1o6d50iJHQTxsHcYtgj0szW2KdSfCvhOYL54c/f/03MwZWDqEN", - "rA3L0EzX6lfq4BCjZg8tcsGNYEbBJZ0AkM9lpiqTewwLYNPIICMpUbLMZAViIhUJCAyr7bs8Xz8JEPLU", - "YLSGBYlapcj4dj8lhLRZ90R7AY8utK35OcGNunssN2m8H2z/7nHovArZ4QCvcpS6PkuK55T2Hq4RaFSw", - "dm56dx9CXN27n/juHwv8+oeNxUr0+8g+pS522WO/rlQuRvhCrz16GDvb9L+9YGq/kfrnccHhEIdmgEeQ", - "MHdBQDk7+Fv2GTlQnd3vgQ8LDMQ7eLZMyUvk4YZwlpTonfhyqcWSl+Je1qiHezXh2O9zkwBX8wehl+Iz", - "X4pzH2PXSQQoK/AfcdgIOWD1INEzGqCAXHrMzqskEcYsqlpaEs5qqdZZghQXdoInA4NFJ4MXPjEBSuIb", - "GulvpJKjp9fXjtDJ7bQmFzTWExW+wfCC+OmWVTUchbfsoEtqIxOApENcFC0SpV20lLi20pPnlG9gCBAW", - "AxQIpNbjjEFN/vHoP7W699xeUEqjdwnrAmi5s5ARbA6436WdkiLnmcTkWSv6f8+KUcBBbv8+kQczwnYY", - "vZGJSjO5PIGSs8OwajZXaUaTTgiZtuffPwfyLUTAtdcgVg7h975T5YqXCOLomM0Q05N8T7Bez4+fjdmb", - "64LLFG73Uhg6RqW64jpF0CbgKNPI+u9An4C7jXx8iAz1IshsMbx0gZHYnJlIaPC7o2c4x7OAt2vmwTHt", - "7kWefgJoMRnc8LykRmByHj/+WV1ZbeNK6Qtz8vjxRB5bVUSm0W2wUZVu74WJfOqUF1YVRujS+I+gOXGd", - "GYA0hK/BRzqRz/w3qbDny7jt4z4lhfTCO7OgLjhcc+SU9B2iKp+jTJbiupwmlTZWnhkAR56i1XiapT70", - "RyLVI6wATsSvp2cf33386eTxY4hmMXwh6CrwzIH+vJCrUV0KnXMIta07amIhpbfkIbTyTxeQ/jJ0i82u", - "7LCvAJl3LmiyUu/qX3NAh0tWrmU8lwk3ZIb/T55nKS8Fnsm5WPHLTGnPqaPW9iVAdLyvViK5YFerDEzq", - "uAxXhLCsxVpdihQIju1pqtcFMDO5VcbwtJHWSPYN47o8zSCcUvMNPp1oRsDq0YqY7E3QaK5lJHn3NTr9", - "TUn9x03i1GCSes7ADaTGkEpPf0EnvlgX5caZuSYD5B0+r+bGFpK+sDnp7LZmY06AA3GzWGRSGJarK6Fx", - "xmxPg82tnLPWg1UKiT78glt93Etp9Nlb+QlAQi0owALJj06Ojp7F88tgVW9DRr3NDtNxprbdYAVyVBH8", - "9rvXyH9C/+nS8r9NBu5v0yydHk8GJ+zbeDy+GbLmL0/9LzdOkGcIrYB7G0g6UYg4T7fxlpFcXGeJAtUn", - "S5jSqdDA3G4ANtUA6hpuYwkQDw4CBgJIwFIBRxxxQCF6sMYRJZwrfqnspk4rpGoSADhmr3wXgoAi7ZGh", - "wBAAj2Sgkix4IjrIovXiHdfJ84P3vChVARIFYBt++OGH8Q8/oEmDij8Nin9QFeTGUemnncLPgsJ/Exsg", - "l6jL/xnKxxz8VoBMd8oTO9/IhFLE2sG9tP9i4aEN7SVOk0yiJA69Y6WVE49Kd0XmQbZoCJ04MFYgrWLy", - "ha60K6GDimOttRsbY7SKizBv/74PtdFwYDXIPTKb3kLxT+7yitV4ITZT7cJJbqvLh52Q374/JBBYhAOX", - "OBzMRmAg3fhk1UBB6CjMIojXfYm2p15isUzi0YSsfDzPk4EpVVEQ+vDKamJ0Y20mA/tzap9tlWQjJpW7", - "SNmap2IyiCLmBBJ/lxvIRd6gCDgAVzRQZtf6R7QZ+5IpbtvrVIDNRcIrU7NlrrhZ+VjIg0rikNJeHDh6", - "At1qeYi+m4J3+B7v28HJwL6/x6+rejei2njbYDOJJezJhkQnO7YjtsORbqfYYd+DNv2ZH9ST7o9Yc7lp", - "wD3S66Iq0D4Z0FvvlAX0q5g3PrwZdgAx+lhusFnmyG4OICaiZO/fffzbL5+np5/fTf/25v9gQl6yS66j", - "+yxw2bmQPGQM0mknKo9IDeETvNrxaQWnGbsSPlXDakQRfbIicOS07IQFmsDSa7q4kzjspoky6FHYi1ZF", - "qF6kp9JcCb0TtNTX4dZ0NeoPzc/p53egidNfMwDtXVRoWboScwpbGdIDqg5KaSNVT6QWFOJi32eLXF0Z", - "eradi7Iq/HstW0pWFVbCrcqyMCdPnuTQ9tgoeJ79JEq/R2CxPK0U1vZaJfbx579OVWLGQRXRQDqzPi+V", - "vsWOAvkFScnen39gjiLIP6E4qFllZcbstSjhtFn1mLLW4eUsNFlTPMixQ41f2xmLBeQDYY2Yzqv8YppZ", - "SV5O57VJuiZR6o97phqy9boCJ8g0V8ss4fkUnvSRE4g/04s/k8x/ad/0FIfLrzhwvcFFU1RzR+3ZsA1u", - "75QWPKXsmf2/msK1B/81La/lHStJcE0p7eouFRAZjdUE7lgFUCkUPBH3HQxklN7x20KrubhH2/AY2Od7", - "2NO8KIRMp7wsBSBi22NylwoQC2gqZKk3966FbJJ3r+ee3VjwPLfyZepJ0BDQ+SFqFLm6mpYrLcxK5ekD", - "1Ohfi/ZCf4D6SjV1kuoO00jHGQXdkVWk7vR9piRMfq6WtZzc6XuSBmsCq9jva9p/obS/08mI1HP3AxKp", - "7GFq6Wxzt+4PV/F9dvttFVfSY1rcu9I7SZtFXpnVXT4hBXWvbdn4cu8jZZ+oU/Av2BO5V8PNT/duudZ5", - "9mq1/uwO+7z++A5LVH/sVK27fq9ViYmce32/7/rcZVFAH56qS6Gt+r9ne81v929bXQmNNr89Gw4+3LvV", - "Nbf/AVFY01TMS48psO/HABe++4d3uYH8V2mmkSS067XwZVDl23Murmub604f3OUE3+P8uk+N5IVZqXKa", - "5EresflWHQ/Rj0Lwi/06o+ZG5aK0Ww/+x3nr7/o9Pu73/3yXneULQyTmXvPV+NRMkTNtOt/Q++SOj8Vt", - "te7/etxa4x2ek9vqvPv7clvNez04t1W29wt0W4V7P0m3VGjuXQ9BTt21Ii2SnGdrFyh7hxrIbjNdKA0n", - "ebNzPXVowJ10Mdh5Xph11JS9RNqWuvbrV7Vv4/uO3KhFeUf9t/npHVqGB4qzG+7Xcvjp3i1XEoyCZiXS", - "6RXP97Y56koGRsa00oRki1fIXSyO7R5ZRQQsqVv7xrWdA5a7PmKC7kP2zfYnWYnkAuL7nLUnkghETAS/", - "nr5ndXkWqIgsM4w+D2Icgxu21RSBw6KNIZ5rYyDK9FLUYX4u5EiLQgsjZAm8ge1eQdV3nAHsFsDxTqWJ", - "wXxJRXGurJJllkN/BNd5Zpcg1pMX7HehFWWMp5VgUl3dq3d90MZnzVnBYn7OfB9plXpnreNZsp1w/O/T", - "ea6Si5h/Mdwi9vU8wsC+mu08M6xmq6Fq+rcK8j3D3bWH8MB713c2sKntJUT6qzGlKO5Vl7i0OtZ9aoD7", - "UPH0YSoh6W5fh/tVBRCa7l65z/f7NR+Lp4FUhNM89ynbLbYupcxWJL2XUMjlXUx5ngcPBddUG3/fF/3a", - "16uPSooH7BZwqOzYLyjb27HPK81NX9eAzOt31K874mC/fns4vS4QXPX775kUZnvQjS/o56GA3m+Hdm6U", - "7p2Lf+UsQA93HEh8BCLN+K2UfO8k0APPM8n1hq1teR/icgDcEUOWrflSENlHJBk5ygzRIhGh+m3ZnYAA", - "19laTOMsVx/efXiDNFfYvydX/JL6+GSZLdw/C7kMuryVIBBGvjN5j2Ni8L2MT75eihCz/vbgBsoXWVRw", - "IULwbqLWc6Icr/IyK3LhoDgc2FVnNTSXF9NESVPyGGnm2dlbdsHc7+zg+MnBxf/PfnN4OGYhu/b3R+Oj", - "8YNmdJsAluy2cwHTVqOcDQdE5XpbNO22XnZiaD9C/o0jiUXADc+EK/lajNnM46rMvJ7UAVSBD15MpIdS", - "CapAlZQgVGp0sl+kh8t1f2UUkGMn/nh8NGanlDbuwLXssnGZsrPzt51wVt/NwcnR+NlwALSKU9+hwclx", - "7Fq8ymSqrgKC8sYuOX/LpJ3SnPimGRZntnhzn8xAk5uNb+ejuDM6iztGvSiSlEaHIec1hiRM/Woz11lK", - "S3V3VElbNwSVEAzP3SAla5SeVRZ709RQg/Z3DB/qbDfKJ4DRxhmzDLFw7tGK+2SPRu52lHtXt4lY6GLT", - "tF50ItI+0DoHOIa1kHQLVCfpu8mgkdkjVK8D7YuJ1Hpxws5EkhUaaELOuLxgb1EOj6h+YZAW1bDKYFu+", - "uBWfEKte5XwitYHKcnxnnQOVqq/LHaq6tvmGrTP5ZM2vHZEpd4fN9tY3fuXQNvC7RrIaTpQ2i2js3wfC", - "Y6gJBH6RPtTL8wFtSfFu88o5kAdIDMgSwVIFsCYl24gS8vszSnT2WQDJhnnigw1zl6qdgDlPLqqie0AT", - "wlmp72hstCZQ2EyreizR0feQEBHq2SpGHmQ3B+Y4zWwPUPrPNZcAKBhmaPtZCAa2rUNBfHUkbwQnJRhg", - "OH5Mjcx+b3hgdiQY6hCt78EetBf/DxE4xWiAuoPbmxPIbeb3YPl+gI3sYmsTyCywG1iYkoNJjvGqXCnt", - "M5NDTH+/8jl0ZI+9ix/8m+xb7Mw99+y/41bbb1NZ5fqlfUT15vO+CrA90NQaJCmC0La3Qu5SkhGYgjvq", - "i4n8SauqMN1P55uA32DMTvM8/JUDpAjzyLtmInmdTgi5FrY1nucbupZCxsGno2LFjSvODp5+fnVIYc2n", - "8F1Wbh4/PmFvMrDN8S1tM4gIEumQIcSqlfnjVhJd5ZIiWZ5dEI/iQmhIPOGNZEEgkvVowkg6MpxI4BHI", - "ZMndjQ5oiGUGSSelWMJppK4RtUAkDfROOVQE9XLrO+PW93y4gfqS9urFBu25sx/wocFLD6b8hicrBgED", - "bKHyHFG4iTfS5ccZv/1oh4Zd6bwXIEePuFkhjdj/8eT5d9/DGUfwA0iAGw4qg2QKJ/Cvk+OnzwY3Nzf4", - "93Y1voQ9ysDDcTL4Ta3kf6f2x4laDxzj0OCvaiXZayVsfV3dsP3yxsXZdn5r/srOOwFzGLpYPa1FGLNX", - "CjagYUC9SayIBFkZ2Ww9YfmvyQVDx09VZaLsGf9MJnYgCzJg+g8Aji4z4+AwlJ5IwJtOssI+1m2zfInv", - "scwwU2Z57oATrIC3Yp8Ew1SLgmd66iZwNpE1KC0xaWCepKmlyZDNAdW8FHoN0DRC6ixZAeYWeVmYFCI1", - "E+l5DLAdUFU9ufTcpVrg+xaY60W+aQIs+FYHw/rf03r03t0U/mwFyKXQfT82hhzn8H2IE05brHPEP/vd", - "gjtK+2SY7s6O72EwOL5U6b0Nwrtgg/ZDCt8MB56BY6vhzF7OjhNkq/nMpR1S/6j411vno8cACxaxqbeQ", - "3voMbc5tx3wa1NTblVtN4v9aq7c9r8213j3F9VaePKw4NiUBnnEPxQzhnVHSKcK0lZpfCm14TsoKIFbY", - "fyCoj5VLmX01m32pZ6jJ+QYyr6jU3SEC92Gh6cVj/y8AJP5YrYXOEkgU/oN3JiRUm+xSTNccFqCHymwP", - "TF/HZRZUjZxuD1c19bVrHe5pYnebtm8hk39oC7Gr4lOe8zV/AwZeoXd3MNi7H79ltbmaTDUaMynBnlXD", - "ghPUZ3Zp35wcwDQW4C6qjBizX87eA/HGrNL5DEG47NNg9un9+9MPp9OfP51/mbmcWHp3vEE1kH0AJvWT", - "x4+ZtA+RETSJls6DP3//F5Zma3M4ZOvrOc/oRwTkODg+evrc/czzfGTflvmaHTz7C/05nmmpYNxW/XyC", - "Zr//f/JjbSJv6MfEMD5o9wyFEkyWXQ6ocDAcVNoWtu2cPHkCNMMrZcqT4+Pnz54PbtqCrIe+HIgFA5I4", - "WiZkMyfYd0q0f9Tu16Mhe9SZqEeHTZKxyGC6TJ1+dPXtTsOMUpzrvGmv7ZmDmNXFbp3mUE8/v2OCCA/G", - "7BWAfBqFGFUlbLNgW9ldlWklQTW95DpzTAT1gHs7U0cR6Gwr8ZqfEse6/rX3PBKHu7rbgSTcyOxSsNN3", - "4aHsXIVrfj0t1YWIBSB1CRKwpN1CDpkybtG/+75sTjv8/mz87OTPR/MH2GOlWMNLo9JxOtNSq9wwzWWq", - "1lYdYpn0U6kkOzgaH42ejo+aZEyLXHF7ANY4W4OTp4GJ6ijiOiyztVBVxCrUQJBjVAyQ2SgKzK5ysMET", - "nudmvNUiVqpiehFD+ihGF/blX+QIDa75WpRofIxXUsQcQEkuKtNTTf8cHW+ZIy8N9jzqf8yJLIQ8fXfH", - "GxK+7bshXXY/3HsEF9G4+z6/+Xj6ziFCzOKCKmDVxdZGDgczF3Z2TPta7b09rSgf+a6Ono2QpstRBw7Z", - "8XfPvnfXZacwXanPjv789La7s8h5addorAoheQaXaKoS82RZZakwT2p9oXmRejSNgbkYjcdjv2Qng3i/", - "WzcstNa9QntBOr7Uq0fL1HORNNao9y5pvx7AXPJJ5psAG5/YzSKi+BOkJQYEaD7U0m+sg8oIwz6cvW9R", - "pWmRVmA5PhyzM5Go9VrIVKQn7Ol33w/Zd8dPh8xqQbi4YI21K7hVrrtrunfut4h+nNl+0d9b726XAK72", - "zoqG3Zi8yII9uVXRqLdGrWiEoQxY4JEBYBHYOd1N8/L0/M3UVrqbBhLt5R8l7+6qgeC87K6B7HMAdzpH", - "CzB/ygTsejwvI1V/xh+w01anYf4jdmCveLuA+930o6eRa+wP167cHl4W5ej5+Pj+x66uKHLOhBEyEftM", - "q/vmQWd1zwP/b6L1/RsqUBER9ocJlDNVlULfXYnC7/sUqaAE/d0wziqJcXF2hJ6C36o8dS3QaYrxSbMF", - "+gd93Wa8m4529umXL2/Otuppca0Lt+6TLcrX4bC3IChew4lcKrXMxZOlsDsjKHF0dDxk/7wS8on9f8+C", - "X/4y77FvFEJqmM4xz1A740X2RDs2ud2UNKVbetqtI+2qa9iDO6hstBO2qm3NZfsXqm7ZgnmoDUfuA5N2", - "uPMNsG1ub50pvA+QyHCRCe2tQ7fW+mjIHt2y59pGo2093Fmxo33x9UFk02eeXPClONVltuBJDL40W1IM", - "Rz0Us+JPv/v+hM+T46dRyF5HNOJDZChxzu4ebtBpDcBSeTbXHOz73BgRD79yHEN1++K6xM00/s0ouVVI", - "Q2+onlvm4LUo7NNAJhHLv2PJu5VXyvXqnd9IkFRLxugYjHY3IYuSh53j1ZF21WNHWscfj8dH46OtI4+T", - "tdVD/ltrnfzERheCPvrgFjOSxlEu8g2kXdI4ou4PVyyTtxejPWn2YAhsbuaIi86Hy2XCTH2gwM4t1GGf", - "scpTt4MysXeng90XrTkQX5G5qk9pxNkE+V/buvEOi/nFDU7xDr2HjRTChthLKljczgYzTy6Po5vsHidN", - "C6AOqCmmth22UlfGI7huK4wIrruv6y9Q3s3nmVjE1rV//3eykiITS7NVVzN0so72Q736cSGAgrV5iD03", - "2n8Vy1ku5LIME7PuRh4c0z10KjSwoLTomNnBvJFRdfiHcJ2pkudTDDWP8LVUa8Cd51dI9OeyZ3iuiC0L", - "2CqZnWBEgscwpyzPm7FrwPXp029Al3nIVKN7eNT93uiGmzhWw72w+/fIjgKM2vvQ2iL76r1qoNCeu35f", - "b5x/k6V8m8k0COltq4laeIycbeybr31hYgCFtD9zWygKZXkleFiw+B9wZi8ad8FxaHE4OnqQpCy0inUB", - "rI+Pgsa+e8i2Hn4jQef+kHrrg9uSlgg67QR4TH7f/6S33ohI2P9HNIkzN11TVsO2+9UePOQH/gDkeW10", - "eJwzP5KvD3HM41wWnn3vv4IvD8GMFqqS6b2UAwxRmZbZWrRzKh5wv/7rLpr776b7bpeguoZRxkqJlSq6", - "UPin+VLprFytMSYK2HYzCEIGjhz33Qk7XyldApSKLTPfsJUqiKHsYK4FT8vVaAF0SA6ePltKSAUkfeoQ", - "6vNS8AT1KchAImnLiGXGamMNTezAlFoBNhJLkI46U5IlK57JQ9fLSK0otplB/a5ZY66usDpTMjDjHIZ5", - "RPVsBWK7IWu/3l3m2B6216lJUG1HAAUgv6SzQvEF+mCHm/0uAj8OPCDaE+TL0bxgHDuwB0qxRE9ZOFft", - "ZQN3kf0+WC2qozHFmWT/OBoef22Ex4czW09mY5rvN7Ol0NIqNuelKPbkuSSGxh42WTu2SK71nZUtO2KY", - "ibaa8++ubPxxnf6DNJkejfYNELUp7dmsHe2WuAS+V6fdvmCQxHWVGcG4fdWZkn3/nFUy+2dF+i+DByYc", - "1rrA37KXO/PZQpy13SLIXwoLSEkM3z/vXNvYNP2Om/q/4tlREyA/FD0zxpdphAZrBXijjLtzTss2tuO9", - "J8DZvv9rbDH3X6kHF4CmFJFcdp5n3NxHZxLpUuzIvu3lOtmZpgufqnHb10FSx73mVWhCdetq4452MJqf", - "Rw8mvvaxznW2pP03PG4qIzT+QWn26PEjzOfLgZMMXZ1hOiXk+i1zNee5XS5Ztj1ckAs5jdPww6OFaBJ3", - "YVd2Q3CyaZdv6smKMjL7+Wp35ZbMqVaV3VfipoDoksKXC1PDrYbqPKaD4YCn6wxcK/WcUYmur+X/TRlS", - "WxKjdk6E+gy5Rn/woOu0p23u01YWXtjPSgKBVA8Pt0PNAeQUdSHkaM6NSFlKOU2s0JVEgRU/0WZ6IYry", - "NsAZX5LZkkQFXVcbAwZwVdtSMVjJWOVYlpWKLbKSgpqQibknTPpCyL7OvzFltuZIYQ2RXpnEzvvmbquz", - "r9exWqnXt9V707esepeoHJpoh/HjoHJwlZVEWBn2z4rnWbkZT+TPIi8Mw4uDqaoE0jAtcnEJgK4BFtLZ", - "6U+syAqRA07NfMNSUVr1XS4nEmtd8gJEcSouM5fnjhut8InV7DN1EDB2FchxiWTeKL/zDSISEQIOS9Wa", - "Z3JI+0gLzSVwNANwGqc/CM0yM5EJTYdIIYuJAI7UYmFE+QShSIH+d9mXcAFNTpe8mBZCJyKGrHaOoykr", - "3cAigu6UK5gENhfllQBctxq/1uH5IO25QRZLaoUvfRwgaM40dKTwPFjzawZP1sPxRL6GSTdsMhD5XF2Z", - "yQDyEHCqHCgETL0WS67TXBig3scSJuEQ7/RWaUaSfsieHY2PmClVYdyk2jGYgkvDnh39N9ezoE8trIFn", - "R8MHflPxuVF5VQacE62EDK7T2p4BHfM0O2N25tYEGIcJf0nk6gon/ZLnlUCEDXFN/LHNyYIHTnCUgNK7", - "TQ59ND46fuhx4/6D49wd9N+EKJiSQOreHiCj4AvmdzF7zFo1ttf9aPwduxCiML4+W5hebbngpmQrni/q", - "nelO8XgiHXAdiF/pdqyhU0qaGEEcQkIhjbo+wJ3Z/O4hJ5OuyynmfAP83nayeKAUtjPiJwRJ8ovCPgMy", - "WYcpOjy/A0Dt8/BtE2nHSkiAKIEPAanHsVVmMtGCG4ySFEkG8zN3KFkgSvlSCwiu8VKkjn70zJZsLcqV", - "SmlL7p8466Ea02kqLmuKqp5NF+63TLKPzHGRhqLenTCBUBewIdpb7nh81NpysS1sPx+x48emTFNxOQYO", - "fjeDHvqP5/a6yjOh3UWkJGC3I1BghM39+CG3WOyW9uqhkmIHntovQq/xk5vhFmCDGh9ha9EWnsJOVYcq", - "/bYP9inbQVTY9oHV5jc79iPQyLeVPRNLcV3sVvbXLE8TrtPdSkOpc1DTd/ugm8u/7Qu7TfYo/pqXAorv", - "06uXKDp2K/xKyd8qCQduxy5lZr8PmljeO5WuMba39kYl79Ldp+Wtfa7tVvzd5z3W6SehXqoKjHov1fXO", - "37zOrOxN9m/ks8o3y10X4Cehzle8cK18dcLtZZXlqdC9sRYAOKx5Jm+D8Y1Z3T45/UBcg8KsUHmmunxk", - "dqKUTjPJS6WHzFTJinHjoGmHbMbzXF2JdIqwM7PhRM4KCI2f0oU9Q0romVMPsDU+z8Wsg58FtWL0Q7MS", - "RMNJNwH27tda46irbMXbBBdFSld/9JHu2KoZB05t43IefPx5ZYS24y64BmdTkmdClqOEa8BiSpQsM1l5", - "CpCdDLSnSyHL19StWDggTczUGQpiMW31GmImhS9r1cSVyAvoPoLAzHEnoTmPkjaAdtzACrXIPUBlN2P2", - "q32fOP+ELQeKKVgPV9wA7DiAW0nFcGRDikTTl0JjKo8wrCpshxDZjswevCrVmpeIsdeYtd2DwCKztnSp", - "elsdHq2cPoiOLaMP0I+8rDTPWc7lsrIvx+Bn/1ZzqNVQRWjVe5vJFOyono+GAfQY0GrPVVWyNU9WmRT2", - "BYBPXA+zm9snwUZwHTMLgnKEpMwNTTt22t8tGPwZauVLANrkG3pVsyTnOlsQiQ0DSQMKJjxOpRBpH2WI", - "fftANyTPp1nprMP9GX5FziUMsVQqx6x9dnA0enp0OGb/p9CQhWdI8GDiNiK2wXdCj9kpK5TJ6i3qARIh", - "rWGE3lD7TIItmgEieOneRATpJhiIVpKpzFRzR5aC6EnQPJdKgs77+vy9PfggAuw5qWSZrQVgIeXZclUO", - "UaO+5HmWklVIiHTOkwsCny94psfsXemQee15Y6cQ986UdqlndkJGEJGeC+Z3MTQphT1LKOYEnrk5N27P", - "IVZ9nXx3FE2Ha1EyAvRgY9l3SstsbhS7d0xnR+Gk+i1sW2p08Hh7B6NBCV7YeYzqVSbL2EU1ZufVckmo", - "fyDHgIV/ZkWOvbRq+Hl7Xzkoa/sL4ppDGbDP2X+VWgh3i4FjajZmv8gLqa5kWDtPElHYFhdKT+RC6Sv7", - "WmvuYluHnaSFFQewQ0pF3WpdhgP7x9ihR1LeSMgGXJj2EsMSzGVM9E0GXAou68GOc56LSxroRM5CpK7Z", - "mH3pnEiswGX/edRElCg4DTUMGV6Y7efhoNGHuJcIr3h3fSPabCQ2VOlEMA6bbbTgCQC2utc+WkjAELXm", - "csMqqYVR+SWgKKLsYgU3Zu+DhFfJtIYEbCEKU6Q5HlM41zhZHqQUELxTK9Y+XQqtIVUTBRdWzbKF10Ua", - "7rV/IBcBoDS6G4fQMoeDmveMQyzL7ph1RoAYBEzSruFba8Bet4/+OjsOQT+BbNNebCAH/MWk5Ji9k+wz", - "gOUe4xJkholkpaxyRQcATi9sENAkPN55ti7yDWkSIztPjPrHCqEJ+VzEKUjqtI82NYW3rTmEXFCLSEVa", - "WNnxrp7zIV1H4ZLYAeMywtRfl02/p7vat6Zjkabwtc+q4RX/ePRlQxLXnFR9XhzQYGPSm+da8HQDW7Fa", - "IzEGrlOwiD3M+XJhN0LM5wwZvY8Mq8tgvf5mE27KIaX8eHx02DRN/iVqONo55VtcW4WBu7Colim7BfFd", - "l7UzdQUYwV5rht0IIKyrDctKdsWtpPMvBpiqKwirqPfAuQtcsHvlUVu7e2SnwhEB4U46/fi6Vg7wHMPW", - "tJ3jSZlv2CN/qB9FVUGnet22B1rCrl7yq1WWu2A61H8yZy6ML72dsDu+Nmk26un39zYc7JyjU03N7bmn", - "9LrxIHJIPIbk7p34ye890tZxiTPJMAOVvbRXIIE/jyeSgEoqcNeCYplD6VB7HLdZLxjIh/ry7LxyEzLm", - "GDBXkm86kOMEvDpo7xswIbjiXtiXQq8HJ7XMR9NBfLb8TXuyg4ktwNv2D4LIrdDzchDXhTJOmgj7zgRG", - "CrwmRnhNCHkpclWI/Z7Mf6c2YveXFmuEN9/tUXLmitdHA9kbEXwCDgmql7vIw7rxnZTruvG4WAa7yl7N", - "k6diutdan7nP2ot+R11gi2mkI7kgl4fnWYwa4Nz/xsAPQWo/nEd/c3tRHZinAgU/0OSV9gp+886ueZ5i", - "HfQg5Fu35jkWhY9EETM1RY+Bt7+VmieiXnIY1ciZbXS1p23Jxam1D8kVipNI907lhmXGVMIMGdjicO/C", - "zHFjqnWBh3vNU4G2AYcNQ49mB0Id6KlgzGWPErt0Sm8egWYHeRzoRUeTEb67m5diJk0peLqPBhvFxO7V", - "sF5xmdonu3gJQSxvIExApLvxnbTikaCGqRa/obUhIIvt5x9xgWF4XhPXmylVJqg/d+AN6W3bKB32MhUl", - "MAlsL9kfQtUqeFvj/iTRyJ8/ffp12CU1iYTnhKsaIyXhCP3Qtwx9veybkJ7h+yH0byklF3mWlH4PpZk9", - "YGu0S2DMC1Cq23+uMqGtxrCZJpU2SgM3q+g70z+70q+g8Lkt+4amolSFytVyM01WXC7tNPU4tqjcKyyG", - "n9eYMpuPyB4h3N9386r292yroy3aH+/7qGEQ7sJBdMoMPtwcw4LTN0DCoupX486gCdyFOoXgDdsZiHwt", - "6K+YSlVSqppTvMISRVVOS6WmAFUUKeE4gkXsc3sFIkF89OeqMKUWfD2lMcPhwHCt9qf+75FvsFoHzro/", - "iRKEZIO9UKUCiZPsvyhcTi01X4Ppn624TPMmd/ke3EjulX9/MqPeA41R23fbfkgM7yP9NMY+GVKUNby5", - "kQ2FS5ZJMCDbYpWsAZAkZfC3YBnaVwh9PW0b1IKqWr/FFtULeVdtA67fXuvU/2n7Zt1t0dRigewiUxm1", - "8J7XUenOTIP5fD2Ts6j5EMaD4ZZNQmOCtfoa82V0r6ejo+Hdrij/OuvuOXeTtOZip83oBe3WrcjznAS3", - "VDGuWh9tutSqKpz8M4D7CJRQyQp0a08rwbVgRhTcCpEgDWA8aA4cqpvO7WaBGrSQg6839hqI9uO8v8GE", - "S6spOndCIfRolZXMdtqUwECaav+qaJx21+5w4Aqb2/pwyig9Avpe28PgAWNYVoI7+Df0KlFR58SMJKP4", - "JiGCu5OYbe82P0kdo54n1nIH0H21zZiIH35t/zlYjsZcfB12NgQ+O9HKQUy7LhsN3Fs+nLCeDL8WZAiF", - "dwI0aSayXCkjfBW8ZJx5fYdBd1m50qpartjMdXM2ZqdYwSPDJLosZlTFbCKpZSJpsn3ES7rMN7UBWqaO", - "8tegbVkLgYysaOWiuMoRdGEiKZLBvVO9Wb7UKoeyte8LRwbQhTO3yWa2J7w+GfUJGs21uoJXTUB/9a5k", - "dmXBCOWy/CqZlRPp7CQOwIU2mhaX4MMYMuDyAE3FfkCLIlXoXEhWlbxAR1VlBKxBMFY2m1qldvZkRjnw", - "4BaZMdQ9ccw8KRGL260seh/tQoxgHehpVq8E+b7yK74Bo0+eJZAb4IM6IcFuIg2m64o0S5B/DTxKHhIV", - "pH1mGAo8mgbTcsS6Lp5MpB29qspg68Dr3i9LOF8+TpwqzwwT66LcDBu7fSL9XtXCe7HGbOYPzgwjgROH", - "klofBNr/ga8W9XBnRJ3IhIONE4yZ7NM6K9nMn4YZsztGi3xj503bFwj29/Jo/DTEL8/WOL0TibtjBIPH", - "3AYIwFzxQkRhhUOhtJMmf+q/2IOxxUveXVt5VYvqnRsJhedOjQA/5MvN/Yhh6PqNXx9IT4kqHS3FKis7", - "qzBNKZAsch/zKxfD7AoRcjls0FRII0ZUwMqyFyxXV0KH/ty5KEsAgH0LDrnGnQaKfooJJNJg+IqQKZel", - "GXqnnN1xvnG1QOkxt3cvHAx8FbnvMEzbamIYk75yTWEMMmxx7wuXSo6wYWSJNn7LkkO4A+0bDRAmakM/", - "i8GCuk+BIHznrRQ1bb577VRP5NQc90G7UHfevR5AVfDww5D8B6XTP8cof2ArtH8K2fRBsqHp7OzsbcDG", - "Hk0wpQ7Dp1gr9Lwn36NWETFUnCZllZXDmmvb3lNslS1XwV7EqwCZu63yPt53gbFD8dWFDisdUyWVLhGb", - "GPvhrairrByzz9wYxg0Lrz7mMwammLY0kaXCbCV7lZNjWorr8klh72FV2WVYijH7T2wCbi99CTpFptlf", - "zz99nEjIxh+zzxhLh74sQp4j2WDrMOEFhTfwyN4uE0kKr1VXrAIlFgsBEV5Yiz2os2mWzhiHo+izOpyF", - "tCfb202tnTqcQ5crvDfimq8KargZ1masndxKP2dl/Yxp66v2RPotecszqNz9JXTHm7DTUH0jBmiku9+n", - "AQopqGlmz8R4fB7Bp6jN4vapQ1T7lAWvSao8Ryb4nTwIvucQ6m1vvliE6mXt/9+psjfuA0B63IFEuLMM", - "nlEYtbV0eq+lIJUvFvnWvfJg2nlZctuum/ZaO/XHYOxUFPd4fdhJL7gWspymKokDf583n6ggE+G2VleS", - "ciBTobNLkYJojIJNYxP2oRG9I3+RWRn6/byGbV/oWTBJpWK87omtL94eRHTGghc+FfyflfPIj6SoSm2f", - "oFQePYGQo1QbEupXpjcm9BiH8GkV826uoQXhzk5dpfvIjuxqlSVIXe/7AzGm+J4Cncw+SsfRSKDtku00", - "FFV7iLhatO+l6QOqrczKO3wYh6DbSXy/7zF4UF9IWIIBTZYIausP/HDQyAboLLBrLObo9CiwqLXbo921", - "5ayyPeCm/QshdgOrta2jKDd+zfBvvwutGpyafdrYS6uH65ZKxtcOHdYHXfqcb1Ibg8c5qmmjzIzoqbCz", - "XtbVwYjGfLcZMV+gdGzMUSyU3SarVXuE0q0EYVASQpxDmR1HcBbyntCpX1ei9Kot2A2ueVIi9Ak+wPwJ", - "rzGtcWsuSxHdlFBVJCTN9xMKNKlHMokPmwisVWRCw6OIrQ3rIX6954R/1mqRxUIrX4PjVKRBNEMBZSGd", - "AWYMo6G8ZkxGjtpE5G8RZ/M1ojRshtWIE8hfmMXMG7+pbKvN4a8qk67z4KXQ2zF/PthCwUfOY7Y9pAbL", - "BZ+aFdfpVhlyDqXCz+ipc+tHSpf+k96bJcgfIys4lxtyEdTOgrb8C5TsqFm5Y3CuP/kaGNzDEhQsHTNC", - "+1uh9kqEGROBddGZNNvWyDF4nPceT8NV8UeMp+3c+PcfFrw2aYnvMjCrKztlaQRSsn6x1voStIKj27UF", - "TLMwaIudTWut21UaPo0jV/r2uRrWg69Py172TICaf2mft73zPgxneI/hg/HAOY49XoGSaFs7PjpC98Bd", - "R04ZmKF08DHXR7afNczi0fHN162O0HDAWPdeo0VdGtUYvHvLKzXC29T5M1hZFbm484hDi9Bd1xvjvU8X", - "iO92yxQ0Gvu686Fu/rSH+xCU6Hs7D7eJj3qv9EiQYcsV2pmFyK9okht8vfka81aSIjwiy7YdZ31p+AQS", - "DEKmnWM3C+0Vq8HR1vqjRWt8bih8oj30dij5bSUQYKVbIIzY6PzaCeDoTj7mwvV2oA41sipjzkvRLeOS", - "tvs/jvyGkFHbt0N8vA6frf2zcxZMK5nGNpv/XV3eshfFYmEVsc7PoEVOEQKr+yvm5HT+TPps9wevXnZ+", - "4ZLnGxObUb5cakBMVjLyK8Jo2qXOYh87w8xUCzs7sZYLREHr/B1U7u5iKC2ypZw6pLXbD++Q4ncyuRxC", - "LtgV1+kT3AfODA+ubwzNA9t5qbk09mVkWKrgjPOiQMdpf/DKGEMKW5bgcObuiPB6WtcRBM63TA11GeZt", - "+xAor9ZFBZHTa1HqLMEso3mVXIgygDkbT+QbnqzAjAexBpDimIpFZt/7ECrlMmCDIQ19qj6+XYkxQ5h2", - "QZaEeHbg/IeM/1nXnAmxDoQRMAzrMBTi4HDlfJr1RNqlBDZXp9o6lzVZ1iCgKyvJ12bG7CNFfbhGxy78", - "w+oAE+nCPJzluRmJYJVMLWBbwOETVulsjFarK4zg8DThOPnN4RyYaj1k/HI5ZOtMDtmaXw+xxiFkZ5kh", - "Cxz6h8OJrJetWRHAWw4Rwm3IILqb/r3KTAmBkPRn/9+2uqVQrYqWQq24WU2XOkuHbCmU9yYf4lqDjACk", - "yQOTLSXkk8hyCh04hCET2+bJRM5ms9+MkhP5bSIZmwwKnSViCiObDE4Y/NX+3aoN9g+TAf02dL/ACwV/", - "go8nA/vLzRDro2D/TPRVB72KV+cSBYJfTfY7fHd8BI1M5A0MIQTiql+7XlZuO7uuHGQyRrMnPWaCpugo", - "eHPUkIFosFGLwEHu4TdcgApl5bgECZ6UFc/rck2krVroYRxTZoqcb2phgC2aMXtl/3dEwHQoUyaSBCJB", - "OnAZ8zsOmVFs5lSz2ZA1g4GGCKc2a1y6M3+o6shLstVQ5ICu2viEvcRpzQu363chgEEfiOBx0sjMj+oJ", - "M9k6y7nOyo3zejchCN33EwkAg+WKywCJkIVAhHA2gkXAv9tJl4LrESqPaWUf6RC1Bel9+I5NM0Mdaaxo", - "M5P1eB8z67aQhZY+04vY8HDzt9SCY/r8LVPI3oM11H0FwXkwXS44oG6vPd1eB/BQqEHOMAn/W4EL7z+n", - "HTWoH3riZy7TXMy5Nszpv4yeo4j1IYmbzTu9XE4TwbmSMjaeyC/u+5W9o5IE9qmqP0TDElItf/tmZ36M", - "f8L/+cjX4uZmRkzKRLb/+PEJ+8UIw758+vSRHXwBuOFPOhNwF37C+LePCvO6DsmtBj1/djT6/ui/Eaiv", - "FmkFdyreFSs/6In89g0pj74ohduBOnXjZLLtzeklz3IISfxZ5IXQ5vFjYAqZ1d/O2IidwWwZN1LIil4X", - "9rhB96l3sB+9lmIrxaU0J/Y2GLEZUgZ+4PpC6Bk7sMLm8ISdpin7D4bvAvBDas03JEA9kTQalQ+pogyC", - "SWfsIJPl4Ql7B/+J4tgUiClRf/jUfZUKzKmwbeNWODxhmPVGYaEKMyNLPq+sqICeAKfJzCS6mv9crnOc", - "jrW6FIb9/OXDe1byJSo04hpiHu1+uy7ho7VIM24/+FXzghCYfjl7h1rlT0L+LSsRNnOtUp47oY3z/0/7", - "3RvCIsbp1pkhIONESacDmwZBt1u/l9xkCSzOid2TfTthZsu+CpbyltIsXL0f4UgyXIcfj6imLzRvdqC3", - "1eQX4sfJYDIpJwPXk8qUau3P6wmbfcnKXJyw5qkCQI2bm8lEvlTppv3rXKUb1x/N3TTRHoVe/Yfg4A4O", - "JuLbt/9+ITY3N64yqP3btye2JFQ2keFOxwR8MO/B3A3Z+/cfQHCtIdaKgF3y7IKUuIkscSTvZIlMOOSa", - "/k8MFUS71ETyqlwpfcL+yqVgr5WYSLu9/vEfz76eMJ4NiS1pnQdH+MzRm7u1h46eE/EeHIoTT5G+zMpV", - "NR8nav2kVEqOcDzwb/vpT4q9s9to7Y5T9EueFxfCXGTyyVLhl40c3f7tFnPuR8wU+wl1T5GPh6jOE/Nw", - "Xw1DCXOp/l+sdhiX7DPb6xmFDfjUn1hldKOgKxDuyy9vR3+p64UcMqt5UTaXrZ3oXuh2JVwKq6VtVMWu", - "IGRT4TB8PSPAsQpGWd9VkHkLUaC/VaYE2YMR23UfDN12IqUQxFSVhVbrgkR2PXAt1qoUvvYV3gnjLXcF", - "fvXBCjtW6fxHwKpNKp0jaK0VZG8FRWxbOWmHZlxb2ZovhXkCojKo7fPrt3112TvURVuNHz9u1h3KYES2", - "mistR2m2zOxD4PPrtyZo5YsttUOXKYTLdxpqp1ly68jZzOsn1A3ycYCVBCUGReDY0aF/AOJQIMyMre36", - "zYWHzElRxdei0MouEMS4B3M985YDhOSm1cK3T6HtjSZaO6jmB4GNOY7eHZQZaPuIuxyEZnNZUEaCkH1n", - "F7Bbst4OQdkP2bVIT9gMxZ09urXAZd++/UeGyQVWHsdr+fbtSbYggQzWD3jskMkkHd52RiGZwMoGtDUU", - "OQTv2+PSkV/xtqPhxUiKPlrz61FVLv4ymm+AbPr777579n0o3e5suXrjI2c6BqvPWozQNiXSWu7BQa6M", - "aD0jHNBNLTECUdk/bSQogX0Z9tOKAL4RGstH2CYIwCIyuxVhG9ljKo1gB6BFHVqFl0tS7tSCwWPADJkY", - "L8ds9g/7+mJH46f2/z37OmtUUPDkQqRYA0M6T7eoasHyrCxzMRIyzbjEap89ZbAO7OD/fn7N1koLd1Ef", - "YsXngBJIXaJkExCFM3gOCTOjbhMYjzlEUDmMWA5/xGEcggUEh/JtMqBKJoMT9o/jIftuyI6Pjr4O2QTD", - "PPCHo/EzO9g/2/93/PVmFvasOeZWF/Gnad1TmpIKUh7Z+zfUWSrn+0zF3Ay9f3PoRJe7fR5dCp9VV6M/", - "BTvLWQ0RAgyAvqzCfqmyFJ4iGH5Z7ytAjRzHzT9tD8e2UA1+5eB5B9tg3EjNLJUT3EEICwZdvVu0JQfP", - "c/ddI62ILmmFT606CoZBnhRcDutCq0uQ26BOyQRypjJTA+450zvM9ERGAlBrgy0zinFvfaX0Y0yx5EkC", - "sZxABIJplbYLE4lpSf4jn/QI+dxcYhYVnT2A2oeI0HKDhimfLgedbubu8dyouvsLyGpxnXeutcB+3xSl", - "NfZdpQGqoVqvOeQnJ2Cr2BP2bg9LQdPZ1EdQ7QlgNmC0x9Jj9gmtdZRnWRsKIcwfJKEpuabjCOqbl5WO", - "+oHYVRxnhN2JkV1iz98rtV5jfCxLuBEGZCdgw49KITmYF8jucsJmkwH+8YQna3EyGYDU+MUIPfJM2/T4", - "mgwqI/TJ8dNnrthrZ6+wE9iqlbDwsGgY0mbl6CCagB54L/c5ui3H0x1vxbdYzblPbWgb1wpQ/+sk9VIx", - "ahqf4eTVSDocQQjlA9rSQqQkAlFDHpPOwcOKtXt+1Xh8BJcHlBaMs99UJh8ZNtPA94uA2odEnAGpLtxu", - "qolcCU2ovwR2ZpBsNnUJLuKaoK88rGxwmRNKLYavwc6C0+wwd20n7E50xZoMSEg0UWPluShEaosdJPAy", - "hyMD7p8h5PgCo2vCS56rpb3Ky2R86O3pGXDwonbBS6wnk6WiLpDuee4N1SFjnfSLBdNlTh4/biC1DR0o", - "NmSfghcNHljkkxzbmj+GZnCooOWzH7ZVniFruGGbnrOhZ1Nqas1sxP6qMo8STLNCU/pZmXKprXpE82dH", - "0vXqwCCdjwa/J6fKpBNpgH4ajEto+mFOYDK0klliJgPn4bEL3/DtBNuQmlxOffcCX46izwa5WJTT0O9D", - "hadZOhkMfY1BCfvDTehmap36Rn+azfsfmm6ogibS98/+nBqJv/7pmxGJFuVJsZymRt40SrlPwyEHDWLB", - "mx38Ve2Iju6t4qhuSEOGnIf5JgA5ozXEl9t8U98P8J7MOWCYZtIh9o4DC0GpSCxwkDv1JT+CKGcKtR/N", - "QemrWXewK2gRcCnrE0moAfbBCcC/0J8Vv0SK2hCVjXKgy1qdodeFR2Z9ZBgim0+kw11tNj506XdgKTka", - "P2VzseKXGaA/1zInMKfbg258oj8k+7kmsSkPugvjEJdCog5bcwDBTLqpcrNTT0ZmJhKnQ2mEKEDYEp+t", - "3l0yTNbtLliwXgjIrEUiMjQPf/nMnj99Stqgf4s56WpWWQHPUrlQ2l4gDv4YMlorbexIcdFPz79MZGaC", - "fM4XiEb++V2dmp+sRHJhAmXR3msi34C3J5PLKjMrPwLabRMJRhKcc1QHQ/gb/EupRce1UztjCI5vza/f", - "g4V4cPL0u+8B9dX99/EtL2cCaR/VoOwQeULBqTHITk/ApaSILghOo5uqBuKdncgXjdGi1agxXCwJg2a0", - "mPVa7sEg1QkS20NPagYE7cII/Xcqe4c8zzrscWcF29N9dJmy3eMvMBPUaBlWxZq17t0ZbjkKrp7V780Z", - "cPq5dHSDT2NHLwY4N203t9XoND7RoA8u1b8mzIBnzUS6hPgx+6wVBj9bGSSKXG1Q3adHT3O74Ziig3hh", - "JYq+FBr82Gu+IfqOrDQOeKCmlcAk+yA3vGy8F4nFAzk9JjKcEsBvd9n8TlzVX6MCWb/A17ygJwusQG2/", - "gQMdjt3pgiAPOmRudraJetKnsY/ZeT3gpNIaAF4m0tMfOIJzYBPx6wjHNlhL+yRywREvGIQY/QaYIq4T", - "E2n4WowuMuk0Pog0tlUReEH9V5jMNXIecgy3bu4IbNJugXo1NAcDKnyRrJQyCLNEZecbp+NSFHz0kTsF", - "1hup1lkyaDxHmkw4uyO875or8wqE3iAMCd/GDVEHeXkuP0UP3jF7G5gNvWIcQk+UqrhgBdd8LRDHAm43", - "H8qLVKKECVSjCrk8U0IS6ppAXPzYi4m0Qry/wBhbwFg6Xo4ZudTZJQccqLmX4FaXOSg3BTLXsOOjw/FE", - "kpykbdzfDGz5wG1zfHTUjo4zaKEna42BV6MZue/r/CoEu7Qzfzw8OjpqbaCncWaNMGJ1l/SnmhyHAnRv", - "Zwp2y27lfCukaczqVxnMsrh2iuWQwHemPM8xmi28S12E05idFoVW18DwG+o9FFSZqokMopSIhJasi5lm", - "6sr2wX0GuimAAdc1eFd5VkbIblnNdXviHmakGoJW6GITPZDrRF5lMlVXGJ8HwtW4XqE2EXCWjNQVysse", - "ss7YaoZ5KhE0DESMwLgUslCeOgszRMOTZZIifjQlG00kDY4XhZBpB3ACjQuErVlmYjTXgl84JxRmgLur", - "ZiJ3yBXaksXzAi47BZngw4m0PYemfVYmphc5IglH8dObuz320Z+epQQyS4L9yHbYjpSwCF5/qeQIZ8lf", - "ttxekgUkREL2QcJz7ogj0DBC45hIAp9PVTLCMSXqUmhAGjl3gyXYWJA+KEQuxOZKaXt2JBIcPkFR/IQ2", - "yJBR2N0T+78YZWpFyRP7jycgVEq+LojvJc/khZV5Se78MUFXZ16vhj/iQ3PmBkCZmS8msi4H3Zwxs1La", - "3n9WNbE1kyWG55Cw5KIKqR7YeDzf/C5SNoNnojedBzGJ3Ew3qgKq/hkEwg5rvWQ4kfNczYdsVa7zIfk3", - "KDwWrdS1Kd627DH9XRdewL8mEgIwmyvnFsBjjMP9PKZVmDmj2Zid+f0BFip6NwGVAygw9W3lm6ANAGey", - "4OXKxXA+f/p0IkM1YsFzSKh1zCmmVPaR+9fzTx/hays3ahHnvHNIvExp4Ab3/XzTiAFE1KaP3RjPGURM", - "zXrCPP+Bcsehl3gug9oO77K92kUwz/3m647IGPYcQExVFBGjLynZB/B6T3t6W5pyKyPZ7sfU7hafgzxr", - "uMwKoUcFMNuU2RpWxcrBVF1JM0SLaU2da4YTaVUvn4ZeG/5cRDlwQ9krlzppNemfuWHrTNp7tDbwMysd", - "VoKn7H/9j/9JIb+JWq8F3D4EseuDgTRfLLJk1/DcwrPe304EC6XuRsLxyiOmED6uf0JDbvjCUTWh/LHD", - "keIqIJCaSKJO9aYtPM4ekoq8JI5pDqNJ7BjH68rgVnbve8f/gVELrhgRBjCwjMjR7YVtnVOpyk7xNthv", - "HRtBfqK55hJVQkhpRXc5XzubgtU7uX0yAqFByMZFDF0tV+esYfdyc+MsXPUdg+Y9YmS30uZF01CIfkri", - "x3KiCjT02gbZdER1GE7sxADYGfQaxATSlHwb2AYHJ4Mn6HAaeECEAU/WYgC5xnZGKfO0RPOGExr2MQSU", - "J9clMKZAkIKuv5pC8uY/vg3A/GjC9lJhlQIUSTdf48ATNdTFb0bJ8Rm/+uBhe69Hv2fLkX9epWMoAmEK", - "d4EGqNXqdv5p67AgmMwC/w8Y5UKNGmDN7Nq4XFgHftbgj1xl9krDuLUGkpljFwrByMAcCoqfvYZBpwl0", - "vPFE9oOfoXLywulU4MdAnQRCnHgm3X/aixoPsf/F/TcKQnzA+h/pP8eM5oMCHqQLbtKiyDmAJ9NdCHe2", - "ecFklee2Sgi5dRoBtWEP6yKTmVXUXTdx8yOs53giP1RlBY88OmuXghQi1OJnzndXZ1QEyu/QPRLgWEWU", - "aLCpOHy3iUR13UVfEYwQJKyiTd0lfTtKKKtDYa3eUfwQeu2LxitKNBMHAoNp0s1BIfTJVZ/KP2TcjQvd", - "3oRj6i3pTkWfSCIy8A7z2fOjH1icMGH2wsor8J17EeqbDKBim7C3oVoF0Ns+YsnWifN+O7RefXgpOfW2", - "0+vyG/c7voh+Hp5fMoj97+P7v4/vH3F8d9rulA4dUg5DSk4bRY1wkiELFX1iPkAg4Xn+JOelkAngtKVC", - "LRbx2EJQmUZsdjQ+mrEf2YKbUphyCABMAIpl66IS39kSc55ziJyg3sFvx/g1pCMFX9EGiC454N+26mI4", - "dnYAjR2yCvLLJhLQoMiI1xyr8sSednN6YnFHOJZvdkpu2o34cA9PSzvRfzvntWfI9APszyGDc+hBhyNZ", - "Z+B9rf1nEK+MsZ6JWs8htRlOasdTR24CkWSFVvYRccblBXuLivbB2dnbQ4pSuS1it0E9XsfvBuTP3tIX", - "Oj1UKnJg9sV7wkX5ec8HwzD9WHT+MfuQvcTAuVk3N2GGOMlBJH6dJtYhF11kScZz2EB5ni3BR4u8wi0S", - "cbA4Ovq5PaKMj4+e/+W7P3+/J40qoXudhUF9y1zNeT5y7jKSMo0RXWUXWSHSjEdpXJGAZrq+jfqeiHz8", - "ez4VPM0z8KqxdZbnmRGJkqlx0QrAS++ssfDS5IbxiUwUxQrZZ5KrgydaGRd75V6UQ3e2faPDZpK50JU0", - "ZKqE8oRggCGehTLliKLvwWDzDqfSNyquC7CL0ivLHxPbRzRuGnTIgfv49PM7ryV9d/S8ydhfRyN1CI/c", - "aNDbQIJJs3dPPsGjkbNnRyOcOi/8XA9b2/K7o6MtVMm3wJLhpMSYJb2dwM1c24uII2qSkbSh/YLqd4dx", - "JM7dGL1c7zCIpLdGktmxlZcc3GxdkB9erKZkt9/ZU091mlhXI0AckBauFlYr95PdmtRxa4zQ3UjkK9hV", - "HcvnJqgP1UFeQsJvAyPf0XFF1u3BwTncet6GzYFX14XY1Oi+DayN8Ihj/IHX0x3Q70S+cnldPoXCoXvY", - "RwCgRBhiF4NMA9kB5LDLNt6ObXCnKaHve+cDfvfcvwZSDdnnV6eo1o7OP74BtzoFSPnwgSg/bw9DGFBp", - "MeJDYtki0AIWYJqNgvU6KNad8EZbNuE9jqTHUfw63BXfUrADF2KPBvMmXqW9hWh3HOKq7q6ieT6qFlya", - "lfpEEg2saiqcxODk7sBb1XvH/4rgxni713TQLAHAGa3W0ftaqYsINKgLhG70s307t3rbB3ha2+iscjB2", - "dXcwzaAr2/kaz4lTNsZFRmyzNVmgBx+qITp8aCBYmU0GACvsNdJ24Fegx8KdNSpWmgMeWcEleCIZ4Pou", - "iNw2W65KZ+VbEFJIrQiOWCpg3xpxwl7a9zz5Oq9dIQyENtV85HmYh95DjU9KbrV03yj1iuuyZm6Glkwp", - "iql9dJ4wx5nN5lpxqyDYPy81uFzp2GbavsdKu/t9vgCpg76pmhlaCpF6stdrNKWvNqkImlptClWuREkX", - "iAFsCbJcD10S64qQC/wSBK3xuYFUySeJkoko3Ez6EQaAvbhmEFxDsztA+l4Y/2A4sH3rx5c+h//8u3OD", - "tFhhldpOwPwSCoW+lNvR+raQ2n4R60JprrN884vkLrn2bgSGsOEhKYu48D0fJ4OJ5+mIWgCadk+0CWuT", - "GWb4gpKcSr3pp9GMkgB2eOKhEpbk3Bifgh4iL6cqmboMpGlVDx35YdOpCzeYFjqjfCFTcpnON1P4vflJ", - "mtnpn1c1cWLzd1MqzZcCPy3rOW8VQ1oYLYDEGd9dbcrO/o89Q2dfkS38i61ro1pzWU8nXM+Mcqe2sye2", - "tgb6ku0N94IWJowyRqX7kWFn9qcRAGOyVOR8EyyY/TjCwxjn7tyLs/MXSa+rxs7fjcv2zTVPynOly90I", - "bLfzSG+rwXuEXCXvs3W2Zx21/h8Z+k5DCGlOt33wS83A2STZyuRy7xqg9T062qYEBmS/Mzdtr3xg2x5y", - "7lNVYgSRfwOD4cY/Z+wFnmHQZ52OA9gG801oABYyBR1uWOtgQzQaDVmiRUrghOjzGLqMSFIjlpmST3KV", - "OBg+RySNcMl5lmTEacRlna2gCtE0ebMvK2GE678LiPNRZt6eoQWYhuBRgvHE6BvE8UPuBYKb/bNSJY9h", - "rs8rbZom4OOIAufNls+f/vD8h+///PSH7wJDwXE0QpFfT+2tDX6OZNMfeJrJ0SK3KhOaQXgJGPZmyFaQ", - "T0KOa291IdNJRDG+a08REmi7xxcUj89YmKSbMKWZFkJP15msEJvkIXoE6E3tijtWeSP0JZq5wKOPYaQn", - "GMgWTiUjFxGCeRuhMwywcHAkc5VuICOegPCKvDJt6ynle4P2j+g1CS/GE/six144BkkMY1lABOWYnXoO", - "ACDJDtHHsLuQthJuewwktrX4UzzPMFgKMa1oR0M8IcGC+LjPCL6dT8KFs9SiqfNLdMtq3BZhsO3l51M2", - "uhevTwl0Zk/7+DRowCg52ikMMxtZWimBBm5KgnGBYmBsV+sCnrNk6KDa5DKTQD6x22VJeSVf9wqe0Pxq", - "aovYDXsmluK6eBDt2XNXR/QY28h2xZrKxdSKM1HwTL8zphLvM1MGZAotmhYKAXa7t1RINFOSBmtrYZmt", - "JsYqDy7DXg4k8ih6qpf5hnFW6AzDezFiLabG9aQVnglbBHzuPmq7zkWRfB2v7SKT6VYzE7nzcdb+Zr9o", - "5hHQjfHd0bBHttNMUVpwkEfQ5kUBo4aXlN817NBxEWnlSbldaNv2v2DZqNkXS5xVcp+9oKs6zZuGWHAT", - "2wroO42cfxdFLkXNUqwwkhTT8Sg5EZcRG/FeGcNw+D/ir4hBlIrrF0GUasv/pkBXsas+rFHTw2dXwSsj", - "pt6jBhq6qdbNPyVcJiKfUvDClC6Y6PvlPqeANs0thwHG09iEUZXwrQJwYonZ+OnIz2bOE4AxCy+2OlFU", - "KteJVMzhBezzl2gvk46G82oLpJkpkMmOKfDkoYKGCVUZKIwurwAJVn1kywuWc8whkJdCox/Q7iZBWE1q", - "7gI8eNhdAqLwDjDV2BPBpAVRodtFCAzaqWD/AlFyfNQrS4LgorZQoQ5jdlI4DUPG5YZoSi5dpkEhNKjV", - "fls0D9ntoun4aLtswoqijHve24lpEOH2WwjHFszdEEkyWD0Ko5l8qoRTmUKMB45PGLlknElxJXQwosJO", - "kbY9+L/+cTz64es/jkY/fH38p6jx90El6xdfW3vPwQhNNTcbU4o1wvGYQiAvl64aNqGAK474EiIdPxPe", - "T36mSGluiuMLsbkNR5AiPDAqACPaiHE9/HBWU9ICAAf4fZ2bFtmWGx+HSHuVgXimFqAYxtBOs/Tm5uTb", - "t8oAeEIcD1HJKUbARojSnFU7xMqwXX395v2bL29AvJUE0tTTdyvqR0gq6Z1irugjw2ZKTqsihTkoeLlq", - "BRpt2y9ucXxHPxWxLADfyB5DfPfx/M3Zlye/fH59usdIHYiVYbM/GVHiwiKZf6Lyai3ZgRXE7gm6Vqk4", - "/EOGjEduukOYBj0afLTGlc5KUeOWa3VlBSJBPKL9tTLlRDrcLtAu2juw0GKdVeup3XkmtuuuVkKL3fPT", - "2y6ccHQ9WrmbqHNPetlyJEMWxHSVyXKPacfaTuHbn+2nN8NBaiIEga8D0B/A7TlIlJSEc0VgwJ6egWBb", - "zv/+3kPtjAMigpnHG7FSw9Z2M6txgAze4xSYX3uePfKO/RLzgSbSbkd5mWklQVeBTIJ5LkygCM6ueD4F", - "QfNjrpb2oThzzmgcT3ux22go0Ru9IyfpeiZi59apDLFWEyURKciFYNHAGjS2MOrPP9Ep88hJ7eg9jJqt", - "wzbJN7XgeW586tQsS2csI5p7ra4ArFUqCsnEFsYTeUqQjhQ8diDGy/GQwGgOXZwgL+mDRw4d3WWWEdbK", - "7Ns3LHBzM8PH+bXnAEGm2QuxMYBz2SPUZ+31gAm9t5znlPKvZFPcN8QeC+X7RM7+VEkQe6ogaR8I+Ede", - "KIKkZwfgA6qdjbi5Dt28/Al7O3XLbCsFFHT4M4bYIgpdaz90E9FUYXepESWqLSvIKbHqW51kgvaVr//y", - "qyeY9dgNFMJfTqQXAcHOd6jw7U0VWTZ/WdmF2++22jK7Ys2zPMjgod0Kf765AXN+5CvH7l9/BX9pfIDs", - "nGEZlzbXX297rQEt5eGXuon7tEtsJGUVRqQ/2jWIWAJOUvuQ996smDoHHZ66LKvdYWDCr5HOvz2MoLdB", - "YXzL9YmGFC2LmPuJqpIPsh0TdPgmAKbGcNtCixF9EzbVxPcMjXlwgU0hOH6aVGWcRwTuBZf7mRJzB4Iq", - "4m19xSGkBZxsaOrAcHuqEY4BRkZC8D5mKKC043BZZQbArI3khVmB6QRT4kot+NonxXXf0qBVmqhh3qPK", - "YxnM+uJyhDwyfoeOGRBCQSEm7HaH0WWlPeKgcs0oM4DxJbcPPTrur16/gpuOgzFHovo3RF3QUP78RNZx", - "zbNQA5uNET+yyIXrX8KpjjrvRqurmkOKimHcO9FGEdSHeyDVF8eTmb+7iOspW0rIbv5f/+N/onMCR4zh", - "eOiUUFcyoATrSq2mgtxRWL1+6hIMywzCjDBZ0RW3immnqoUWYtd6bFlMUtxXGOF7NJaJkaty+9ENNg2z", - "X2w7v3Z7o0LUeC/YU/D5pwCe0vQe54kMz7Nt0zidSUl2XhUo+JRmH4WShz2HvI5820thP8fP/HR17uJN", - "4WVzY2pQ4WWvvLEODGghPqDDRwLvUDOAyBWyT5TgbeT+uo18E37F50Xndtnp0RM8U7rmYnyJ2OdPYH3o", - "DN3B3IFfzGH0BknlgOmEMY4wM5lkP735QhsCs9XrIO2YtxifYTFrTp24jmVqDRulc1PQTwaHPaEq3MTq", - "b8W8YDH/IPMj9LFEkYq7nZiixtGZ68/vELzBClKdLZcCLXK2gublMmTZwmVnzPOYcRSHlH6S+cYl0W/f", - "COc98aJniIXnQkbVom8b1NzsnVWeyPYyD8Fjmkm68J+gGG/lkrW8GiuRXICzM+qns7cqxNZeiilFkoWY", - "VoGh1C3EmgLIuvQfhGlba2ldTwC2MIWAsmiJnC+nGKIa74X9nYzJfQVMOcVE1imp/FPusll6ivuo6Uh3", - "TElQp1O7b7Nya2WmAoTh28oBgEW0PeJ6SKdbli2mTHYlenhfdX8lJWpaY0x1e0pDd16DyKVhNa9tvcVt", - "ess67Hn0wudCx/qkip0ehCEylirMCb2gh/i/n+Q7aYTG/8aXt/1XJhP436IyK/pHnsM/eJp+UedUbJ1J", - "/F9+PYO4C99zj+1t24MX14wdLHJelkJCoMBLjCdu0dIDK0Hk3X5AT/bOc33IQs3OSpbDjm0JX3LdrRUX", - "tC3LELHl83LVSj+rp9FNmyiTZvqZe8N2mqZXZLtteJj7CyRYwVeYvchZnpVC89xxgQ0pZ9qjMR2Cea71", - "eq/f+KVivH7oEwUQq21IYyCUmrFSTaTkl9kS+YDAZJKolGI8XgYmso7GofpiHMJApc7IIYpmihksLsCM", - "p+uMgk9sN65WmdUW81xdQW7LpdA5oSY51+DY57DBteNiikztO7TK5hV3frazzx8wVfaS5xRiap8CE+li", - "lBaZzMzKXkd1DjxKAvuuw5cNJK/YYwewcO5qqZkJyO4PAaoiZaEJV5RVga8+222NGXuM5Oqiyn0QkWOI", - "GzJeUhAUskb4ADSX7QxhUugepXEVQvt6AhNtJFCMKIK1KX88HrOPyIsCb1W0FS0rrrkshWC/C61YVeBU", - "sOdPf0DS8zoSENl9HAlQKyg+XOzBcFAvWY8DbXsI67aQzKbg9Leg65APhfbgelPYg1NBrcRjoZ33tS+G", - "0NfGHFIfQPHUcVEeYdZFlUWzarZENO4YkU2x2DXDWKvpRybobyIAsyqqtLoPdgWj+ezKt4O+afZhib7G", - "jBn1Q42KPn/69OsOaUctiUTVuJyxMNzbj6VezW0x4C2MnZN+ev96M/UwAHdBTjGlOd84JIERpbnXGOG+", - "UgTCFDU0oYf6Z+8kCxIMGgDBYOlA9LAAXJIgb0FxX3O5CVoZkrjxSJIUCYhBJNhf2t41K7gPkswM0OlQ", - "+IlHPcUYDqJid3GdPtbZSl3uYF4JmhWMSvCO9DjCDnw53ww9DZnHPQavCaCYQHwL1xu4RN3UqapEIiCw", - "L9Dh9HGiGPwzZPMK0FHM2BEyI7Y7EqnCSs7RiQB0uTlRKbuVQtsVLD16PulQYY9MUJKscEO2FmulN8Mg", - "VByl+tDxxCGVhn0L2gsAq63njQYSTAYsKwSHsDANm6CeIR/JKhVAr0vde8H+U+hSXAfBRSi26n3y9Oho", - "XEOUIu6xA6NHfDhopaZ3wY75lPPOyMd7xbLsRalDYZuRhGLPsULEd3gR19Q5mP3l8FJjkhCi72+JpvHL", - "CAXJPNb0KfaJYteNABvEBWo8uEz21912I38zEWKPheiPb4nRY4Lnwp6pWg1HbsStS1KqYkqWGke3GMcJ", - "rOkYkWatzCjLzCGqQGOUmo66E8Zq4wIGFucaZBrlFkgdJNpaIGETdRjRv1HqZbWbDks5sDsHUhhr+RWG", - "UaJSwlrXC0rHuSpXoUmcOTiRXa77HY/+H3NYW5e239GRO3jXbCsUUR1QvNs/eqVWQos9P8J5a3/0tZvo", - "fcoqie6PLosTD7ZKqBCmma1hjdev3dsEFgu6BwxxrwmwCjd6FPaYgOHgEoa43wR4xXvzEYwz9areNBAd", - "PbIrspMOvGglzXx0+Xx8NCq0ClW2EzeOm31yH2oBGAd1PfNLUKfi1zir3Vhqf19MSf9Kb8Mwr28XUFCu", - "hPZ6WyjYA/U+pR0SRan5AobXwp7zeuvcIfW9/y47c1CdHk8W7zJPXeHvOHFttYkAuzu89D4R5wFsc+e5", - "ZTlALHk+rAe+2fYQKIPO192p9hp2eDyJWKgRFIon0p8yf3Lib1u0O3651aOkqdQJWsuHABwyrIlxIXae", - "VPGDR48fHaJJpybOBQ+0q4U9evzI9ntp3/IBj7ITTPVn6GENB1dS7rDtAES9UtHBcPC46aFyJWNDLpUW", - "f1XzSDwdml0oRnmHXTvnyUVVUPmuyR/zAsjSIcJCwSvToR6laM7ve6zVh7jm4ECiN0KbrKP3ff6jrxog", - "JK3qC/YnKB4YqmqjMsTsV9r5bfpTwqKA/TUwNkmLHWYQWgPNZ0qoMbtOQ2PwQcR4GNGRGXaZmcyKA/uY", - "ooniVrkYYRCb0htW98E+l6xm7pJF5CLTaxSNe85Fv6eD8K/qWWoO8BeZXTNRqGTVEKOkpiG9OpKbCb0G", - "he83Naewf8x5sa+hVKwLhRa5C7GB5xYgUa7VpUhreYg2TStHbSUI8ezq3VFw/6bmPTH9EMpPu8ltwBG0", - "A8nCi0zoMXuFyNf4aIQERyS5KZGHAU0T4yZV1vEPbaasaCD/t6Ph8V9uotH83i/kBMs/K1GJdDAc6Apg", - "w4B4NSHjG7rT4B94pPMei5w/mXvtYX84M9A+3Anc1Ad9zD63zzxg3qeqgFccmg188SFtFttr+5y4w/bV", - "NeTX7qnmLifM70ysZcxOGbr0Sk+ZRKwc9cFzkFFrgF3tP8MTedvxveKYJVufXPYqrwzx1tpJBs+w0qUH", - "pRLOnW3nzXdzOJF1tSPq3BCIXwqYYJkSoBOtISy1QWtSTZZiEGdWLTx4SI1+P3M+WvrTtNSVhBfhrBHV", - "xUtWr4G374Ohg2sCOMuVXLaaQOWuZio5lQ6Cyn7aHZwLjOcXBA5qTyoSAMzwu9kLn4sSYiYtqCcZWhyW", - "GiBD7QQh0J4UVx155IkHN5iYU+MhO6SxSNIgLU37bO25s+uRR4KLQqgrnI9bbhOrwFOazk43iA+pwQmP", - "SpDt1+Gew8Wle4BKgp3agBu8i9+jTXsYybay3XUe9T2K14copmx1UjB8K7VxPlpXTGdf8+t3OAl/6Uaw", - "9Z7tmMu8ohCl1jGvhX33rAdkykoT2k3jwMejM1EG9m98NAMg896oeTmRLA+VcSc30YeGDUd3NYnNe+7C", - "PsS401qSu4uHgoFwOEnzBgiH4BXkQezoQfg1CLZaBYgNz0/EvQYYyxYEEJRQT3HPGhpTvDuNwxMxUZYu", - "dKhPQTlzL5bwcsM9eiHVlXSGdG+iuIOG0QlV2aprtk4vKZ7D8NXmZix8mTltL/Ie2ypv+1S6vldb+w3U", - "Hmb87e8epO8zU+4pUn9T893RX4Onb0N+QXpxW4JJcV1Ot6SrS3ElTDnCjCRyT+FVSZn07BTN0YR+bZdp", - "Q5ipWZ474qKQN8GKwapcKZ39TsAq2pMbA849FPf6la0YvTvho+b/Ye/vm9u4kX1x/K2gdLcqtg9Jy3aS", - "PStX6ncVyUl0Ysc6lpOcPZcpCpwBSURDYHYwI4m7lar7Iu4rvK/kV+huPMwQQw5lO7t1v/knsTgYDB4a", - "jUaj+/MRjuqIMEOgmLQVrLhdN3nnQPH8ePCB4i+pA8W2ZO6c6ghU4YCpbvkaOkxQ/kDlrteDjwBfwyuN", - "oP+8nwQfj58fP/9ifPxs/OyL8e3zzmnr+b/vGp3T8X/z8d/t8IR/TmbjnjTqEH2S6MU5ofUGgJ1rH+oo", - "9XUcuoJEB65HleD59WSq3iJ/lQOa8izRBE1Ld5BiIe8RXsq8jI24H99dIGWQ8YmIc6EEBL0i7o/7eqU1", - "2ajROH1+/Jc9DM6/jY5c01LAgAC55xvvGBVcxiBOVAvmOxc13H1CYgNRIFlTwbHqoUYN2ZRBKNZcyYUg", - "DnOXRt6NWzMvTp4+XW9w7J4SnDo2wzyF5AA0EZ4G4Tl4RDorJ9bafqhaUpNeVnSfe7oUqh++hh5EAdLu", - "Gpjb9ybMsZxiDobztdB5W3kkfED3dySvENAAror4xgzl010DAC22Q1Vv3btANsmTJ5eyFICNvta5ePKE", - "PaI71xFb8/uZI1OYQewdCPZXx49PfHN5Fdj3psqz/jlmGK7Y69dvWK11Mc6IYa/QuqRvw5jJzH+654ts", - "2hwfvxDMftiKoK0yFxlQP9BAaaDD0FOVAYVH4BtwAK4QtoA2mmAe5zExwnbfIEz4Nd94bhHkHiebhPDh", - "gFqhKITLseVS1eaEASLlzOF6FmAemjYtwxjKQBqRy91Rbeh7SsMJAFOtS9GpegdygvcpjokM8oQCPjjQ", - "+p9fvXaV6Aq9NUSsCEgPtawbLwynjW1HLfFquNJ3fveDaXas9wjMAkDDCKNiB8IlQ+ZT5WM3maRLWTu/", - "7TjCDAltnPVMxDMAzf7USuxUeTQWUDubCEeHwGLAl3An+I0gtBq8dLGiwHi1xLumdMpE1qybAkwj6mAq", - "PknBkG38GIAqQzwckFHMqjAOGdAOEQLGIDHFlLKuYS/x6IbC+VYCcS0EMeGAhqQg+HuqSCSgT1LdklaK", - "R9KPO1vootB346Y0o/RAtqgUaMTtboKzcQCMAkJaXpUiSyVQgUjPrK1eiDwVnHeu11yGIGnmiyLiCNpZ", - "0sG3C1V/Zhhtk2Wl12WNKc6LpnCUjLcyxj1mfI4c64GUj2W6KDwJb2ungUjV8NgteAORRu79cFENKVzZ", - "WjgmSGYaWadReLIVl4nN9mwF+mThpk5DotuggYdXX0t1kxp3qxGNg/XfQtx2koiYzybQ5UR4/HYvcWd9", - "ZIUcZ7yq6MbeG9iToc0F9X5OzUo2mYZ3hhEvqbvHVIgMhsLYN+ysh3tlkpn4PsbJi6McB1iB92/f/uAN", - "FDunAGMoFguZSaGyDahXChU2tSjNJKqy1lYHGVFvo8YIlelcvNcEczLBqP40bIyf/L0MFK5gCCzw6ue2", - "jQWRivXxRK1+NcGFDNHgZgWvPORzhCSOmFtwA5H25hCWKPAHITZmf4gnPoeRjtajItJLVG4hx42du/mc", - "Koz2wa3cAUZShE+jUHEuHIRkGu80YU202bVGe+NSYVsZRxJQYQyjB3z1lpxV6WozVW7TzzubvTUWrH1R", - "FPhPhMGNaS48Dww6senGhxIh3tuBwhfxBsYd3wq7OwIjNq/Z82MI4+JFQfRexyesZeNh1i8aGUENODvJ", - "mXDWVABDy5prtppn/3bCYnsNqum3xBiygMXxUs+P9/ll7IwB0kFLLs2Q6OG2JCNnT1fkyQR3RxL7pU5A", - "194GYvT0LtIktAEMD3/68L2YZRAwF5lTjp7PD5qMPogaD1Ut1wYSYAHrIXkhsH8tg+SA3aR2SKIJ5UwS", - "sd3V/wwmNYkR5c2jyYtbRpRFjpTykuNZsXVUobOi1ZMgrReKlS1ZfdR/CAGVRoaXJwCP0GEuFA4Oyasr", - "nhMRe3TCis1/XFrufACN2qYIZlsMwfDa2tuMdH/jzT3kNvOqYoyzFlyY7DJYhtRABGBUm8AON1Xu/EG2", - "5q8aPEyPdMUaJbXy7IbUNvO4Y2ZOVdtwh6pHkQ1Z+VMFBGLPRctmHE1VMCxj7lFvRdLlpLNESeo5TTTk", - "BbVM+dh8D62IeKe2cAdomhwtQWDKsadziuS163iLhe/oO33Hcs0ugo+HvbUnnv/fkadqsVWaowPgBGg+", - "Y5nuW0qJK78fDcSiqiQfYMu22Nn2PiAPZxD2MSAjFCdHKqYF5Fgx944nHXBZEY3yzNVuORIm5uQwHU/Q", - "1y2rgTbjz4+3sSbfo/GALkRA3QROl/g4MHJEJmGPRjnERCzH6Q6IYHMIdydABrZtxfT5CZwaR4zsYFDC", - "C7bdTC5S1UmTMBU/38vqZuUX0v+SLtczXVWioB0huF/xxql3YwhY3iJbafLOugQaNPdh1HItMJBBrkvA", - "P6tuRTWGlBFqFCtFhSgtWfrkA0bz4NUDVsUVvOKTjffata8UzMvV1asADMNuDeJwO2syabqChTJkb7T2", - "lnFWd5IqJmySQ1yDLqolyRUXEZ61PIPb3gtKd+/1XHxD/ooQZes2gLypWmk6H+e837K89ocdwUEzZa05", - "QMRMK9MgBizhCqHUpWOE26w1e6e1Vfp9i3UqULe5NVyJQtxaEzw1a/QIY1QxmyyPY5xjEtZHVfs4CVo8", - "t8vn8dGDKWlbQY89sXy1XAtT83XJHlEs3+P4ioryZ++4ceBSw+LurPDMQpjWTtaXW16EkUXLoCln/qiZ", - "cFY0yyVeAAc7IpxMu0PpanwcC/P2vXNHZsNOsf39byNriAZIKrbm1U1uzVkcngDogYl5HS/BVMGY+suk", - "HjSe8MIsiEQSwIqedVwccRs/lYw5fsCu/DuRRwmHUqMDiEC/k0lLKbXd/ajk35pIXC/OYasr9HLpMovs", - "ro5BE8FmqviymmXVC7l8frz6op6bP78QL/iqWqah/N2lehxotKsXF/6Nc3rhN+h/n1gRBFKwnnxZCnHw", - "KEL+gnmX3nv4STRitI8cChGNrgHz5hMcTId7mPccTXsSNl6/fkOXAI1xPMlhoceysRRW5sfPJ8fjRcHN", - "KiUSO1TUWY/bDPNAcrdKHWkrjog79BE362Fe1P+kb6RGA2dIqmW/t6u9dKl4EEVEb5Jq+RRu6hoIgm7B", - "Oe/fiEMrBnlwQivStkCh72gK93/6gbbyHhf3A7HTyJp1eGneFO5scknR8FlXVp+JGFfM2YSEwj5ccKxd", - "nYS5IxJOO1v5YFPdk3qSHT0D7TFb8xTd4HvIHA8uukjd2BdS5mgiVMsBPAw/SvyIaqNjsdMOtYO2NHUg", - "Se/FrdTFhNH+mQluLBAA3KTc2pqLFb+VuppMFfjNbBEIU1rxW+H9Zv6GABw82fat0WOKZhJluEvDa3+r", - "ai8WkI4mSsfeFKyREZNIIomhkdZ08M1H+sGqUeZxMhL6A8xtO6LhEqNt8OzOSnUl2zWAzAwwP8Mbzljc", - "e+Chcu3vtY3GAXc2W30Ooj58wYX3f9srsz+m0VDAh8JgJbmNHrPhQnJn7/kzKKVtWeDZSuQz2Pp7L4Eu", - "wDAgbwj4Esj3Am8nl/zuClGryLhauonhRQEXEqBikjUXxRo11q4Dqq0i3LWjrkrWhrRrexpK3GyHthQu", - "uGZ2fvaq+Utb9ApKonsN5tbM/Km0r2m+KFsASWPUOIo7cN6NNCMehPL2dP6qWduxxGn6t/YoHA2LP057", - "Vw9jHQ8+2W66ZV0J4X3Du2t6XwlxBSX7V2Ii2b1L/C9VZ335LQICtrhKxmxNVTtoK75pcXjCnbuWCYsA", - "HqwICYDTAtc9dVmYDvKDNJQB5i4UIzJcthKVCNjDENFDAoI33OxRx9s+gtijGThR8JfHdEzzgz5VeI+L", - "148ODAQqdbRptgAY8tyeaXlVM6VzAm+GESKUNgrqiEUmUpsH7+B2z5wwXxXdaNKVC0Z5wYUFwvkGDL/A", - "RmbLA7PoJg1MkdpYH+yV7F83AxnY/Z18h3GdpvSE/QS5bszItbSmer1xE4VRduEKxr42Xz//4oR90xTF", - "GE6braJfv3n+hUP7sIUd9vwJiyJUiGxfdWAIoXY77SfsAs8mt6IrBriSEO2LYrjtWyD5J+wdnQPMSpbj", - "ObeS5e+wkCp9Xsn8hJ0hTxvSKpQFII/b4bILDT7w7t03zKOxtCnPadCORkd2IABvC7tolV4l7EYCrQH2", - "c/u5ntT4utr0ye87ZOvekuJgL7ptZTuGWy8WM+qWdFdC5GR/vhUJ4cvRrSJKI/CmYVU7/UppStXgV6I7", - "yJlvlunSjm3dBV042jZ8BWEFu5gPLdC4494wA4eK2Proi/3RIETJ7d7uix7oJeDt6e6L4+N+nrV9/R20", - "m+pCnBojl0Bcsw0/UOkkIoguIKwbCfJMgzHutWYcqhqxxjSwXWBkuwP8t5Wd+PdO2lH/8JC4WCrBEYFu", - "D3+o7gOk003dAyDRPnvYqUPFYus48T4b9sgfnRY8qxu3Wz+2rSfd9Ujcl4Wu8GYfn0ZL3tVkz5ZoTiRX", - "tL4jInBVJ9lno3Bbt3fgfR7cl2hlD3OupbXDSGhPYWCrSGd2/OO3URrvFQcGbs7g0ijcYkBmPBERQNwb", - "5aC2ZvQfhMYPwFO8qlHAjpDyVvgJ3Zq8Hp6P9yG90B58P3vyGY5EUVC09uO2POkqT1J5pBIurcbCUUpK", - "E8J5o+Ltw/z+Qaux54TymX7gHqVrNlTNk1+NVp7EYsK+03UrodehS4UI70qsNSBagSE2hohzTE42L9EA", - "asrWr843i0yEtvFg1NuS29Ix4MqD1gnAeTVtemOII6n1mniPQzZH2zrtMjX23B+tuEmgD7/Wd6LKuBHs", - "6rvT8fMvvnSNQYh3bFJqkOeb2g6RNYIxHezZl3aI4KYf8P4ckQYGliAn1kIux7YdDIJR+r1+APddiULz", - "fEZ5mClOFAg1sN8oeG2tQOIIzVvUppRZa+vClH4yF9xAQ+QCWBYeqBYN37nIeGNEWwrI+Kb89USc5ujo", - "fqzXsv67qHSAgYr64nDgh8wZvUYAvcPfA+HsH7IWU3hI27CjPoZ8z/FS6zyQwWBcCndDia0a2Pff+tf8", - "uciKBCaPI7nsAdVfyCUSbrcY2f7xWx9gYrucS4Sa2dmMdpTu75j9NOOlDBRwo6M7Xd0sCn1n/8nN2m4/", - "MhcZt7sp5mMlNyKXmrvTaXZfC2WkViG3b0udQj0pJXp19crjEndZ6Tw0R8DABCGcsDNe8sweLQgxOuNV", - "taHUN7xlYxcOcMmVhc/4+5Q4KYIux0Z0sTFiHtaW/F91tcGoqNnavETLGSwAwreAxB9oASw0SHHf1qgi", - "3U1sFjUBzAiHbpGtpBIBlTjTeTp8ZiisMabeR0X2MHg4AfPQVbOMhjLGs5p57PGkALUHLxGaiyYwGcpJ", - "k3VytJ+vN8JK7te1DnVEYnKArni18UC87jzu8Yes0CXD2TuijfPaJ9u3IhXfcnVFFHNgXxomKB+Ysiu6", - "TtUQsiRUjpQK0SGyFuUMND3EY8CfDtzE/R2DB3R88m7G8T68dQHq/d9wrX9EdzigmcidHrAocq3SGuTq", - "6hW6V1wLEsYs3jvsugVtX3UNprX5GSBpKA+XG1bzG5EMrYhu8dPWcCqo7DSAYND7US6R7dNRQih2ohRe", - "RdFzUZ/lWgw4x9FkzHr1wStSANhUikQBm1QaNsW3p0fJeIf+OAuYO4qxyKLb03acxcM4x93FmqMbT/Ns", - "XVGWpW0InAP8dX6rPeiuhmOLbRuEAjbVgmfCfPD1rSjdFW5q3xs5+exREvb9S7diD5LAV6Q3kL9QV7WH", - "NYKTtbTnQIIJI0fl+5Vg17ZV12gQTpW/5nY5GTCQNTIyUGVzUWi1RGbiSOrDIpyqW6tM5xt6e1MKRyAV", - "eXFdup5huSjrFVByzBRme5qGEqpqZAdJ3iemBSCmetzZ+NZBMHbp7zsN7jJfaParpG57FQM98PY1L5uL", - "pYQwkL4rsz611tnYo4duGEjnzQXCZaHWC33H+wn7zC4TiA3vuAglOFSjgHnMePgYykEtyfHa3ZmifH54", - "NDAY619clXRECQADBqmF91oXb3QqVOJVFz6EzASAhzWMtzPoacNui9Zap2MwKObCvefRsCODg44LEBRC", - "XvgZ3tilMX60LswAfEG8N/G3SI/KCOiJECOAEnc/3A70LTmoGVffi43Zi7tQawRiuREbyBN1mTReqd6I", - "DavsQRpDJ5TGvyBMwtF2jjBTGlxRriLPRpvSbpSJA9kOrbNfEsLQp2o6rBn4hmeS/6zWn8XX9HBJFjIR", - "ThhUyh6BSrYfZU1ZCgrIe9xHmopOhIQzRiLJDjkZ2pnZyBlKAIswWnU0SlMFpyZMMcUsvBux8VRcBq6X", - "riSgE2L1J2x6VMu6ENOjEZseYZzh9MiW+wFDgQEaxRaDpD2e53YjmNijCxb7WRZ5xiusapatGnVjJk/w", - "2SufD2Ufjv3TSVCGWBCYN3kRNSpSl9i0GV0uCfc3VmbfbyUqYXesRoC+RBdBk5pjstLwaOU4/+wQMmIr", - "LEmViCneNDEk6G6FPPJCazd/klgrPp6Ha9Gg/DPwT6PnPyEG6KwyEfUpbI9gusQ0zilm5pNnx8c9Ybq0", - "oELfBi0pJ7ydJWVr+SzczPctqTAkSBcCS4rAJ5dSwSbSu8I8vcT+qyXjMopw7aQG1a0qyK71HYm1EWox", - "FiMk+EaCO8IOtKdR9uP+7Hjv/ZVTYu08nHyHLAVd9KGyBIq31ihIIny0X4Sep0Qo5QK8Am++v5jpBDm1", - "Uiy8wzPntRjbY1xKSoW6nd3yqicY1ZQaQU+FupWVhvs4dssrGd17oKH99vLVD6cXs9PLi9n3r/76OGkq", - "idT1Pl5PxFXpUiguJ7yUsxuxefzw8xHWHeJbA8bc0PHp7O22BzvDMvGLDp+uPTt4ETMchS6e6i1F26XO", - "orr729R3RYSksYFtDGajZY23YCBdWOYMWBlG8S9C3bZ/mOs6fcHoWqQrcfjNlSMjgA4DLBkDcDOinQJK", - "vciU+9DLpUCxEu6R8ONjxPv0t2YD75M+wR1NrdkBNwu//60KUrbuGnfESRwxFaY9ArFAwyZ4KDOt6koX", - "rCy48mlGERcOXbvFMjJhFxBGrKwOXyBWIoXIEPYXzzy2D4xt9H17zIJsdIYU6RSBSg0Fck7wE9vK0RHc", - "Yoxtqa1dbmK64pYYfIqIz1uXS9Sp/kumRSXMarAsEJ25mQ2YpSCWtokg/XbnxEwSWHl4+6kR9RJrjNFI", - "OmNMWb54M2zA+nG+CoTWBbG2Z2BJCAUE9d3CzQELGJOopSErCHKu+EZUEFUIcObIaUCLyI/eipsQOxVf", - "eVO46Qfc1qGW+7mSdQxk2dZFPfyyV5FksUfQAZ+oE6E+lpWukS3Kq0rHXYHi6ijyHu/d2bAh6e0D733B", - "Uk97BOyvmOru7ogxSiNMEhyjKgFMrCcskB+7aFOA5yg5JGMR5Jt5mmmFYQe6rOVa/p2cIbcU4kcBL68h", - "oPG5K4T2LYbIlbpeiVpmvDhhLlOS8dYDl46PCcgASc7LUvDKDjfl1MZBJo++25y/YmOGX33R/Wq0V1Jv", - "IXgufK9nP6xKUbXCdgeHDv8s5u14363oYbKjkv5+e+Y+vbwAq/YRDGrNrl69u3z1zlly1vKzFl/aDIMv", - "z2of1OSui71nM02U46HiWCkqKwitsER6VYk7SNNcQ+rg6Ai2PbhUWmnk1Uo6fORazEpRSZ338DDhQ4pf", - "OmH5VznfjNjdV3dC3IzY+qu1VvVqxDZfbQRcULuG5Uejozt7OD4aHW0S3x4QaL0dzotzMMnFLftWa6vy", - "cDbtrEAYM00SBR8DbRN4IyrGC8iIw9hS7V4/a0yt164Wu78h17KoJZQsK5m5TFTBzQbJjQnj8sr+++TJ", - "k6l6NmFXcqlYUzJes1Vdl+bk6VPjWztVzyfsW1F76UFSRm5Wc82rHKs715k5efIk8frTXGcmPoNG+mbF", - "qzysgk5c6KYWMzi37bOev97U4h0U3MJODXX80vd908s7Bo8pinoY81ifkYcfQs1DmzdlmbXgwhP2lOed", - "3l2p0/7+sB5eLDaT/kSJvhwMQ3Xzaimi+2LQ4ZOBoZ1WqOD6V2XitFjqStarhN/HPyL397pswHVfi2rN", - "TFQHbCy/FqsT9h+vv2Pcv+aQYWEnWDd1w4uZVB4/4YS9gd/YRfjNlsxWcmb+1vBK5CfsbCXH9AezNrgt", - "UIoqE6rmS3Hi3ILhJ8xTqKSxlUWa49fC6rPtVtjjUvigFRxfVXqf0FV9DqBRaWRkXdUEKhWoCsmQQtgq", - "NLG+gjsbR9WCBtNXjLuf0iB6tvJvHEvrAVw3p64FBljXgdAVOW1cQxNcfcJk+/1l7e5+SN962Gffr2jc", - "PAMt9GG+mew1p7DGpHrRVR0plwNGEvobVE6JlThIV/J9O4w+E7OysFzypdJWP00V3RVN2Gt9R3oM5DhA", - "h8JtiahAXwBmc66zMdqi1h6fKmvyqKUZob2ZjylDzz40I4dPqqunxHPsUrfhmanHkPY+VZBvZUZ0vTKm", - "eGKC3/UoZ2MYczrOY/7PouBL7OmNKOupsgc4vWDKLquCXZ1/7wCi0Lr0UAFQ6ZpvnIUHnn+pbEsotzwX", - "82bJCr0EKhFHXE0/u0u21OUJQi3NkGRZajULgUzJOz4sz3x5Cv6asO+BNgKPlM4RPFXXkJ0Eh6tZwWsx", - "g9BmpGMAlW2uR64UCMLMjlrEko8zAYXADzvjRTGLiWlhypBduFMKv5qsyn5E5DMjlgCMajKu/GM72FN1", - "HfGbz9aiWooZ7CKuWA/WyF5Gdk8RHpnmmVZG5oLQ6ExrtQzkaAvfRZFOuJDhgBsYeUn0PQ4Q7Ymextdg", - "WIQdV3nbolRRWkEk0721G9RyRjnffuBBxuSaV5sZuh1xnRCWkjWD26O/Fuu5qMxKQuCPlYRSQ80GqWn9", - "NOA2DnyXdpXOal3OboipQ5se6pY+gosz5KNYuxMgQQAg8vdRb0W0UkS+d4INc0Xt2BL9Rdhnh04sfpWc", - "Hb3doOcM1eJii5O54P1L1M7n9YhdY9t4rSv7l18bQtzYv1sTgb/hQmkUfVzkfasiftel5ScQq0Mp2q9c", - "Bn+y5YwabpczrEzwr0I4L/ZGVzkyGuP6xQazQ9uLdfdMNhmtwCGUyRKjNKRiebcrh65n0F2KApsSC9k+", - "YhCS+MDJhg/Yf1DE4Mz/wMuy0vdyzWtxyBzDnkfaG6BxU6F0l6uNAedFKM1c6a673JP4MMg9oR5NVVcK", - "qNxsLQ2oINsHpWeFVTkkxLjJQLytWnq9E9qA8nILBE/kWJ3BuBg/AvFj96xnJKC5s/lmF+sSlCHTJ46p", - "th23Rg3s3jOZs1qK8bwS/EY47pwSqF/pgmDYfYy3gBPX3lZaenf70s0X7gYFV0S3vGdBytyvQ+nVxdau", - "Cz9bmcT9IdfZDGsEza7glUxXAhU9FI1X5s3sjm/82r5Gkw4yBWZg96Skl4wCmHTi3MzI9wtd9IkSjq10", - "qows0Du/1rcCvQXku7DjARfJiW+/ZCRunrfFy7l0DJX2dLaqdLNckcA7m4pMYqlMjZCauwBIKa6o1+eN", - "vQvmt7P2XS9sxaSlwoXA9lHDqZc+VfidtOd0p4OihG88yZd8KQbqPk/Xvc8WxcdIfFNwpUQVGoCefie1", - "bYG1dnNLYlmfiBwucij8wRyDhU7rILKKwproCHikRpPLpX8VRcvHH2Tgw6HGCG5gli5uNR2Qrcz8fPfU", - "i0uL5spbd+nSPRIMbSrkQmSbrEAcjjQunxHVLQqrL42QC4nNAuec1ytk+9kE7GtCnYaMQDTKyu76fwmP", - "C8f1Ed76m6/K2hCAIMRVtIuNHdBDrDShFZHhHAkZOJgRgfqI9k5iWbwlvkX/RQh49I1Ksy/agQxHOAxV", - "37FsnJ7Bgm5xwDDgqdUFuwe7oVFrAL9HQYKRwXVDP9MvaGFAIZCydml4lPGCV33lfdG1vHe1gpe+tYNH", - "QmawK/AKL6x2aYusf95eITdS5S2LwQArNVoNoRwtSUgcDD/DvyQ0M29gN65F4mln54m3N+gQS/XH7RJ9", - "/c14Sd4E1B52h585UYKX3H179Ksd+br193bNW3MB3YteiXBG/OYOs9RqauusF/WG+WGkEw1gXLTGlx64", - "wXE/gwjVTVkIf3THP10VLDFDWyJTYQbizIqrVEt8y6NRdr6Bwgc/uildc6looyD9G3sDaK7xibfA4qd2", - "ZFWzFpXMZqDz41nwP9hSC6mkbRL+ZnsMW0TU77alq7DobC54BZXBwd7tRLUUZD62XouM/FnXd4C7gN3l", - "YZ5xCxP5rKSJpClG7a25NXajjSXlX0HjQ2+Xa+2IkK21VYS+FjUyXW53ATfbqKKx7dYwSLUWrfutD8Tx", - "objlB7R2KNBhK0iWgUWE24ZQdbXZ+ooVYTy4FrOy0k6lb/XWyxh2BV2qMxKoLd3jPCTOsPDTHFsw8CyX", - "xn4k7x750QUWnwNHSSeZHdSZMzG3ysSLZhbWX085Ugi+OCzyD3TQdbbKHgf6VXD5c2N0JsEeAO9/21LH", - "bDC+EMjTYQd5MuCzewzb2C4Z07lkoL+1uyGz5P7aEY/2Lp3cevFPu41QV3gOqpdWhX3gLZaUVt/S1S1t", - "H5WPVfEQPeV/JE82shm1dznVFMUM8T7CNpdUmtG2A89hfPcpxB2LqP/02S/GA5wtfU7erZsWct9iQk7X", - "xTvoZLTla8dTRUtzZ2lFir93zjjsuvfksTUUO471g/x4FBlmN6k0HSucF/wVkIOg2ux3OGLFi0oIHAxN", - "TmF6gInbpbPjqA/0MGjHwR1BGMCmF/+w7WOMMl+zSkfp4EMIuLfv/Qjb7tuKlyvC+4Nb+R0YOLvdUXGF", - "AbV+SxNCIYQV84EAN2KDghzoL9Eh9VosebZxMwYwcNoAig7elvmcr6wVHe7S12F9tLH7iHEl7teXn0Pw", - "fvzTs30j1sdI8W3Ur24iu4MT7MAH1hVXxgrKhJ152ENstCMWQ75EwaJPv2R5ACrswBOu+ca/giMIL2IM", - "DnFmUGiHQ4kIzSFW7qPRkVZiQMBXe753l+1pzNFvv0Tj+5EgK5XGt9WG3t7iD+oMMmSD9wzpiAIY5xrP", - "/vHdeqsWh1Hn3zz6xTZm6zIpOc1ELIiuO4/kiFx2Steeq7bAVbGSNUYpIulQzssaSdl2tQ7DGmdOJ2Lz", - "whC3r5FXUlS2G+D0jmvNVrLIK6GOftmGinavYM1++Lcb1R6iForEViN/2Q5e+859iUFrAh5hz3ChyukZ", - "NJDAbgJiuxGI0UXykc5Aj7BD24vK3XmBZ+u63f3raN1P1XVrwq6dTNjDRSlUjj5rkJARK3WNoIIFsaaN", - "/9bwArJ+XCZ7LUVLhOaw3OcAw0mGrzQRiVwL3dLu1sA6HQ+gk0vMWm6hjkLwE7CpnbAL2l0JdrTS6xb4", - "qWcsdu/aVyHugVw7J+wthkPE1WCgBN3G2epaYU3w5SOi+DRbvOMRdUhb/B644UXKzGcNPlhAeCF5uCS7", - "PZ48p+EhXkCtgHwL2XtZR0hehguOiHwehjtydrp768lU0bba3m09JBqFy9spmm9qMV7oamz/wcJR++VU", - "BYheGIIAd8SqpsC5KggDqNbdFiObnWv2+E7mghhK9YJ9+XnEYsd4YTSlUBhA0wU0PXuYn6pgG93p6mbv", - "nuYoNbubf2erHxA46+2YFgqgizOI9nHkSUCa8dYQ7Nq/kbUAsXKRO71uGxB0kvSWA44HR6rDIEHwXBK4", - "K+7q2/ssBSMlybUoTsknTh9iwXiRNLUsCm+PZO29z5lKkMOnavOhZg1ZipEkJAahq+mreBCG3b12xpCM", - "3a2cuF4jMn7xUBvHvvQ1NyIRW48jVgVLfrDF3joCDFkC7x5m4+Ig6IovxRsOsNmcYMmgqNyljykMsZNO", - "YDYqW1Va6SbOCI/uOLOVyG56HtlWtjDqoodaFVKJ9LNbnjXNOvWsYw/hx8OX/Ku+/lG7C8kgTRywdwQ6", - "/+p+xRtTi9yDxR0SAAv7Il0Z1dHegrrDQ4EZ/KbHU4EARPhuMkY2o0ycAMQFr88cUP5MuEYnt2SPB/fA", - "9yOYpwGoawcAqFF77Gb+y170s4zQwAgGzDUqrnKrObvmG283Qk7qAcsCEWTjES0w+wfiqaz04ddSg+kc", - "ComxXFhRTS+JdVjO+zXPTgWwBSqHfWl/Yse4hQHrxHFLczNr0nBg59LcEM2HJFTUoRm0Yl3Wm91Js+gA", - "XnG7sWdCAt0hr3k6JqMw6722p1m3+xrcYhhFuVf5IzA9Zlt3qkruW5iq8SYssw6CTngQoATeloGljYCN", - "WjkfAyEGNiqD9LZ2Spf9vgaa0o5pRrqUkuCi2Mk5YNkG0xKOK1NX0fTohP3MJarBd3wB6GGltudJDC4A", - "xspHC25qYeoRixM6pkeQW9eq4lLM7Zx//xPDZ1DMUzu0ii484j/cIbGfT1/HLwlVyWyFtP5HJ+yyEpiA", - "Ilj0hM3FAhJg9druyM76ceHIE3bKol1mqpgzpSqPBIkHBMxnhc+/ZKU2EARhLXNrYofSzkRtfcVW64La", - "6eB6TQ0S+awSJZdVcJROojGBjrcGhRcUT4htQV58h2f6KAzZv7VG4d9cOiZxXbdzIIPMuFxIPyFQytfj", - "niDMV0pHvncbxHaqiJfFXnbBBEaZxyUPywfBvO1YAI45MSGs+f1roZb16ujkiwiiP1LRcKeTdSyqTrTp", - "YoEBPoxu0OlaLH4rxE8TM8oZnWUyrsi3IpHzWRqICAC6layy84O1OaPewYpKH9HsqSrWDu8OMzv8dIzw", - "mmw0VZAZN2JLodGxzptaOznoxPrhV8FrLk30KUR81hW7nsn8eqp0xTBexR2P17y6oSAjRLbC3GwIvfah", - "SdJE8UJWQEP4zgHsjBCx6Xe7TZLH09G0P8wvcYaoKxeEUHc/Xuqx/cTY3Mhy7HhyxoCcKqqQeb+leNdy", - "OYyyCwT1jS8dAcolbLHSpbTTxUlCOi+1qZeVuPrP1+zs/IxF77g0+DjrvCOmw6fiXagX98Ek1x8UHjQC", - "V1jUvoQpDQ+9SYkyTlPzQqbxsDbR1i5qTL3ogeT0ORhO9lLWFVR4SrDer7yydOgyCSYCyucKjLsOFLyl", - "sztzyfs4GtzLw4/poZFbw+knGG/Q0wiI7z1/PfnUOoAHqe5MhrEpzBzqoe9U75B/zbObBiiZCpnVPQDh", - "p4BVErC60R88h1ehyfbdCTtdz+Wy0Y1huqkzvY7uQRHMZgm+GXrv4tyqXEKy8aEqI+IDQC+hG4IIAcM7", - "rxaAx293BVlITxNg94tKrjGLA4xHjN5yRDtNOSM+8xlkI5nenGoofIplX0FRN0pHI1cV9XPGXc931/YW", - "i/txiirEect4zQu9nBF1wM7Vd4Zlz7CorypcNWx+ALnDg+NvQ6/cdvV73xXc7nYN+W7vCMEdSiSxA8+t", - "HRsJawfR0cqdnABmMivs+b8i4Zyw63CWuAYYc3/XM1WmFIjbGfmnykqM/1suWZRDZF4yJe5av8AbmDYF", - "J5ZruBc5VezaS9C1WzwBwvcazvyYcZJD9IBfioSEByEDJJPSRwm45TMLT5jRDKO10dRyC0i40HtYW1MV", - "LTgHGwTfBN9/IVVebCbsG0zkB7JTYiz1C3/NN1GIEXbgJdNwaI3uDXQHX3XDcg0Qc+DhT2WfbneqD2Fp", - "t/qYbwAIyassJwDEIJUMydrxydek3EpRjSnt1au5D/pmlkShudpBXYAbXfwdmpU4sLxXge10oO0ER6bD", - "2wiEwTM8bGlptmxkbs+7ye7u2itpV2vDCZqjPi6ENE3BNx4zKTVEL5m0yzurW+vBTZA7B7ul0JP24lVT", - "gJIL0MsRoMXIwWWMjmj92D07nonQ1biKAQjaO4H7dqnpwxyBXcdsehc7QKTo8nCV4tOwRzeAzyQ9iH6X", - "iiuA2txH2LFbRqgD0MmP4ortHfc38cGne4Fr/5rDpTqTauxx1XGPZP7MhODyhKAtqltwmPCcLtPsP2f0", - "yt1Kgi9+3sgCopc67iAR7oXtXoVvpS6xfJWHnlZchk6AinJtScMKtZAea6TUiT+/NbAHnEKhbe/AU3Rh", - "TAqM7LypgoXL0KnEcjG31q3bTRATJti2LhcIggLBi+BKBl5785L8Td2XAVMcHCaCQCCICQe+jIu4/7SS", - "xrr6TtyPhbJCmns8itAod/dyIzZkfAPiXnL9hO8MQX6nwjjADv7dV5HW7OcAMhiNGcKPrHiN+ZRAEFNW", - "GlAYc7ubAEW9Uw3pVrcoJNMg6DTArijW6k/70krHUBc9IE0njYIz+4RRbD6i+9hfxnmn09u3AS+e90Ms", - "Rp/OddYDd0ZIpw7FMQ+krTdiQz6oIKfGpZ7aNxq1c3SByW1mhFCzFMH0G610rZXMIKTU1HxdOuvODSwQ", - "xCAhnDUNqhzZf4aMdbDkZkNmORydWxCXrUmnPb3TvBK8dg9oVs/G5vgOtmAicW8Duw2kHvjk8rg5SPAY", - "GtSTR59eXxfo5Yb5BjTVTSwUmBHp4VZBGTmhKCs9L8R6sitzHyOXd6/wELURqaBK8HXs7KCc3LYmJMhM", - "On0hYLusMauTBtJKqgRkAwpJ2zhszBygRrI6BJj4OwkkP3d0VN3FSFQeqCB6RhxwbftMGG5qB0lEWibM", - "sXN3dPQPYaL1f+uDl5utJUy09l8eLuMlr4SqZ70q5xKeu32wpW72aL4dzGjD95t3+A66gbms9pGUSRNm", - "3O5uEabuioMFxptar4kAFCbLb0RIOpdKeP9bI1S2Y5ujJegK4pJvrb+Dtp5hK5A2g+7yQz2jfdhTa0xG", - "KeiIrYlqlKxnPaw0sfmDF01K1mOT6TKe/6HfCQkxwzL8QaFC1UEcPGTufjdq2/Bpqdiw63aNm5boRdLg", - "BTq9f0UGS3d37az+3kNFZNOmPeZfI2QLYCrArWCsnkDkElYmkEQbyMlRKVDHiFIGqsPyrNZNtgKfjgT0", - "Lzp5IFVUF9NhNxntbK2rHauYK3QolYSR0QJC316e2M/BDv6t00LCwd/DIRGuQKFpUCpt4213W4n7etYH", - "QUWeLYKGCke3+xq/BBLvBg52zgq1yX647v2z7NWgtdkMozc+eJ530k17y8V++W+NcNuZNHXPfoknrAH3", - "c1xW77FsHyU1VeWmOQzSqLs2Imn1crZntf6Hng88f/6q52ipJQ6CqDmSOjiYBiGU2dkbtb4RyssPbX2e", - "wK7kZnCAEiCTFjOPptG/WrEkMXtZGSGYSvYNXMxZFZqP8Qj8q56P2uVzGJgNK615jAahJ1lxakwxcQtQ", - "3ECKDyHmAr3RtWO4xYhnQJGHrZegrOL8DnK09WDaBNqRGZJEJve+eyZKna1aPJIhn8zOpxViqmvwSO/R", - "CVrVUjUUE5VQEK6vNFxukvdqBnFfykqYD+6yNEwUcild4Elmx7Qph3Z/oZP5q3EkXKubtn8UH66EyF00", - "SelJx3/V888M3ANB1XlLAsNG3bOXQAREz3ELfBjdvlfCDmlGJJJA3QGHnJQO+1XPk0uaWEu5MXKpEEO5", - "qyiC0TV0YB/u2xl6DPpVzx2KtD8JDRI9+EBgsvkw+cPDrj8MUbWDqUz27vSiGoPM0WQctOmXK25EiuzN", - "BQhBARfPFaY7vtKB3dFuRaTG3WWDyOFXf9dAerXHMU+BdKaHq+ZHIyoHOesUiXdTwx7FrnMxr2eY7yPV", - "8pqtBSe6Obc2cTo0XBk6Cv3IxrfnU4cSLuY1rWJPvo9RlhAl5RZoYNCOGZmdLoc4zrhNcD8Cn+8ZA5c7", - "MdA+fNeoKFFjb9DHAA/DR7BiSImMYiNh1A4QQbHrTnrC8EHl60cmse2ntsbeBZzaVPabS0CQ18soeYWs", - "dtwrxa4NhbcmVvR4fmubbzDTL2wXwrDGuFAcY5W4N3kx59useCkYN9ZSitqWsMvwC62o4lTO3s+goIAJ", - "D3EWrH5ncr0WueS1gPQsclYCrzo4TRMbeXQz2VQ+mogURMKKGXS88Eh88w1wwEn4jQYEs5rSHuKwU+/C", - "NP9Gg1cCt126+6DbIaBsR+2tdHwNYzfS4Kjx3mPM53SJfLWma5avujvs/g38He3SWxcE8Z6NAAIPZshN", - "76Vhf6FBA+K/NDehd+V0jmS19hY+EGlYEw/M63g8RoyrDSu1QZwPBDsvrYVcoznUnhAn4Nvb2BpbAy3d", - "z1H4MIW2SyWg1gU57M8wpLA/b+jHSsGZwO21u3enf8gG/xG2lAOPyP7S7Hc/JVNffxkye0mgC0r/0wsQ", - "xwOmjsjSzIxo0XbeDREYeUvQiU3EIz62gSKp+qEmY8fm6D26xPrN8YO5M6o0rR0KKDiB/AGJr7yrCNxf", - "cwOk7pWoGuf3DNcfgeSVWNSQqxeCAtDWAh9tMMJIqfTQSPQQvYTxjSvqqNLAAjP4gu3DXJGQQBoNZUeI", - "Pq430t3dZ1xteybZreTxvPVtTsGM3TXEDpapsw2EFBUfmOes7E4jwXaggFim79BMXw8dE/rILBfLiufW", - "wltAvMWAFrv2EXkPYAO4amz7/9bwiqsasST8fVbcaiLcfXBb0WI6vLF3ohIHNnUullwd2lA0hXYyubt2", - "UVEv5vBRbAroZiuB3hT6KIfdh2yBHroSL61ynZkDvM0ubaZ9u6g0K7RaAma1NPVgaXiok71reIWR3nK6", - "s4toX2kZrbjRdC2tUoRzhlXuuRYGQmEp2Y2nPHyDvCjuoHnAaGeFACoopzZakQH2x3U7BmcYpao3SXa1", - "xF8Ntqf8DvJbW0bNoBvSg681/MV4azkN/V4A79zZR/+NrgEf1CGmsTUij22Bw5oSsKMPuNXpbBhKb1+E", - "41rbppFAbTOsfd1Qv/6blVhuImFOK5TIz9XGzo4mZtRxE3U1bv9m0bvljbZNz2APR1dDHZNwh4ls2ozR", - "nQT8tM8V0ncYhaMyCcz1wJLqx+QTxVv3eQupG22eDpdNF7nt6koul4Ri7nN67XA1FeUyzui2qBUrTd1q", - "BUrHVX2UQOkrH/ba8TdhhG0uAH05kNBRlqesV4Fs1+9X9hMmwQEHR/6IW7QdOwJP4V17wrer0lv1rmZ4", - "Nx1N44vQqe6hqYPnVE8I7u1mia15CSa4bSiiPtWacnP9CPxqtKI447i5YdDzjeJrmc2shgQ+8IRYAQgU", - "oIhhdhejlwCtLBdEFIx5JhP2M4Zc+SZQzh1Bk5mpAq2ba/VZjTj9ngyNqsckY/qG/6Y0TADWST5ivrGM", - "V2KqxC0vGo6mIxG31BoaVq2lEmyl71gNuxvlwpgV0PHOycF0UNbvObbrPTUhFaYgFPjkUEh23KFpRiUZ", - "DYnjG4NY3DViHCh/D0FErXlHvidTdbHASJtRVIe1ZfydLVeINMfuZFHYfgeShQZYVNoDsX1EqutiOLjR", - "+7pwSZpbWEOnuLvJLPQCnNMUaX8latRZKKE4RrDE379/jYC8xlrY8AQY1/GekZgfgpvBFkdAYexS22zD", - "Hs3yJk4O+GC8t+u6LiauTsQCTAL12XK9QLJ1EWNef5w2IXL0wxp0KyqTzJ/4mmc3QuXjNVd8KXKXNhEF", - "20LIG+gNl0phJuztWsL8gU7FSwxY73ht4bHG0sccawS/VcWmL0b7t93byqVdD/0KOTXE7745Y39+8Zcv", - "2X9cvf2BvRHVUjCoprX/UCYHe4tSS4R5kHMHOMn5VFXCniHkrSg2L1FyK7HWt8C7jOXxxgTdeaBl3QJx", - "ofl8itDXLOoS5h7SJF17hxZebbi5maT5kCmt3PELJy956Bnjc93Uvr9+ZhH6Ach9KCuow1CMlJE1fzga", - "McAsRK3c2gkvRTUmvIjQ3tC4mEc9uQXGt2ZJbQ2X2kbE1YNbAi62CcClfUSydY1rCRbP9gFR3/VxNJ0G", - "NHWmwplB3xnH0V9vnQF3nMbk38UM8I92f8mWa4GXpHGTDsGk9qJFWcPD9g1Q0AmaeRf3GcO6fAhMQSWW", - "0tTA56mb2shcMM5MKTK5kFnw3Xw0YAKHgBYM9wGQWhGGUztLq1XXtiH9Sxj+NmDEYdnbF+t1E2dRMKuj", - "cog5dDlN/iQKLhnQ5WBs9KE+5EIZMfP1mC4KFALWF2a9hQT16r4UlQQe4YJggWZ41gF6UB+rwNmiEmbF", - "ABmN6KHHgIgRHRSsIvvuFFNlPWzIpIUtFLck/twgbv728P/o83seBPGXxIJwEwBLsLIbPnrO1uTpbyrR", - "D+uXjONGcNNxpZEM3Sc2h+wkt9Php+PhIiIpksvwxiGZqGcObsh1jUKXCm4iTKEL3P/ecCUXfSywURLq", - "zvxp06ztJE8OTGg+Le74hgLbX4Ib59bhxUqE0iq5rEbOFwAZ2f4qreAZkGFHo/epc2BvZbFBWFVST4O1", - "8c9i3noxoZdLmU5Owc+y08sLSEh5pCtmrfb3pz9dvP7r7PTyYvb9q78yoW7ZLa8eHyWjICFhk3idkiSr", - "mNHpMKzdEd3YQzq+EynwbTN7C6gJv8eVuXN3KjsCSRwk9unFONpq4F0nWD03TXu61ULK/sh9q7i1O1Qt", - "VL0/XMT1sOJ37Lv3b14zetPaCSVfCtOTkgPsY7ko61Vbu8+5kdk2wh9CiEN5wPCDYifsGwhnxIeITl5z", - "lVs1HmGMU9RPfsLOhSiBWzK8sCYEvbISK6EM3qLQqy1gBmyWq2uQgt861J5Rah33jim3Ai4YdfH08gLQ", - "R+iBPeGiXUqG9LvTb8GUP71w2TlwXhrRqR94SstKjENybhu9HYTOIcRXohDQH2YybXsBn37y5ErUTXny", - "5MlUPZuwK7lUrCkZr9mqrktz8vRpDY2bZHo9Vc8n7FtR+zUMMphzs5prXuVY3bnOzMmTJ/71XGdmEteR", - "PHSQ4wTM+jcBLuiATRFe9Y4hx5zDvcvIuYcm7Lzzi4m8Kp4Fkmik4HQj18KlKlYY4+UQUsBhgj5G9CK9", - "BNcgZ85RF3cIUW78Ac5BBH1m2PX9mAPQOB5ZrhkcyXngqsJLKtPMW3jvuBMkIt8ULzZ/T90Cn9ITDI9+", - "JCbLyYhNj9xCmh7Zv27E5k67P4TC/6/qdTFzFU+PHiNSPaGzifvaedEAxRHc8Jjg4GA6IVbCGiOFO3X1", - "EVKjXpIKSOn3qqTIb+Y0JB2K7Ps0jRDqXmlj3JEQlt8hMWmtzwCIwIoch+S8IKdwFEFI7YYzFGUAdT53", - "AGxCghOrrUo9YyKxAGxp1TdY4Cl4GZBuFaoJNwJIQeZ69T5KSsH0RFvRVNFbDkw0SlqHmpXLDfbExMxo", - "hhRTdGFqpoqiDTAVEwwh8KQQaiPwiJlJSydv9a+PUTVYaIPExnac0CMiyEgAZpqTS8P5j2fzzYRdufus", - "qfJwkk7pgOcV1iYtoREj8sineG5/Ssfj0VSRDDy1/x+xnNfCqpmn9h9PfRoxgkgVUt1MGC3dHFbbaKpo", - "V+VmttENeJUJRNMfpUZsXuj5iNmlOyLfKWFsWitgqiJlYrVZLiuR1cXG42SSMlPtAXJaCAzwqeKG1bIu", - "xIT6DPJE840zY00DqciXG0XbIrfCVGFCMkbXeqwMJMnJdTbG4EhTV01WEzAtUWEvNPFk41xItYQWV96X", - "b3vlM9xZYDDzXe3zbOO57hDlgwdPrxUopPORUwLw+uO0F51unPa6u2gbeW/Lp0+YolojVcnWbeVca7M3", - "1PBrKOQgZtNmpKjWiQfdsERb6peeFr7jaik+cTNhK7D23WzNQZv3BGg8QBNHVUv1cave3Vbq3UGbhvqI", - "9aUl7r4GhBWEPjZ9mMgmONJqjfYCgBEk3cP28Fzwcgb2w874JCwBceH4DpuL+k4IZVe9gYTLW4FfMhP2", - "nShKu8tJuHrEs8u9NeKsZYClHNGKFIYC2p0huZD3Ih+DUxRKispRzYX463YI9nGaJb/klcPF7B576JFT", - "m6BIy4LQvslY+2w6VdOp+gye2heAEsQ8Htbcnhjk3qHGsOPI60wjbg9WUO/hY9ArRniUvORVCm0WJIYO", - "m3B5ZE3utc55gVAi21IEoN87K0oPCGlj78ey1exFyYKno27hqINaF2fenm2vD6FOL2Iiem6MtAa5tSl1", - "AVCRiSuURmWDoJvpw9+48laD5SlCSvpUJ3lx7/j4huwdI+nDQUapt7YH65uoj53W2m2WniKWJh321joX", - "xa5x49Wy6bkqQAsUl55ehPr9KweAJbqm92WppMNgQuPSw1LqQi83BBf4EMdxgKyrqTJGAIGEIMDpeuwO", - "86uiUHU01j4L8dveUgNnY2Kke+au5Vum9EEPJHwrUPO50BoPbBydBJBfBdp5iDO5D66znQ7rlz2Nz04A", - "xaFeZXFfFhwNYCtZdyvkUiKEfA+xVleyJ1Nkh+v5nYOEdaxowJ+AVfMll8rUdP3hp3zNN4yyUqMu9zDg", - "JLA0Pz/+yy/tG8BktHECSBASG7fd1SQqe/3WFVcGy34tllJFaYht0TMblc0KRymy81bNc4+k96WtL4ZU", - "p47hah/P/MENVPSAu9hWQ7d1bfg8paLv2YXa5UdbrWp9cc8Qn0EM4K5Uz1oUwhj29uyMPXoLfktTy4yd", - "aYV+g2xj/11XunjMoqYRhYln0wMH6MprBETRDCFLjzJe1kj3RgENzgjxWuK/xqidxj9hgalyuZFsJXhu", - "LRfFCq1vmtI8HrnLeeOIR+BesNhA+qeyx1vIwgMci3oFPlZJCQV4d9jtALhKyOkDV1HgU8QkJojo4Llr", - "upkqTKRGjyNljBI2iFq6FvFau0i6W8nZ88sz8tX+oF0QB9xPQ7b3kycntgUY9q40m+K0T4+YUDnY8OTM", - "wfHF6A8zVZBqeKewfUZgxLvrPRxkIZIqmjipwEcaJq4HpFSk5EWAJ+xGbOIcDvh2l8wPvzbyxP2yCvBc", - "bhyJYNKNuqisxWLoPsQVYomxjvhzcLAPiPKLVsc7wfOLWqyTF/sPUT+UYPjggJivbScDP3LXUWG7aif5", - "xAeI+vBiOD91uZNGmHtN18fcsDfW4I4/MsH2JqOIOnrJS4Xv5EDdsyOxlDyrXIH+SaiXCNB7K0HR4ywP", - "nO0Y2b8vztrDBkVNcaDf7BIjp5kPrMalawCXwUrtVFGzc5FJQ+g8LosebTNZmygP8lYaTz7Mq0qKCnhv", - "Sl7VMpMlV3W4h5ZuJZB9AIg/O4ibpirARexoFtAOO2R9+8waFmDJFIIbkY/YvKmnigfonwgLNJBSdREl", - "KN255f2NA9JDu8MYRKHpcbew/z0PW322Vsgc8xR+SR6TP8LaJGlOhqo5vPo6sC2zR2UljDvwIqxSW34a", - "83jv2tsVWp8Q8MQ+T37XnOWi5rIwHrgdx6sl8P2LDm8SZr2Bo7R1B02PQk/7aWJ/YGUlcpkRJNcgHr/7", - "Eq4O9jfCe5iJ9fSDPpsMhjiPsTm5MTqTEC7g9jx/8hkGzugwD4YyFGDdGEkjF4loJxqiWRboUYCasY5/", - "2TokAUKp0z+zRvmUW0JsdACxnUdGGPgYqI1ZoQ1UHhmzrW9Waoag5B9yJPNDEJVKjmzUoQdsGJfR20mm", - "0u4dHfjNmGmWS1TzuShAqbY4DwC71gV/xUfkBD7EfibUGeCBpqKvqIcuCRCwm0KWRCXgXAl5aluUDN1z", - "dFs0OuIQRKAHCqj39BvS96NTcLxao7OuD/MDaYX7+NqdAdIp4HsgHwRd+HhZuuPGZ74MpIk6Qv0Qn4YJ", - "7nTwWTglaylkYG8X2POOv5N/fnlGwFaJDukM9Hbeg0uaxEcDak2E/6NMSoIg6PpzBgEepzG5oNFRj7AD", - "uz/ng3vs4Qj6I0peEaCSLm4Hh3EmDgEJti6r1/Gcw5EZgiGieY/VOmK80PZM4vT/rTvK+g2R49kYwlGA", - "gyN3CU1Z7Xtqz47OZoOCiUPa/g2p1fCjPntoV84i9MFWNRe2X/aMcVAeyU/xUZ8lx4A9AgdA9/BPR34K", - "UGHTo+PpEYDyGCOq2rcrl3g9DLlB7bFNBabsXLuuF3sW6hW/FXAe73cjGVeEltYQH5I/j3yw/yhgj3Ud", - "Sa127esmqvIzhJbs72vW1HqxIET1AR3FzJi4G7vvlLZbtHWqHJiB0Hsw3Y5+/UCXXU/g/lZXztEU/6C+", - "UFWeYPofSV+O4qVZ6QM4Gbe/YNXkFdWT9JdEwtX+zgCpSNI9fkhrQYfh+G7X/9vA+XktzY5VjnYuwu/l", - "IesngQECBVfA99tfypnP+4t8yLgEIvJ9Q5IakJYMbLMq0+azHa0Qzk/9e9CuXeVheUp9nf9Ibn9yoKRp", - "9iHxjECTDq45liwzQ47/wa95nqitAdV3SlQzpXMxfF9qqY9dsumhGGZeEZgdYSuxVHvFsUv0fSGPqDOg", - "5povRT7LRSFqsbN2LAid3V+MuNn2loNDyc5yn/jiqD3hqYukHjFNiGBbvFptTw5MalCTI5OeprTkbQtL", - "WIbD7Jl4X9i22w6UF+9E+niytVdm+vTlPg1oqz+HjvVeuB6qondY078MaI3dTD5aW1rbxaew+kOLe63+", - "aGMcuim7WmPD6vc5CrgPDu33P2Pbf8g8BpSWQV2DK7VeKYy7tgtXYOszH3EpHdShujE7BPTTC1ZvC62p", - "kQL5H0PiZAR74go7mAmpluyNVkt9/vXY1JtC+CsmM2G+agx/J8Co6OJ9qjg6C1zuacE3ohoxUSCzOaKH", - "83y81rlcbMagflnFM7gSnaonTy7Wpa5qruqTJ0/C9/ztjv3sD2/f+4iB3IfrhIx7hyCBHGzrsgJoiFJU", - "0E+ViamaNzWBu8v6M8NKbYzEAC+HoyEJH8KHVCCBK3iaMui8yplQpqkE2+imim6BfbDsVBGQB9wgSqAO", - "xgs6B/pEihfS8rNCnDx7/sLaEb4qOJnq8ujk6E9SZeAOrlf2r8mtFHdWICBi/ujkmT2JYkG8MvYFrcnx", - "kxR3cNFApY+eHz//cnz85/Hzf3//7PnJ8fHJ8fF/46nsUKfXI4+oQVGAI8zxUg52ppA3djSNqGqTzE+N", - "e7tNI2AQTTcMbq2RYptJxSKiquHnMysFb8vUKbgpbSsHJHQuKBeDZMWvJrlgsgb8Q/WZc5I9ggGgJcXw", - "E6mUhs6KR1UUjc3Opf623FY/ujxgODA9wolNKsjzktcrO/iYqOGT4P40aYyoJoqvBaa8/WlS86Wx/9YV", - "mx5FTx+3INmCCG9vSSilW95NSA4BCnw3LOwR4kxGqS1/apQR9eMJs11iuSiFyo2jaYyIGkirTdj1n8qm", - "KK49hgzeGYNQMFEIwlXikOMj/tbwArHBpMHsK2jsS1ZWGlOsGISWj0W+FC6Vi4KDNwTKjMRAZiVLNt+w", - "awwmv55siYAu3TreM/fvk+hrbS1OmECu25FrnxSG/d9bdQEL1f4Jw2j/gYrnT2VjVvj/wh5B/sTz/L2+", - "wiJruBv405rfb18HjI7ux/ZT41teQeCM/ebVq/dHI/vf2dsfZhc/XL16Z//+8Qf8/eKHs6PR0eWPV9/B", - "/16/PhodnZ6fz96/nWGBNxc/2P+e/tcROZKQAqkPlfvUhaDhLoIhcDBNgT4pgbBRppbCOUSZZwRkZ2pe", - "1cye+NijY/ZV9Pfj7eP9gJwRsuKG2kE9ejnnNWeP5IIVmuci3w53GN6S5AbwNTfiy889U3PMmBnfUM03", - "9TZoz/BPp1XRlSOgdGFx0Ty4dYnzYZUxpcXaqsxXtt7H8Vaxu6XdDeKwls+sAjC72w9FPqQDO8F08qX4", - "gD7UuubF7E7I5SoV39KsAU2J30EnGJYzdB0HCHu8Xk0YbBqQBopgN7Io2EIXhb4zbQBWrAAyEdroT7oh", - "rH7sBubUDO5HclPFlf3LwxdFUDhNsRvlcb+SyRtEKBDgNzJ74TJ+kkbWTPBshVICvHAabaDDc+icvoHE", - "zgGJMVaozn3h30ZHdvb7wBm/kUUtKpLyOaEmPhLrst6wryDIF158PByBY3iPWgtnOESHLc6kCgciSLeG", - "o8eHjO+a3yeAPF70kaQETj94CbeWRoEntqXOD99ZbEtchwZStrijV60doPdHbk+sZFxz+hoTaRu2APn6", - "iNoCAn9SzdkeGwo5+qTN+e2DNZTYcY+cRs17b9V+jJfXFf4tZpiDbsdadtrASzEhuqBL+xBj6koIh2Bj", - "6orXYrmZsPfRr4rfyiWEEq+kqOxPMuPFVHlTJkqcn28YoeLCHhbQYUgNg1M+FTEyF3w9u5P5vmUfMnPn", - "FVfZCmPLxX1Z6Eq0PxPnjT7fy1zUg8rxA197jELcoAicgxCa3QjSIKGXLxzccp3N3KhtkrBhSW33Rd+S", - "Rh1nTR8SWo/PWAlxWIfBioo30i2aNTuF8LwrKACZND36U6V1baZHbMwg2R5z7LSu6a1H+D9wAynNkFre", - "PLYvn+n1mo8pVVnkAYsZ9umLczNVAOOgKRYZA8Fj7FiO3yT3KALfvIMEt1texI9GUxVS4WLXGCQycxOd", - "Q0zH40Q93Burg7KTPHh6SOID8fe24IrHAa44hlXpBz1sYQx3QA1dRa4Q+TJ9BiaJydgrplASFfZkqjxM", - "CVC+45BeK3M9YtcN/HcN/8V/2v+srhEa5Dq/fjlVNMjGY+pc/+UYij1bvYB/QNE/5334wBFYsQOoaV0S", - "dlS1e0IuGTf3VjKiga3EQlT2xDHBwNbXQi3t0ny2TwL8aKeE4MdAUfCd0wXfVroppVo+DJwxJGl45YIB", - "kXYWEYo0Q8zlSrgQf3TVIORvIAqqWSG4qSHryh8wCHmz4NlNyMXwgtgE1v/6kKxcqN3V7Vo7giWOf87m", - "KNNoYTs0Y/CNEUoM6l4MFsb+UHXSgBSO5zy7aeWcHjVGOOoI982Zrmb4jRl949Nl+Ub0FGEj8C1JfjeS", - "7FZ8bTMvZObJY1COIYwyMHOs2gSrbZGIm+WbMnEjP+mmbB4e8V4JJLQMCgUOXCU2HBUwofWnU5C52ZHO", - "jY87XG1B/FHqQQ584H97ImSdzg745QPSoYk+FKeEltzOYPDt9ToAlDOVIf38+cfNkKbRd+I3OGU6Um5A", - "hXTGSyLzeCh8QJhdXO6RW5qzzFcfT7SzghyaWC7KQm8c5GkPq0pqicIXZ+Ej+9aDq8PBPZMCopaj3gnw", - "V2FlOtKh0ND9YngYcuuhkK3RPIL19LCNyVlZCKZvVXasjxaC22OCXxkc4TI9cI+JMsEkXyptapntmMGP", - "pZih1Y7l+MNwGOBbD8T4/RDN8i+jRXbKGNyrvLqvhcKIzb7QBjsbjUq5olIpNIAvtDM0o6clDtD5nT2H", - "bzVCLqlx4TxgVvz5F1+e5GLx+RdfJq3SSq/jpoRXn02OJ8fp9C88/EUETNA48xReGT+b2P//isp52+Wn", - "+z73LPW5zky2mtuqbMdlGo7dJTfmTlf9UVJK3M1KKtRumxJ3Vy+y6kV9+T+NuXtb5bEfyL+yF9Emrj/Z", - "ToP4pO1mrUXNc17zHZm2v23DBQdABN7UK+YqiRg/HUBfzip9N0b/Fh7UEDWg0s1yxf7Ufh08rpN2kENu", - "D8U13mgdCbgFqoXilLBxxLM1XFX0RJiG7ruxma24WfXdSzF3L+VKM1t6wl7dl5o40qUBalNmRNYA5nsl", - "zU2rzUf/8e03m/9+/pfmzXoymey91fptBBfdDtYoVPOrXqlci70z79/udjIlBT+dnu9FjrOHjp+0zAQ7", - "zWp5azt5DklPcGv+0+n5YzbncGJscqmZEUugBHAZnR0B4/czKuGJftKpmOTTocLhiP1obR5P2BX+bBz1", - "Zb3ihBHCYc8sZA1hNoumgA78vJKmpDM74Xx5ADxG7GIn7MXx8fHxpIe5U82MLOzxd0/DXQ4pFm43HBaC", - "bZzd1kW2ct0zE/YtLwE3s6q3uoNsNRP2nVyuEOOkEYZAYQWNwIgtBACd+wrjXu3oE7RjYJewze0eoZ/F", - "TVOt2VywG1HWE3ZFnXFNgr7k0mQcCet9A59/0dNAalzJ82S7LjlSX0A7eJ6HVFgAWQXcLM+EhnW1hmUy", - "MB/hJyCbuAJ7tc0Dsm1m4Ykb6SlYyTeF5jkDFzlwDyGKdSCpKQTRTBec7okw147QWs/ci5UwomaQ/g5Q", - "46wSuhQq7WC4FbM44WRIoie9dseLwE0z5L2MZysxg/bLtcgTL+96t+aFXs6cwseXZ5kuk7TlgfuEXqTI", - "GniL4Vv26AB2oDF25ZP/w4XMwvD/fPp6TEWs5BYuG9e8ZMRv4HnOALVd63oMl3lR0YFUPz0dNCueZKJ9", - "rZcAfS63OopdxPeYVKYWPGd68VE6jX4wA4f/A/iFRkfZSmQ3GNGP/YJk1aFy0335rpJ1LdQD3pfqVqha", - "wxkFjIzB8hfqUA94pxKZkOWu9h9UzUrWgxuhiwL335lb7hlXs5I3RsxUkoPUOQ1KpDKWa8EaBZyMK8Gw", - "EmJrB+85isVacAPnzbPLHzE1d6BkpNsHXOWJmKro61AEwm7zphA5OcrIDx3Ihu54UQCTF8ubeuMoBQ9v", - "XFkWG0Cxqc2M32qZJ/WO8rB+URsdN4U/bgqrqG1rz7/GsNKXwZKFMF5rpmaA0gBkcA7+K+O586kf1voH", - "LLruy4cuuvB+pstNUtRea3DxhKJWSW1Q4lC/AsAkwBGh79Nbiod1H2iEc7EQVcULk2ZCXFa6UTlb8+qG", - "GVE3pWH4RoCWcRffdt/VdpLHtmIngoCe/kDparXtwOEtYBSTA4z36jCeUsGhCxwixBBeSWv02NeRdCFM", - "g1B1ZZeKVLV56e6VDFOoFyCJyRzeSTu0s3mTL0U924D3OmXK2+GvRUnIc1Au4vQWsmKi4CUgKth+YX0P", - "bozZ7JILD49Y43ImyKRIMpCw12tFt48iG3f1wGbZgw/Y8T2rRi2FoRbVXBZo80c6OjHBUjHFlTYi0yo3", - "H9AuOzcHbIF9NVT67sF19I7MeysQ9rFdig77FobJniQAsJb4dbIwsYgtgvY0YmbtGsgHjtyDB4zYDWGR", - "J/vsMrZAccEGaAfBlMD0RNSIn1ImPmAiYZk/5N37XVsKLo4ivbXYTz6ko/e7tKz7ZlqBspUuchf48nEE", - "6n5WFlwd2H9H8foBYwB7sFkN+XC0XX/AB2EfHjrmUPjw7xDg4GGn0ej1MBMHvrhzMLcGkTZxZxThrbuG", - "KyKA9hoxf7YZMTonOC4x7zA4fHQqYbJG5DPyTqQcK5V2Dj5/W4VOKCZVrZ1eGkWNr0QpOPoy3PUW8jqA", - "uXx4I/vF5KwjHjCMcVOIKZSZjcpGLBy1EGDXH4uzvzXW8n/QED5MzQ22BHOR6VykNooh6BYhEhyS10u7", - "GvoYh9/xO+esQicDHYOseVySFKhlYHB38S5Odhx4MEnEwJFMN7FfHs/bX433d+OaiW12O6a/RFUaAOrw", - "E0Ob5xbig8724W27Y8jBCii8d4ArILy0lsY84FuRPuo5sRxYizkMhGVGkz/4vZXgZS9/Njq2dOUiDXBv", - "HiM5FAIoZisOioysfHKqVYK2cYTnrkbRYXW95iXSa5LuBQU31nfKylyzWBC5y5CV7X17szms6oOcrcHT", - "lYui5jOpTM2L1CHnwpVkVMalvNrzF4b3RYnPP5++tktqIe8ZVPwS3c0GoU8hdIRnK4gjC+77YUspNHnB", - "i8LOykNbXYl5I4saGkuBaStZxnpfKllLz2/4STohVVYJdEKRayvFFivwm1HhwGspQx+Ni8gpNkwoKxT5", - "iIG6upNGsL+LSh/cQHRWzMydrLNVaokge4nxRE0AMm/Fek9rwwHKe0yQ3QhQ5A5upz1izPBEnQLOpdsk", - "UqAAtg1orLjZQJsTbbT1GaarVl8oXiDy7lsJopoPbjfW1n9SpUOaOqyNhPOpMjFGIaVGP7B5qQyqnc3p", - "zn9fe17inaRfSErXDKCKIAWZN/VKVxT2elCz73gxq0QtsQGJ5tspc+GJyG8q5w0qB3wPNv/I1AM9Zj6J", - "DoDG0gm5t5Wx1CKStoNy8Fwx5KsIY42NizXaQleCfJi4TX78fhXyVjjLi8wdXs+CzTp0b4rreWgVdCk1", - "w0vFQw5vnVcPMKA6bx5kRaG/UdbrfovkzLkaSaNhaVhwerEA1g9/GYcWeKSoSmnVo3MwhLAavNBjbkdl", - "a14OVRXQZL8VRwZqutk/n74eQ5hyJXi2crGlTi2D9FGPnOEtTbSXwVUjut/dOxSLuuJmNbbvEVnxwOY7", - "J72XWLgF6Bt7regq/VZEyP3e0e9OE5Gv31aHd1/c3X6tiV+d7riRJ+CAa1Ld1KKa5fMZkAvgWe6OywPu", - "HL2GdQG4PQfK4TXQZMQb8IBaolPX8MYTzDP2Xhz61gNGy73q/BCHvZa3deFQ1dV9e/CLhH194Of8W4d/", - "zm5BD+1l9+1DXgSoVa7UzGcRmZm5kWU5WPw8oiAA8g2WB0pvcUFIB7614lU+9J1G+b7lH2NLtfW5eUaX", - "1gy12OBx9+E8h930pmOgqlrcvwJua1ENz6D9VutlIdhZoZucYSXs9CKQZCNlpGGP4P68rKQR42XFc/EY", - "YLx+NMKwUyTfgDopfIudVQIsLW7fPT0/o0C0pl7ZX+lSmb1z7DYXp29YpQvBru1/zVMuy4LXdigAdeia", - "MMNeYbgle2MbdfLkCVuKtVRy7Js7Pj5+5lmlR+zF8Z+fs1yuzWP7vs8QpH4KGq3onEdh9QY5eIE61UzY", - "11JxSD/LJbc7qCNZn6oo2GcuwCQsuJ1Iu49VGGNoqw1WH/bjXGfGNn9V16U5efo0s8M/WcJcTDK9fnoL", - "LRxz+XQpFEAg3Qr7V64z89T31jxdinpsmxpGwHRxx3K5xoj1oxM7HMGksiJmxpndbHjx7Gh0BFN9dHKU", - "GlQMZLPdxkje9Wa8zMox/YZPb2UuqqOTI2z80RbGWBZkYpbGfHHQU1yx0/MzFl5ABKaFNdQfWaNBZmLM", - "M4jbG7mDzd9FPrbCAoaNuEf6TVfq8YSdFvATmBvkXXL04/CxFZcJjpPREdwcvVXFJkLR8EMap3vD6G6n", - "BPnS7kYqLC5y7T7685f/PmLPvnjxJbQdpBZARRIT8Xiy005uJ5W257fdtNbKr8TSaYSgBE4vLzytclTT", - "ZyP2mWjszI7vhKmfffZ4ws64YrwwEF1qreZbydm3b99++/rV7Oz12x/PZ6/fnp2+v3j7gw/whNT3VrXJ", - "jBOSybhTPeK5PfAqSn/vVW2UDN+OCe/5xHb6Q7Qk/rFrfKkguzgfMFaX797+x6uz90yoW1lpBRbuLa9k", - "L71RWHv/iBmKanHfm5gFmeZ7QWBc/31uOm0s3aB23wA3Yb/0blHfoj7TH75HBc1Im1QKmmG4ziG1wkhh", - "oMa5ERvQOsP0widahJM+7IU+WnIXnK8STPA0aj7r0FGB9oSg9y6/55MvxouCA1Tc0KW3d8HFlX6UxfYR", - "Vkst1gCl11RJEig4HhtWcZXrtRIGjtBONLVij44nx+Pnk+PHrfPootAA/R0gL5Ks8IQjA6lS5ewmFW1W", - "jm+YscMIV6u84mtRt7jJY4hrXc7KBExIkxWiMYlqsPXP9rb+2c7WP1xZvBMVVzeH2LNW4EgYgty945BJ", - "AmvpQKsVTAmC94h0CaKeoTF3WQnonpG1sFbdVI3ZK/CxsHOKgdqwV2oplbAtOGHXS7D3nMZxHhkfMbUR", - "UJjsQV5KY23Ca1vvtxVXtbeWT5y53H2T52uprtkjH0Z43S1R4ZDgABr485qVolpLSEx4jF0DOxssVSPW", - "3BruY5yQMemD/1nwWpjam9yPR1slF9zU4+Pjzwdavi17d8zLcjxvZJGLCm1fajdBCJ+++ZC6OATcj8nF", - "1baanS28u9//D1jFV3jBSoslqhtS6V8yI4Bsqxb3kw1fF0k7AV+OFs+EfcOLwjBwfw63tdMbzt4pSGTh", - "04Lv33D2VvpxbL3fZXR/R+OxT5X3q/AupNcwHqafxbz14jb90v5lc5Uy6BCtMjl+p5eXry/wlDI7e/fq", - "/NUP7y9OX18lBw3ShSBsIRHlsLRjjs1HhFaMb+gxR9JG47LQc4CR3CFo7s2d8uAPXh9NpvcLX/JbpKDa", - "3eR2rGYIzLW7sw7ljSZ1UfDbFrhxq6Zf+r6vlkCW6mwJ1wz6l3s22jWjVA/DsulZTfBgjQ475rQ++TSy", - "ZfAnZ8bgha9928VepwbNbvzUXsSn8EJpmK6mivjhc3Yn5mDFICO+W93EMigNu0bFcM0gyLDOVh51a6p8", - "DFpXcLCOTMMNrRVX3LyvRN2UYC49m+yyl6w9HVc5Vc8n1lCzZxi7I8YDxcvyaejcVL2YkMXUPdzh1m+7", - "gaAfNZHlh4n9KOZKivx9dNRVbgkY60bhhGRbUnIn5m5a3dgio8IrHv3EVtwEaJJ2LaTAI64DSKcorX6t", - "JK/FVLlqxp0qwKkpkEFTKwo1AYBEgPHTpcdIBG9ttAX9RFuQYY/c9aZ57Izl/zqdnV5ezL5/9Vf759Wr", - "d5ev3sW/vD/96eL1X+Nfvn53+tOr+Ie/vv0x/vP1xQ/f/3gZ/7JbyY+SemyUVqUwq9agrpDrAvFc1rwE", - "VKyTfxzNK34retnO7cPW/INd0uuDueedwoVUN03ZV/41PO28YkRVghGafOUKnnZeqfmtLDa9CJ/wtPMK", - "WQ09ryTMgdHRRjd95f+qm44J4CyAzQ+AvhCsjy2bmpdylgQ0v3TrwyqWG7FBgc4qUUfweewS/g1ODNCJ", - "tK3YRaDoZhfWIqb4E2RY2rr6ADPFfQIbTFbi72pR9iwWO2in52cf0TBKdjVVvVA5BBDtmFlXBHa9Suai", - "yxCfqtehSNuzoFADmEFOzQ2in4c90oEmi/u64hlm8y6BZLlGlnChAohWklDbtWIll6sCcNY/qCG+GoJI", - "gRDZAc0ouFo2ScAoXBhW/lwZhE0lYFJ3WSAU3BEY+99F9VmbDkSo/VbwXpP3n7822lvCyM6BYmi19/pr", - "U3jcvWi5wXlL2mcLnvswyNwH2Pr/KqO861gRH18HnSgvo10DPe675DzyyXelvAH5bm7sf3PRkfImyXdj", - "+EK4M8o+/H8yie0rTgIQmAlvHw6HqAf10XMQGn7aSctEfKnXPkwlJ62Wa6Gb2qPGBGz6bXR6B5JM79jj", - "wFoWhaQcyv2IbzudFLEtrkL8RxdC3R2dEh74qqEdxpMTRXv+Uxo3XlkNDVi2E3aFBgfhBAG+orAnNFzV", - "GBq6rZX/ZXfWMDrpijFccpbrNZep26pzeMByoTaFNHWq0oGsEX0beSsIeCUg4yjs0m6DhsAOB8TR3iYn", - "O7frfT3jRaHvPl7XutZBqnfJrd9K2mE9HGIJENWSKzr5VLv7oE01SWvB1Y1vp6EQeb/aGf7lXCN+c50c", - "fZRNdGhPhuxEMM5YcDJod0lJhi007u4nLdlPi0LLfZdSD8FX0PbRjZhpshXjhsUuusED8wl2qyE7UlIB", - "Vru3oqF4ZdumyPaushIpT4+7ypiqqRqzJ0/EPX/y5IS9uudMiabixVN3rxG/fXp5gcXRDWDfwCP/JBe3", - "zrd25cuyR0ZaK6bC/N3H+C76A+y7ePYPvkh8x9oba/l34uZ7d/otvQe+EPsa+D2iV/DxRjf24V91M8n0", - "Oq4R4gVhcjGrjzpzp6ubRaHvDL6PzhBbBTo+ujVEw0AgFKDrfQgM1oKCYWs5xAmLkXbkfxb3PDhavPtk", - "RL4g9HR4381oV8DBz7LIM14hknGCU19rs5cK+WsoFOOxb33mjj6zn5PWl0wZUF03zUe84un14jhpcU6c", - "R+SWjJyATKhbdsurx4fZOnHN3tz58d3rVvpoJT+Sx39b6mFlu5+lsYvdQWTCWgDt6qXZLYqxrqRQtd8k", - "psrLd8rZfiWXijVlqwm2w+gUB//6t6L2owu0dTk3q7nmVZ72i/NSTjZYVcrnDXocEViv7PTj/J6W8nux", - "OW3QKybtGK0Ex8gQBFg9OqVLbEdO42wieNNO5dfcyMxVAaIF+5b9NRS3rYTCglei2i4NP3eL/wYm10In", - "SFI1MLkQNQCED3Da6uuVoCsR2ItENXF/nl5eEPip/T2nhM3rp/n86e2za7yux0Luif0Fn0lFrslOAYmP", - "30B6cC7RnmkVWRdUu8rh8h/jUAu5ENkmKzr1+QLGvgS7mazhNEt9IBaH08uLiBf85Oj2GS/KFX9GHL+K", - "l/Lo5OjF5HjygqCYYbqxvU+zFa+fEmSby5EpSaV1EW6FOr1oY8Jyj8uJ4ALOT3p1/j0lIme8KKbqeus7", - "10TIGqJ/I7Nvzo2w6xxDCD3r7EWOKbn1WdReVI7C1F/rfEPHQ3fW4CGo6OmvRAiBCm+fxr5wc0wBkp47", - "/be2OqZQQxe2B8P3/Pj4U7aDmMVAx4n7+qm4Faoem7oSfP0BFZ+tGnWD3dvKgY5nOYQosncC0f0w7kVX", - "TGlFDUGQE8wpGk2VFQkEvr6ye/irW0C4ta9sF2eP8LcTIHt+bGXgt9HR559iUJGYINHnC2Ihdzj+0IBn", - "v2MDTlupGVGeIOgObFwcAHcnIl6IQoocm/zid2zyO7HWdXCwzwkHiZADo9XtSngUcEzDxxZ//ju2GOLp", - "YNQWulE4ZM9+zyF7AyksbjzEfSZEbrrDBcABwPloG/jF77wOMEKNtk/kpMBmPP/nidZC1PbIymWBcv4F", - "ynnqG14xI+m6tcvOeMkzWW/wzT//jt34NsQhu7Nrz5yvxVpXG4J2RIMNeUD8PsxALUcboTeQobjf3a1S", - "793Sr8pChuyqWjOz5oU9asJ7hgBl/SFWV2wh70U+RiwLQoQP4f5T9T/+ByaDGfzjf7BvbHl25oo+4sUd", - "35jAdQDH0is44GJcyxjR8QH2vXbYRODiLHhpC/9oKMHgq+kRtGZ6ZH/+hhs8muaiFtVaKmlqmblmvP3h", - "h//yLRuzK9cj3wf8KoDkkXKSa1lwq57sC/iqtQr12MXu2rO1tcWvsf+zXFY43qIyT6+xTfEdCjQ2fFEu", - "KB7SyjGNFzsDwhmIdnkX0QtCtn0ODg4UDLtzPmdrqRoIDzoDPAN7OvAhxnNdr5zjBXDU7RxT59JmVaNu", - "3ov7+lMbVPChf7Y1RY3wptTWUn0PwwWzmUdw4cXmD1vkYbbIv8y+9bCtoqWBQXoSWtNnTUcaGJLW+jXw", - "aSG5YXrhzo9RNilm2Z5mmShr3CBI84cifhsxK14KckKgTe55n7aObVPlesqEuhWFLsWEfWNN8ZJXRoyB", - "saxwWVwjdp3zmv8v+cvEf/eaWFyg/FRR2iTRAEQ5uJzlwn4GElSQf2DCzuEnueZLQRm9oOLQ1TD2MFtT", - "BawUuSNtXXJbMcbGLZeVWPJaMET+y8elvBcF7ZaO1AKVa6F5DqwlaEJM1Y/vXgMgBSt1TRflEgkjwA9A", - "L4OF8ZJJVUglPPdag8F5smI8qxteeMYbux+mlKo7W/nMvU9+XIUv/bO1KzWiX7uG8fAJcH9o2T9OfP8f", - "OPG9Io0B4AUjUoM+JR0Jarf0GwP9Zuiw0B3cP46Gn+Bo2N7vMbw92ncf8bBvRzv2462dH5X+AT7V6CPu", - "QDdhsRHQInPEW5Cp4ob2+c8Mu+6YEez08mLk/NwrqzVaHuuOxbHDEAA6fLAGrpmsxZqZWhaF3RWxZdfB", - "QJgquOkasXlTA3cHRNfEZgM7yGqYqoFmA9tvNUzVwWYD2201TNUHmA3M5434G66FrqYqr3Q5loo52UAw", - "MTgNkx14df69SR7lQFz9HvuHydEyObz/+g8b4w8b4w8b4w8b41/WxtgyDtrmBUZM7nDuQv/GeSVvhU+D", - "ANwjO5SE5H164btV8qqGVENkaa1XYqoyrrQCENzSXzbjTR8k4xDEbCUyvVRADjCia10df3E0VXBTmeus", - "gfSzrODGyAXN14gyGru/AgGBp/GeKv96qBlcJBdWwB2GedRPb5iAyWS37zHu2HYXABBfeBHyfMZ3AuMz", - "p8paD4C6QBqEcWtTVYLn16O2lQCAcw2OacVyfaes3TBh78V9DUCgUxU1J9PKNGthmFaChY80SiYdsq9o", - "fj/N7v3Kt+uftHPHDdixa8fD5/g//3AU/LGJ/3E1/MfV8O+2N5MWYg6rPGxyZtTZtUxn22rdBThfZ/+e", - "7RzGdCeL96+Fthvw69dvAvonpDx9K9Zrzl48tqd2qwtZ5EaAc2J0Z7B1W+yiG1s3tu6asxL7rzqXDjnO", - "PL1mvGam5lXdlBMHhCC8gwFfug7n21ozyL8IjPftK+QrFxNk/74SNbuOI4KuMSsgExICmLeiih5dXb16", - "jAMQofRPFRoZCAk/YZCfD6FTIDtAMsrZdTu6DC56rslHMZmq0xZzMLTJ9srg18A1csL+1/nbH179cj1h", - "F4sYeczHOk3VXSXhahtufgOxI/vu/ftLfzzFATUjD4YgKu8lEWtZm6niil1DF05w9V2zRWVNnbuVNgIT", - "j8DjUhZcKgAmxXJsLSAHBdMkpyortDeVsFOeEIhdU3ewajdDF6psavYNeZ8IwqsnQBB9VDhE9GFDZNQ0", - "8nAq14WYqkdmY2qxHjHESLIry9Rc1Y/jcPEJS8hEN5aM1jY0+JwsM9oiyT0j4mOSHShQpHYTatYNhiYW", - "G8/LtfJONytJSEuu7zAj9NnxMXsjvx7Fqvl6ze9nziScWUWN2MIY/fl///f/uYOhv/aG4MSHlU6Q38/x", - "M3psbrh2K7WRtmn/93//n2df2q9C+oMPtkT9NWGnQIgSNciaz9gY7CfLpUEECKWVWJc1AeoOdHmRITxV", - "aZcXFet3ewUJQ6eMrCkMNEIucdRBc51vqEYC2EF/qCALnpyAqLcg8bflDvT+PkcNQCdd0HLeU2hlPYsh", - "8J99SRxfpajwFedOnKrW7H/x7DnMg65Ya0a4nyt20Cy/ZBwmb6r802zDjMBwmI7PdSGVrIVvQyZkAX5L", - "ED8YnZk/+V9PFUy9QU8ujYbIl2LC3vDCrlM3pMYlrX9+fDwiSo4IOLeKvanWNjHGXzx//uxFjy/1AhmH", - "vJRGhyCg9TBeqHBJgmiNmAGoNYgAwjVMytB98IvjF16a4BVKOeqLdPEBr2Sg/BFA/EcA8R+nxD9Oif9v", - "nRJLUY3pbi3p7vWRQ38cIf+ILo7Ouk6bxwdQe/Tc4412R9unwFzTf8B91yjD1k1Ry7IQHXW/3OqAYRzd", - "pWajslWllW4Mgy9MpuosnDOIRsC/5XK/wPGLVoi19sS9yJoabjcq3SxXaG/Sy0i5k7Pvf6I1Ur1kjfIZ", - "+bCg4Nbbw6xB/j5iRDBupsqXADk2wSjhYJQxI+dFvE9hfJ/tW80zq6ntGR41MzTGh+d5UwcOiM+Pj+Hg", - "SpABei0ANZAXBdzKG/jehJ0VEvbIdWPger20qgAMP1vqM/vAWuXFBhm0jLwVU3XtJPoatolrd6yUovj/", - "s/e3y23c2L44fCsozqmK7E1Ssp1MJkpNnS3LdkZnbEcjKclTe+gtgt0giVET6AHQkhmXd+1P5wJO7Y/P", - "1c2V/AtrLaDRZJOibL050ZfEanbjdWG9Ya31y22XlUVl2ZCIZdhlhkMSu5tyxfw2gobqBzlQfpTcBnsZ", - "dkRAKUHfe5+94o7OvcwE1MINc/Wjr4ywqOkOVGWRTp6+f79g1p00Ls0hhhzCL3HrE4rxmi3Tqpj32avK", - "VV7v9z8HeN6BguXGYHNUrpcjLTDgAcIs0dj3ZLWLDWGk5nPYs4SIOUVthFqQaKxAgEKVS7148fKG7NCB", - "WhHN0NCqc2lLDqYXOYUCtaWBnnDQZjonpRtiCGSt3HPFpMpFKVQO42jQb2hwoNbQ70rlGhbjljVs6PO+", - "qNk0mNVXG0vkEipiyJp/awNsCDYlngvkO1S+BrMCMET3XR+P691pu0229aUqvbeqwT1vcvroi5xpcGZw", - "xZ48/VODC9nfWjh9U+WA5PplgY+gggRSlygeyP38ILwus6xvkOQspNcL9DjxOcer9m5MyukyI6hYeZdF", - "B3c3XCr7fxoBQCPwL56Dk1LlzPkpZkaOhFnwZQcMMItFQiARiTj2kks9IGBR+tDfKq4c1PCAiuFesk35", - "uWDDnvzTkNlqPJbvQ94Q5R1hJ3uYZBVzgWKiFNsCVOieVJB3dejFOVfz9aNKc5qgswADQb29Rf1JG2zG", - "hF+XW4o/4fzwp9MZV3IsrOt7qh02BVXGFQJiFtyJHlROo+tgoJW6r62hf+M0eWPYZcNMFyNh3BBm+xKc", - "nHLMlA5zTd2jMLUIh0NzSy5d1l6AbNr+y0BI1H59v70QbYndeS0EIdbzGHIwkdYZSEo7CPv7w2v59uVR", - "/FIb8B/27FS7EKKBqSFLkRlxR5GoaVTH4p9Pj8U/V848nAGa+Mk3Xfbq9d7bnv/H872jEzwTr9/87QcG", - "B9b3mQi6WZIhyI7Qd4sd/yyt1OqSvnl9Qk7Mj/tHXfZCq8p12atCIwozwnX9uH+ECYEhbATCXq3jKqer", - "pT/8gZ3UBzdMvhQim64dQ3LaaSC/TKUtPTP4hZ8//VlkT7vsL9Xzl0cn6KKCJntOwx3MwkVbcsfG68rY", - "dJuG1x6Q/dembb2W1mFTndvQbbCrdUoNzYvqndx5rMZz/pA2d51y/paGcTIVseR6o3wLArXVKW7SW4nc", - "VVRHl69c8RCGDkFiKg9o7J6ZzPsL6sjrtqvpVOXwX632cBBLpxt84B10pYLOlGUeV8gzsZKVQbTmoXz/", - "9BhCCj7vzj4yz+u8sO+xx49h9I8f77JDb117zouV3rw8zrenXOUEqcqQA/pvYKr+mxetDBrN7Dr+j+nK", - "lZVjW3D1XzrbBbveUmGzesV8k2+84dtz3J4BrnwQOmwLFjnjsFmg5f38tz1qoV5k38LP0la8qOUXV/YC", - "i/PBuBR3leFFLxZeLo2elY7KoL3RWuVG8BlMD71JiAyOo6k/wx0Jt7cqWOZAO+BtCYVjEyhT9MYEAebn", - "eBg6H6hjPYs6BDmvGKwDDRA2JpdjOINpKCV8ZHfDjsLusP0fj174OQwH1c7Os8yeZtrkvfOn8CfdutN+", - "sJIbC4K1/vyFzn7+216jgVxn5//k+H14FhYZn/b7ffxhe/GX8D7uRTqGc9ytv+0tEgMjuqT+f9w/Sj/D", - "kIhEG1r4eB8JJWlgf+8QMAeSRii+umYCbeS0RC9hZnT8h794CnDzEnD6osoikcv972ELXa0iQWrwhUAV", - "BdkkDLI/jOGyyd0r3dWj20yMxyIDfT6PQRM9CAOgq2XPOeiauxRmoELcbH2rWymJlRi4Y4Xg1oEHF56W", - "ntQvNMtFVkD4MXJGxDeJXQ9U6AvDkBlV0g4IhoknG1xWMB8MasCbd3xCwFvg4/XHrId2JL649fQbnMRo", - "zmqwtDoyI1kfGT14CzEXV4ze6LO92OaUl6VQdqNwYvASQ4xQeNRtiXuofYbtl+8vGjEQA/W5QRCNCJiB", - "uqkgCJbEQAzUFYMgThBQM8SNsInRF5Zxp2fobB8oLKa3QOn11tNS1sNtj6tAmrIDtRhX8T2brY6rQOOk", - "NaiCYUwFFA4MQRU4HfBJN/PXKEodhH6dTw0F17qUSle7Oho+4cTZgQQ1DFkFVKOuhkeGG0NPefRCl2qt", - "KCsGqsbw3Ts8sH12ZU84xnuAH7w7UKBapqH7ybpgCBHtE4wAH8gFppYegoHCi5+YVICA4TbZHG2adz1+", - "JKGN4GlvM8C89Qr89MYT+31Pd+3bxjGstv4OAonzhyD9h/CLhyD9hyD9WwzS98xp2eb2Bt92uwfwUdOm", - "96JwTdyC6NkMEPpKI3pG+IYE9Rf07lhDzIhCnHOVQWlTTph2aP2rNmf5chE1vLxnxGxl8IfjrbdgGf48", - "q6zbRcA+ylCgnh3ZiFQsmKaPtYSPYOzg9ndiVgLCKkDzHQuVt8wOSypDJMaQZjqMt++5sHKi2JkQpU3q", - "4lrHnSiEtSjji8LrXRkFJDjNssp6JexXr575vjDLYSKzZfdGXXttMwdH44Zh32hre6g0mOS2AAr/Rmmf", - "RIOwGHyMc/dbbuAi4JL7h1BUf18X4Pb1trwuoSZ9HS6w+q5jzpbvLzyRxBsM8EzjiC37Z7wYCgkh1Hz8", - "oa+Veo/XHnuV0zMewktKADBIm4h3SzK5HGsJr6CADJh4D4MpjgQv/Iz/diFUz7p5IVibdhmgWLw1MMb4", - "i2bdKvJ+GOO1PkDpotMbrOuep5FzkSfpsUApr8BOQuqmwoF1r7DucA7SVNQY3YHk2LMyFwM1LORoO346", - "ZCXPzvw4LqYym7IpV3kBik0gVlIN/ZoCsaRL1a4n+qbJTXPzqqLv7O6VRRzFanWRliOcoged8Xp0xjvW", - "wH5LkQlIw7WAn9eifVl5OK153yZ6RMIpo4BeqUKAtAE9oiXyjS9WB6K173kzNjrDAt4vSPLUNlcJW+0P", - "VExer8MpvTAbxVL5cgbmuhNeBFDuveCmEYGf5P94ZQAkSqMyP+lBTeGxdMlfM3NOGU5Blhdc5V4GrOif", - "F3C1w0ylQlf/ISe0FqURpdGe04RCABMjQfUpOS5Kf6CukgUVMlViTYHomQMX/nLlodYUpFa/3UpZ8iaS", - "z61Klbrb+y9fXsRjdT8lzKLXCXb+wVXx4Kr48ooC3Sa9vmkzMaJwoVo0v9Klf5AXgEzilz+GiYuczYX7", - "rfpCQG9pVzFWKjEQ1LVOdYEXMPoJQuqTFJGFkLGtlpgw4bL+o8838BuhZwvdQngkbTmYwN2Qy9GTSjp6", - "VN8b4EtsLAsBN/f7CwUUNgth+8kKts+tsOHu/G/hizolB8DMQpxrffcL88Pr7B5QxHvHSi6NjQ2Z+apW", - "CHnKa4Xebl509cAdNDe8nBrut8h/S1sYSlcncQAX2uQ2xHJwZYvYX/iTvhoJdyGEigCTtlVDgW5uo5o8", - "dXX3yggN45KK8ki+DqA+H2zdB1v3Xtm6CW9oY+qpsKgjYFfLizq01tI1r9ON1pcCbSFwpSWUVhuKpv28", - "6Ldm2O7KELhSfkIAHI3aM8xYiJeeRZkI4riQnm8WgDCHftkavD8XDlFwKSItrIBv9RXPxEjrM98uPmZP", - "+zux7ZdqUkg77Y11VlmRUwO4agufU2By+NKKYtyzVSnMucQvYYn3YMegzgsEoMCfiGU91VXhzVXwVvzx", - "a/Kv5/3goEbQ0Rl3lmFyGdMK43h8l7sQL733MwTy6dlMQERAjxl+wQ7338DI3xw+A/yU13v78OfXe9t7", - "/p8UQw+o7v6v4XDoD9lAfRgoxgYdvO7v7LIBIt/J7Qvcg56Taj7odPE1oEZ87aez4ocj2e/3B52B+ggt", - "+oZ/8dI/7stUKrd7zb3Rj6EP/F2odBhLcrU+UbAdNy1b6+7uWrymI1ktYZFG61P+4FB+ELL3TMjWhLwg", - "EreaOSLhbhoxR7dnYl2CG6RaeF7roJ5YSH6nmLpinsbFixxKa/VbEnfdPn7wkwX01c861U3k4Jlw3IsO", - "+C7PJUY0H6bvfOx2VFWAaEW+0oJtXApD4V3QaoTqX7fhh/GbNgx/vxQIbfuhI1CsdHY7/9BTlWuxCarw", - "Mv0c1wXi4gJf98lfSbw/KU4IvdefPXL5icECPSy8WIMMA/h0Ag3893cfu028YXySAgL//d3Hd80kURco", - "uoWeAeJ7Yju7f+8A+b5rHB9bwXZdniVaKfnPSrDwPgvXzqBK+n4woGGf25FUrKbHLjP6oof4/V26UdBV", - "6bVccDhKYfutyVt+rsdhdJ955jY6D0mHywdic2r+DREWpvtUbhq3PSGmI12IFcS0/YH+9XG73vzLSczo", - "C4bvRtJguTQiA0btHCL9waUfkRl1E0NMONRK9EYREFk7YdEeH+mLVzSw26Cu2N1L5cz8gcBqAqu3HQU0", - "Vw2aa5AcrWHnnZd53PCZQMpageBfv7JNu37I3fQwPO18fLc5+W5/gMvaj0i/hUB/8LID+1wgHTZped5O", - "yXAJXIoME/oCOUN8ue+s3+I/9D0skvAyBX/dMrh6SAZaaTMCvr550krGcWPK7B3TNe5SSgQthO1Nf477", - "vJLG1/JLKLx6Q3T2g3CXE9n17dcid7yK8vhAstelRH4+vV4HT+4ueUs5FjKbCbalDfvq8Vc4vKLA0dhH", - "nn6lf7XkbtrpdtBw6YSxNt0y3WRDassG6gnZFsPmXbdTVm0YIsJBFKYRZcGzFUwfK7Z4paSLKgl4bDWU", - "BmvqMMtH8HjFEfw0t9Zq4/JSu43yzDFWO9YjxYl6PT7erHVu0wN2OctITq0V7kY9XmvLLJCLMNSml+T7", - "oqH5BX30G2Mmx9fATFLNDOzLDfw8HK8i9BiYA1qlUlHROwvV9Y5iwHU+axiq7cbCT/Y6DYSm7+ezXSyf", - "bkPcicvlOgOhVnb6SpuRzHOhWC8Jrl/Y69+knR4ofr3DB17Z/uD/95bP1pszL+A5AlOC7pit8JDii+3e", - "0RZLxL/IsNP83rDlcBoxiBWl2KNb0zBhSX6ruiWSB+Mr/JHdq7jvL6XFHwQw7edzT943abPA6K/Ecx8I", - "+zdoNFXIzIBC22j7aibRT8SXF/1U3RWRJIiN6Fm0Ehc4lohvNAF0w7j93tovubUX2uQArUtUoTAC1PEz", - "TOHMRA4FHVaA50Yuf/3X63UHjVv1FoLKoKKyVsnCX2Z5PLnxg1+P7B6JtWBtADB0zQy2AyWwmbQz7rIp", - "sYTvbn6U+1qNC5k1GFRMS4EsU/tbM4gIwrQ+pVdT0rZ5KXtnYn75HYrXAvcOD5h/mekLVYf+1+4/kJvs", - "WGRGEFS4EgihBUUz8xX3cTDhW7qKg76+QAPnQaBegyUT6Rf1vYXjQrRx26KVBlXjJywcJ6w9ILjB7Hg4", - "XKhxEQwC5A2CMwKz5BDnZ4WQpUnepJjFLtYI2jDjO5W1OMpfpJsiv7p0oPdV9D66Aw8IjYNoCGvChTCh", - "kLgEi6bNVzb98cE8uH6pT1Taxsk2Ev3bH87E/CBf67A5FGbGFUa25eS8aTIrGkWfHVcj6+lCuTo1GIPP", - "gUH58wSIHyNBNcHa9AI07BNmdZnnJxzV1c6fW6C5MIjfurtFraO5a5Ke3Us//Kun2nVhF0sUH0wT8Jhf", - "l4hvu8L7qcx5OCLRHtrIyYRf+u4Ow2BvRlZjR6GTNdL6rbio57C5pL4+oidN/A3C2LYWNQnDq8r8Xglq", - "la4dAL4mXrMHl9m1MickaLT5y/rsXMUYXgh2vjwYKx7m+ktmqkJQjikOJqkvQmVKo9fMG6GrQrHqKOpX", - "Rs/IObbAtZZLuEPTVLM39hdMi2SUTqMAhnCtGOUAN/F1mENo4EqRDqfx7nXxlrFtvKH68rWO98T3vG7M", - "a6//00YgSuNy7eMw2f6VEXB3w4JgXGpCQRY1/dRWRLpq94g9MW3YkS5E8iTQBfqfI3X+JkP8EtqPkfAJ", - "O0tyLDa45+JF0TDRrnLfdZgwxdtw0a3LHnm4E3u4EzMpJa8+ETfrwtvL8+C/awqpS0/VXp7XIz3RN3jp", - "lR6kFp25Hna4BffDz/Nb9cJtoNvX4+RQEv2e3389nPrrPfV7eb5wxNZJwrXqvdGF2FCxx1j3ZtIPm4nZ", - "SBg7lWWq36/S373usqnmftToJ0QyOk3a5Ep9V28chexf3XVCcQB84Dku4XJE8ibZHoW4t1qun+YtHzxt", - "kEx+8yknwaxdOH+UrndJSolUU2GkEzk2FNM167zP+k5udcDsEZzg69JCF8j/c5TNBy5/DdezoNwZ2uIl", - "8roVhe5Svr+GSPfy3A/2RnU638GetXKiZr6t1tyEFaKEw2f3TLuD0d5Pve4ORMlvWYeDI7WsvbUke7fo", - "bZunevOiaE33XhFd0SppvtgE7gdKvZm88WWi/fTUxFWCadMTcJVs8fY83vajcHlu+JeZFF7r6L/X5HDc", - "3pvJCb86Lf0gHhK/f19k2ZYAvjFNXlcsyReZ+P2JZ44wOFGrHel8HjBdl/KtW3PDH5LCfw9J4Q+64k3k", - "om/I17yyl4+8qscnMCWg9t6okkUuzOryvj9ZYRlX7PXrN1B8JZYJJ1zqunwqFSqXWPBeKryaqoHd01PW", - "R4C7gGKb09kjBJrKiryu/UJJ71AknSit72n81In37pR6jef1FFtCYLx+K8BZZYWXwGNtoEjtc78EIV4h", - "llinVLi4FFA1PZRoVxNGTLf+CPYCduAry5BeAJdtAiXsmMGbcV74+Rzt/cBKWYpCqtbK6jDX57g10MAN", - "+XbSfu6o+GtzCLYqWjk0Vsj3xHpHbG4R04VYHQTYBZLXJl4KAB3cHtMj5eLuuF5aRTWsjecYmddvEPXi", - "0cbVVYHiT8Ss1IYbWcx/UrEe97VzVKA8z6dqlIU5MoBF/pbwVXjrNB5Z28Ze43m/jLWyF6/2CNSQl6XR", - "PJviNRxG0tTKx+5AZQU3cjxn//q//49ZUYjMnVrnGegEnxkxlkoQC/QPEBEbimg/fvxXMWevhJ+WsLuP", - "H2PtcIBZ6YVWHj/eZcdixj3L6rLnb55+02XOCKj4w8tpl4Uar1D7ZzofGZmnGBaHxNR8O/tTLlXkj5DJ", - "C6iUQnlFk4jENx4WH8q0+9MD629pGMwAV7BU63wfloDoGOuwo9aHMCg9gF33zBgXIZeWz0ZyUsH72MSR", - "4FYr3xEMEREz0OS0TpQW0ocVyysCG6ONxEU8xpWScQmHYaDDXfazyJw2zMqZ9MN08zA1zEaoYS3hw9Hs", - "6TfDXfaqKooeZVrBy7BUfu0BmFSqCbwdVn64y45roYrAl7TI8J5f0eEuO3BAnecCl1jxcznB1CdoHk+A", - "/BUfbR3yiThQuXiPEG2w1kPY8qFfHIQJsVNZEqU6w8+FsX5RemyIdDDcZft6NpJKMBsXCXs7OnqFVobh", - "6ozW8fgle3nu2ebJvEwW04nyFAhB+AYDPcHGsJGYcMW2rBDs+PjlsRPlMb5J8vZR3URp9MQIa30b9E8Y", - "ilSglIiy0Ux8Zakdz6AgvWNpMGOppJ2KvNHQfni90VJWcGsj0Q53UZdhzcdE59jcfuOnkwDFA3+hlMSW", - "TSBlpArBZ4jgT/QNVJVNK3XGttAWxM/Guij0RVUOa5yZnOHDXlXWUDiNj6bSebpSuTyXeZWADTVGDlP7", - "i6TxOa2L05nOPUWeaF1EKFX/jFhYsoT+lTc6J/rzh264y16e86IiPB5/FjOL7/vn6VrkWgmg1qBohb3D", - "1+NzEC9LHwaDlbJgHZ6dsfScZeSFqsh7wNqZUOei0CXGDI+0m6LJ6I1hT9SZVraaCcT2GYJE9JMAyagz", - "qG1cTxgeP2rF1klHe1NXewtLcicKYNu+YIEo8d5ti3NA7wXSbra56MjwCkjv2G8QsBXPy/1HfXZMS32O", - "m1fN/MYNlNdMoHUICMeKz24qpGGZNjhbsAponAjRuOhXaYH9CtSHxGIikMGtK6qLA0nQCb6+XAl7q92r", - "BQi79R/AwT/k80Lz/ETr19xMcNZPn2748U+K8DpR0/Offrfhp0fciddyJp24Y+U3AZrbYNjPef4Dd+IC", - "y9R/vnLs2/h6w76p4xM5E7pyN3AzEOgPRsxqSzgRG+EVLPOSyIdNle0Rz86qcrWKXee945uhnB66XL1O", - "Urs08VmfvQQNnAx5+ozKSVhwGKC1FfQxtoWU0mXSK1DCdpmdcpNHCEUeoX72ioJ+ww/95ni1Q+S4Ar9a", - "h8A8e0xp1UPjkkZQVqPC6xt+LllRWSdMrxDnomABbB6z8vnYeVo8F2Y+UHW2NMwnzsZ65bZ23MYWSAai", - "1Mv0bCbdQBH6sMoBdNHkthas4n0JAnw7qknUUaYr5Wyf7bFh6ZV6XgwHShs2RFtwyLhzfnpUUgAStnv4", - "aS75RGnrvJwfVY7lWlgwbWkFGPfzsk7DBBIwywi6z/bCCuGyKl1rGrTx1O1ADb/e2RnCa7py7MJIMEAi", - "ufixj3nmwJH0HJ8dvMBCI3I2q8j7fSQqAhlmBy8QwTjUgJlCKfewe7m34XBocdHDEgy/3vluCOtcCB7u", - "UqGEDEDRY+eVyqZcTUTeD+aINnwi2GuNzC0o0v7vAhEPoVjkLhv6P3a3t7dL7qbbTtPZAXzFvRn/VSt2", - "/GyXDe2z3e3tUZWdCdej6joL7/t+aS2iJQLdApzRh4KG8nF7oP71P///f/3Pf//rf/6bfcDPT2X+sRfo", - "NxyhvufIjLGtxbV55Fv4H2qhxmXs2UyXkdbCy5aqQoEEsqvAlXDgN1WNAsePfdyRPrUwhtU4SrSF4STG", - "I9xnB8pzRRcy5KAFbw9xV3mDalgbRQMlLQNPXOKW67KLqSxEffRZevKNKLVxyYkfKP9TZYQFfYz5oziZ", - "UufobZPOimLMLrhlpCIs3O33B+pHz/2SoTU4ZsIwFmnMK3a1crZecD7nddpyLDC1/pNQICpRS9Z/ENQK", - "1GhfXgmXCLf0DR2rG/WbEfE0ZWmTyyZCnBb9dMYVnwAub4sYXx1q9Vr6462VCKYYKz3T0+MFUUjtMKkI", - "jj7ucu9MzL0EyoVh3C1cZgaO1Y9+oCjc+cjLBeFVAmx8oFAXwOoSIsw+1JvqMidnwjo+KxEhhm49zoWB", - "wrzskFvg7F6IUTWdoRLv3WlWGavNsObwDS+g70ohbOxEtMHF+yV6Tst4Seh/kBlh2r4jcj2BSYut9G9a", - "jHS6yc01vD6b4wfbHBatRyPZXpGLEMa/9nL80pTgt+AzGYr3eOxOpR561U0hRCOrq2LB4nihHguLkEyO", - "5LNinHVrnzfSN/y9nFWzSOWRgKSCswGksWqtAMS727Cfx9yb2rtPdna6nRm2DX/5P6WiP6PNK5UTE7ih", - "f9+b6J5/2rNnsuxpcsD2QFEUprM75oUVLRPY18pJRb4cpPd6ElTaDYs2AqTSmungx435zPj710JNPLN6", - "8vRPMIX4tz8Qzu9vZ7fzn3/f6/0H7/260/vuXf3P/mnv3eP/tWThbz7Zdzco0vFcQ5zoGnn+OpQLj+Cl", - "ga1+mnj7dGl1IyWhl2Z1BenisulqG/FQGIA5h9eYVFYYZ7uh5hBy8QiGbhnPjLYWbxvKIggALxYYZ94K", - "8EaJ0zOZ4VccTj6o697+qw3Z0FIiQ71ZgWaXl6LYCvfKFZoXufRkOaqctyOe9sopt+F9tvX0cP9Rn72U", - "gMfAmz2BquSFkzbeshQs12Q//ORb4FbYx4/RtoWJQgQl2noBCJxGuIXrQywnrFIAOAcwCIR25VI5LlW4", - "8vF6INyTTuBahKYe1q7H9nDF6A4H9hkrq7Rdj8Od1XO/Xb8Alv3NKPN1L1fS5J/cyABWH/t9v5K9YN97", - "Eo6LlajI/kA/Rb/UbY4NaTMXmbQETUqOh+/ZubRyJAtPDdowMBUyWXLwCwG2M8SQWSehkAD4Yj9ZT7+q", - "v/MaK8ee1DwAV2MteHz9stc84WprJDJeWcG4F405gCF2mdOlLvRkzsZCZaK7uILeVBPggPKmR8Mnun4V", - "jv2HhbA2GckrtM4SIyRZGHDSlwWXC0tyqaMcakxqbXKpuNMGbLtKJVxejLXxs86aNPQ96MRIG3wsbqDO", - "3crjlIocbxysckqSOLo0cSRYolPBCzdlZVFZcAz2lM5FlxlvBXQhNFNmPMGbTHZfG5ZzOx1pbnJ2LsWF", - "bYd8pRHdvNvhhIbWyg1ovnH4wbppi9O/5ZK3rJfCjeJ1LPH5+6QKATzqwjJurgjVhshqU/ulqmZwIWzR", - "gZKxYBklVpGNzm1BqO/SMk+1zFvnZqCcLskkHyZfDXeZDMjMrDT6XOZQFuhCjIIBWj8dqNjxwY8xfo/0", - "sf0X++FgoKCDsjwh3k8wMm5q75UMQ+4FsFffDD/XMrfMFvoi9gxXEL4lAIcnu/7PCKPfRU/A8iR8+xiE", - "kbMCLo61YdLZQcLRCIn/YiqMIFOHPhbvS20j5I/X9PYOD/psv165gQKn8pjLAt0CcD0J+iL608IVAjrp", - "2KADt1GDjtdnneA502P0swW/xcUUk+qpdi/bC5/qMRt06u0ddNhMcEU+i8QwxuLm4KL3vFtpBzPvlUaP", - "0CZtVgde4bNIJnmZ38LLT96zwr/kYKHR4EkGdSZVDlVFaN8GCuKM2KAT96ybWPrdLM8GnUd99gLpBQuS", - "FMVAQUN9BrGjITGLLNRIU4OOf2vQwSAcnGCbsQr3y50rmfqrpirel1xBGZo+O65K2vlzXlTC7rIhktiQ", - "OT1QsBkYBFXTK7wQqAwd5sGv66deb2By+pID3GevPXFfaHPmKV6XrufNHpWT4dMbF3Iydaz0PACvRVcv", - "C23R1RbmWEBm8aDjle9BB4q0zUvyqTE71cbRfWzPytzbNtlUkOeG7sJwaRpL+brxAGr5kTVmhZG8AAnh", - "tNfAzoVycA4CSrURYyPslPFZWcSIHaKceHc11iaD4rmeR+Lhgvuw6Gc0otB8DQ1RL2sX6yZdEMkxXa/w", - "R7GQSosHQf9JPo9EyLadxk8S/Nsf6j9OZf5xOzKH7Q+RO39c7S05UOf6DFMCarZSW5xuauLtTbwOqHvs", - "D1QErRvWPT8ewtVrWoKuH4PtaYvghs/LuIEaYmjPLvNMYEguxIZI/cqylRFB3yOHR+DKuKskS8kZyb1g", - "G/vh5Zi11CK9cCUOwiT2Uy/vghxrSetq7MLn+YXjijqwqaIwzHjJycImydAfVDs7z7I4D/hTDFdkntUp", - "imuzz1Q18yQIwa2dbifkdMBHhquzTrcDIYjg73eGZxjOdhFdNxyKpnqDMzNyJDrvVpQJus68MJxFS2bY", - "9V7UXqXzq0a8wRtenavPoUA6v5gKlVTKJ72lPqR0R97fJKDtYFnXbYlpa49BS2hRm1ZMk0/l0wQ+sGqo", - "i8zEKyuUmgG+koWDEsV0LrKCk3YeukuOURLctcEi4dU1XntvfIMbWwIHjBTK7fOSZ9D5NYsY5F9QWh2v", - "2/yAuJPeVqlJqnF5taG4WZ/2cFQpJjCmVxvrzUdaNAqLCsE4mLuA4TghaeynmLrFnEBTqW6qG/7tH2eI", - "F07JA7Dr6M4RMZyYXOB1pBqFFz9+zLZo98HhovJTZyo37RtRiHOu3Kk3GjEgivQa/+QRONCN8PZjF+6x", - "wGHVZSrPJl02M6bLZrzETl+/ftPjtvePKp+Itn7xB1QAQsPQeSa6nqbcdFwVSljbjYEb4a8p2TngH3Pz", - "LpuKokzeNkZkjv6AOGetTv9ZcU/gLVKOoq9vysmOQdx3EihTx4+38b0k6rwObrvT1LOacBez0GIpusCy", - "MPvx0f3JA0Nyjolg150W67jKeaGVSBcpuIw2jSVFi2sl23qJqVQ2cKP0+i4whniP5y3iKYf7s1CK3csU", - "ynLFvQD284c/UDLISwyFsMgeFnOCQnid36+B+jBQzJvAED3V2WWDzoU8k6XIJR90uvjjYoasf+/DAFcA", - "vxnpfL7rd7lywgw6H+lDiBTwbzzZGaiPFEYHoYYhIWuzIcF1hJ9QGFHIk0oGhG+NZRZuB4tCTkDsJERa", - "t0ABtv7Lvw86TrpCnPpZnMbEqkHn3eI0ni5M4y+QrBT8jVtHR68eXTaV0ui8ypy9wuIWvHS6ZBPQ0OrF", - "bV2DqZxMvbICGT5+8vgVC3uzagFoWG2zT3Ohlwf3b6WRmdhFY+Dpzs4O+zepTq3T2dku+lU+xnY8I6H+", - "vFkw6HRhQWRG/0wOScviP/mmufjgU42FFS5bdyhKVC86zqk0Yizf4wtYXXSXZzOxe4XN4ZmT52JhquK9", - "12pQGLauGvrJdgnup+XAfLNAaW9fhFoSM+5grq+8FlIVZyFhr8usUHkdSxCSJbll9DE4dYf7yDF6J/NS", - "7KZnY/t9T+V++Yb9gYLAdUhQm1XWef5Hnw8GargbwnPrFUa24dethS4Hnb0D+Kmmub8PvJEHuxwn/c3H", - "hSZzncGWtW1Dsp6DjqucNpIXsJCxvSc7H1dF7v5Q6BEv/kZ+sZtQRrAHuAMdV0VacABEVtu6NxvHre7s", - "dkZScRhmi311izA2zZmEXlbm1yfhAt3OFEr2wpheCK/Nou22rM6/2mffffv1N2mph7z+wOv0os/CUEI8", - "pJjB7QhcEWBPaLXyZBBRzYF6EA6udbBdkQ8UpqcScQk7RKXne5ZxpZXMeFF7jbw4jo97WhXz4J0uIOmz", - "MmMofqNn0jHp0NnTVl/n3598+6dvv9156vW+RkzZv/99p/fdu3/7X207ftt6IioZ5opxykABabDyl5r0", - "tcGHK0OsbzBvC777AvK2QjQcZxPghixcRGykQ2OcvVhj/OMLIfQrVDUPd9oY29dn+1xRzL5YiDH3DGJk", - "hYvGO770Rud1Erm3M07l+BSvKIe7bG+kjWNyzLiaM+eJlFT1BahmtkUXx5QBfibLRjvHZ7KsU3ISZT+M", - "le4HHXyuz4UBTxQkZ/MJ1qCGQ8qdCDW3IBWuzm2xXd+ISoP/QuKTm4oZ02BfNYcQr73DMCQUFq4oQ8rO", - "VTY1WunKsn/oUbNYV8g+wrwxuOT9hx5R+J9vqBTGSvAj+tFDvnPdtn+VdknljZ4G6kKbM2HArx5Jvc8O", - "td/OZKzYWwLDFFL3+wO1zx0v9IQiEVCexBiyOcumIjvDQFyo2B/2VMHAlXYDZUSJQKo4P98pN67PDsas", - "QAE3lSXDiHsbgo7grr1uvOuboZA0nusSUhGKOeOKifc8c12MUuuhWllvAlaFibXTvPasQl6ZqrMOvmeV", - "MqKAUXoCh3IVfq8WCQ1u/7NCW8j+2qOV4176sSGFeIp8mOYBAgpPpDKKmLAUMxDSEHC4ManPjy3eSwIM", - "sG1sGM4LzmgmiiLsykBlmrjCuWiGjTb2DzI4uFdz++xIzLQTFF2LBeUGyjo+8bOGZPl6S0SSI4j+XYh9", - "67PDpHHrdGmZEhcDxfMI4qFylhsulWX1pS3sPfwEjZm4+faClyVFSQwUVqHqldxNk0Pab83YR9Z3afIF", - "zOBMzEmt0eCnK+YU1xYRigGhDAMowjnps7+KOd1LUwacjkULyFPuT4KRKpMlL1DxIdYaWRSwv34o2maj", - "vzXVxbIU7dX3HG6IpgHIge6IDnIv0pxQ2byHwKmtEflPv/njYkT+ZwXa31jqHu3ilVyST6+xFgL0/n/0", - "qL22QM0xAwvkWSZKBxdvicIeskHb5G+BB/Sno9cBBC8w8zaO3F8fngFp3vPe3pjiHRfLnU4mBNassexH", - "LgoOFd2syLTKbbP5NVknDT36RpP0SHW94YJd6fU4yQTfIzCpsB853HBQCD/KJK6UdmxEeXB4/KX1xzWn", - "GHtYt9vINfTfyEzcZJJhIPiFXIugNI5CKu+6aynPWCidrF6lntLOU9cLWun9F/sUXsvLaXPdw4UMTMXL", - "573Dg57n3s3byX5nWQve/ocebVA+3vNYYV1vLI113ZhwhpnO6PRKEh6N8EQi8vS00or4sxwFN2o3K6rN", - "RzZjV8QqbJ499s1C8lh3/SleCl74seT/rMRNZoM93VmXDPak9907MN4/7HSffPfxf7WjiLb1Cuk3nbZQ", - "iH9WogKWbCqlULpF9azT7dCVcLeToQYl8raQhxW9guBv7TVU/wzh5++6txohVhOVJ7HWMrv15X5KvJ+c", - "nnY/mBREarWI0CuEZ6X8YvvDP/ToVOaXVNsn3W3kGYcYj702nyX6eJ/9DWgwhmpGUmNyNhO55E4U84EC", - "2xWJFF9FBZpDyj0mHHvtNCjtczCcpDEC8pmTGQ9UquMHWyIq1VnTVmB6BP63HIxdUjhmXvgNVGrJoPsa", - "Pz6NAS0U99WmhKNNkqhRt0LwrTGHhAueKlV4g/AJKUnXLlfR9k8NtmgGJjt1GQVfUr8/BOjXq9ALqwAB", - "51FWYeU5qMkwJxEwUGhyOjNvVNvxkruHso0Nv9l5NmSVcrJAKVrMk+xJ39WU24GCHaYs/6KYf8+yQmI4", - "4FRXRR7yiWKcPHNggoEFyOsUgoHi4D5hfGTB8Gy7ohDuPtDe0XXQ3P3hsD+IVgYbpnYpkV4ejIksd22Y", - "Y5pe/t3GCsWfWhWKdzXrtyIzwq2vfAFenCQWGL8BLIG0bB3lb2zhzz1cp7EsRJcJdc7OucEoKO2mj/oD", - "9dbz8Oj+ozYxk4D967//B71IdR+NalbU16qcjmOa1E1ebUEXqzQN/DWsSCGDKfgQa75SgyFC3PBQLRHw", - "9oczMd8AIQhKLcPmRJMloewLI9E9tkTDfQb1fepsKizxEsJ+ZjoXEFuO1PzNzjP/AmaTQ84FvbBEri9g", - "tEgvG4EMEWlRFMB9z2f4VJ5/s+MnmdHI9+0FuqjpDEJyGzu+sKHXfhKQGiKNXlm0tJLKmZgT7gvGw+lS", - "KC77vJSnZ2L+qD32/kzM14qjVOLw3q97vf+gIiatd9Ar4V/w6iYsPSEt3OmBPKxcchqv38+LjUPdiDuK", - "O8URrMaICrJLm9URp1fzgN4zsfd75jDNE3cFMYti+TJnIvjeeLFYWoDMfCdnIqhubbbLcZD9N10lgDpa", - "UyOAVLeHxMHPqRBwVV2OyvbVRLas2p+Eyn5rJR+iDgV3/YjkH0ZuBjGI8aung86jRiE4eLjCz4sNrL5j", - "/KZ7ebLc0tCMmIj3jORpPbj/hIH0H5+eU2jX4kD/s7LCnPYfrxorCegVg/1mZ+f63bMbYb/CDh5Hr8Bl", - "wK97MfecqOM3UUwshhQlhwMerD8ahIQKCKlNW2fBqjC6PIm4fZeZFFhPOje6LK9H4t+bxfbr0IJL1rLS", - "3XaW84NwK9bx+mRS40CswlbKheOyuAe+2x9i6FoY0iXrupZRe0oOYQBhl1ZhUPp3N3WRtfLilUj4dZV2", - "JS5oduD6D0EgBMLAcjGWChJqk2rrS96pkFjzkxVsn9uQU3MsZ/EOGaKyKpWGLHPHW/MRVDU7xVweSGho", - "hvif1GMdb5Ky02zsWcgKQbQ4yDhQjPlHoUj+KVFp8iOrk2saT1tbCj/4vcA0Bsw3pmyJ+oUy5icvfQ2Z", - "J3nL42bDuM8LDeNL8QoeSqFQOsmZmF9okw8675offOwudg6ZPtfev98rzGHZfCQjnc9vaCCLfad/flzY", - "rNgM1VDpSdXjRZEmRuHMcLxJ07FZ+kdoeRBu9E/reUQi8698XE5BisSKBH8q8/cLZFq3FbNBsDXq/+PK", - "03S+Aklrg1P1DY20kZnkx3CI+VIso6hSLJjUzGqrs9WufCYpHev2zmTI/7rTs4lJYffgaC5s9y2c0O7y", - "hkBa3CWdq2o2EmbzzqEgm8yuk0PUmXzN9L1P4xOR8C/nE5Rlto5TJCB59RJRJiK+0Rhz8gp+6Zd24QSG", - "ogx02IqCz3hj/QcdqI9CbK8oejOpZDFbeKcy9MbUuXJ3exsulKfaut0nT75+9nXgbPWaLbG4P/yBPRfW", - "sUPDMycz1Ex67AXANgYlx6smGbpZRUHVzGZ+CiJNEvXfAfodaR/iwhv8mHPsW7Azb+R4rcYKZ9nWk94z", - "SGrwStFMcCXVZFyRFUTXfA3DnPJdkyzMRk4vtLWvlYViFxw2DJMDaj0Ish2QjxM1wMimaQJuWwwHaIMn", - "Cez5tcf61j3ckQ/4hC7KV1gbGH7dtAVxnxeugmldN7YTAePxTYVVHxbcxLcR9/qiLbT1SlGtz644yVuI", - "T9hPg+Uvt3XvIiJ12X2xHaCNLncqw0dUzrNG7A7fM6GMzKYAUZuSJgBiDxR83GU1aEb6OtXb8XQMecuj", - "eaBoKqAnqe5k7MyTp9FFryy4ElDx9ntWWcwPicBi8W2joWID1IMESJuBmnFXl/SroYMQvpsgQTKqJxpb", - "XBXosEcdvazndOOeiuU+V8ZbLu9QEgtx5Yvh++TAo0SvFhJMjh6Q0t25Q9ZE/aw7jtsfwj/xh3pul0dX", - "2Hrkq89r7UTpE74LwTAZkWmVyUJS4raBXCrdK7SaCNPLhYXiMgk1GTGRgPULjAkSuKUDTCiIGnWa5ZpZ", - "vTrwYpmYO59ZUL+lpNvlp4KiOW6vvvw3l3/wRripzt9qtwfBkflvJakjxFiotuN7f09vd0UiX8sB8/2s", - "KKmYHu2rMY8VERxHcAIhlfEq5z4AFtaJqY3Rh9zkBQzCGWZG9tk+QEp7kWmtaLBfL2DnzD8c1gVAYCtD", - "tU6nfTvJAMFWkIpSGQl4c6BCCe/anoBmVsSMrGAkN1CwLHaAtW5vAhTkk3iYIUp4YGO3lJuGy4249nBa", - "PomnfZpSYARVzcBYsHvDFNPeYqH9BR5xE6xxVaFJQP4lyMDIM6A0TMmlgULfqPqrOY0TfR/CgMk5UFTk", - "jewBoy8AXNBbW4Al2Ad/CgL4nYn5sI5qjVXiucVcBkimIY70lWVD/yZ9g47ushChjDplw2UJXlp/oF7U", - "8E9hAmQOFV4AUDoD1WtgQ3AGEaygrUcGGf80NMBLxdrpCCsXCtWlgyXAgWomEP0A2h0oT8ulUN5+Leao", - "+1lvyXtBUXAzEfhiDazcmjNPZPyCDK3A2I64mtyU92exL/ID0UgSh9BNZnpfNog1sK1EzpEFYHaktQDD", - "EPPBH2TAbcgA2gLGF7hdqA1Jhqohct4EPeiTxEAvpBo/yILLQgxCJlFlhemNeYaFU4Gb/kOPyBEUxhsc", - "QLV+jNVzIA8sh3K8tlnz3k86gtQK00O8d+xgoEojZxLy7aD4x2jOhvUmDjGWgQokxVToVAoMFPFzZjWb", - "akvV9aMMyLBukufVLTyYhmEHKtT8yHRR8BJZCJXdQVU8QhlSFSgv8bDrNj4OFxFLPDzMK+TC3TwrT7uE", - "Md0RN29MvTVBLzAOT0ieb5PH/4Ft36a/fvGUJ47jdINunHFDzvcBpnyvCoO79HTdkM/580g7zT69eaq+", - "V0HYGxLXcpD2Mo1177tYX5rjdcv1FsyTdA0l3KKNJRY8aU/wPchvw1vffrC3SVf4AjS03+JWrnUQaFUr", - "bKlbAIxjtKVj/buqAD9oXcVgSRPaw43+Mrk1GJBUENMJM5OKXztw8OeNkM7Rg550G3oS0fI9VJS2sUrJ", - "Azu9V+wUi+8QQEZ1OblAMSGePAO+K+1ABSZEpYm6TY9qqFkBucqhoCcVMQK2/P1ALRYcCkVgsDaqw67q", - "UqTenPbN7R0eMEWFMUptAMA1qSkjVUS4BBGR1FVaXY3oy5QFdcEobZaEQv+upUKj9mwsDPV9K0nAVk0J", - "onYuHNXMzRlvlqWKNa5+d+YKruZt8Hkq2Xhlvg04r6Hc4/VF5KziZM+xIufBCyy2K2ezymH56nBxzlU8", - "FFSeW+Ts4EXkTcOvd74bxtK6UF863LfH63Qqdr7ENrD3mwyAxR6uFPt6tRvsMkE7/NCpd71On21gTSzX", - "lb30Cpy2KFAlRDZwd10VE66sJm5QZBY2FIed1pu9cU0Ru3xDBdRuUmGkPVlO+Lwyl3DZ9BOYxDiC/KXQ", - "pTfBLhYPrMumv0Q805s4ry6b3uBx3aDvNReSzRVPUWSuWUu4dCj7oKmxnKAXa6wA8T07l1YSGq72Jr5x", - "MpMlh8gZKu4vLVbyZ6VQOez5rbKPZI0ACbYsuGyPDFoN13oyFQQdQSchI1YTam1abzfilZffBT9nqKOf", - "B2DUAI3DMggnQFhUVPwEu5gKgE0O10O9idFVyY742IGaDAX8o77cZ3sqwTPQlcv0TBA2QwwRheiZORsV", - "UuUUSRCqP4YylVQYW2nVk6HgvCPYAG1mEFuwKXTMC6GsOAJHj+dXpRHWVmbzmtmAZFQIawGhlQM06isu", - "i9jGs+vaS+AoEMMONlBUytCmc/MEylabXCoOmTaWuRplplEAh2qxBoMmHpMbw5BBCpSeoAiFCQNtAfT1", - "s6VEkhTRaxTN/gTJcTHVVoSaS41sC4qqGQmE/oC6zbefIX8kYiEam4ZJ1srk/ov9kO5Q540YYXXhz3I6", - "pYGqYz8zXhSAlh6MphozOaZg1Aeu5vHSIgxILvKBglz6cmJ4noCVhBQmIrqWmvbNSufsgtuBksph/FWO", - "d+G+afHecyrpinkECjGIJVA3pS8UQbkQ7slAJdlD0jKj/bHN8fYeZ82UELllPJ9JlWJFUwgUhXxRbHwE", - "Cll4U4DgANztiheNMaHF0Br1FDcTdMIXyeZ8rkOgqXjXVaNCkfC6nlFLffAu1RBvRXqoqbouNU7tv9tA", - "X3+xbuNrOAs6Y58qeG+htNg+Ek/BszMbZuJ1igbVUBDJAjHcnG1x6wAWn5/Id098H3ue4gLnxxPemBNk", - "XaZupXoyn147aDvI8juO2Vopb44zriw7E3MId+VJTRb/p6oLs5yJOUa3BTz04JmdQeCrEheFVKKXiwKB", - "+higmm4huumjPqsxS6OWx/EdZCXQJ6qDw1OZDxe1oLFE2Kg4omLOSqP/gcgq8WVMpa6TCquIIz9Q0gFz", - "BomIgVy8KJbmnkDP+e2An8N64Aq0BkZlXP1VzO1NFbOk5teGOa0TIO0wpgsU2LqHXQwGhq2TbZvmyWTQ", - "geIUuPyd7mb4qM3eAfFqDOxkyzoj+AzCCAkf99HvzlfrtzzSdevprM/kZ+jWoYO6rnN7SUCtz6oSgb/W", - "sjFvmfOeFf4lT0KhqFxS4wALVYQitGFV+wN1MEZ7EZNzvWqY3gCNq6KoE3XZcVXCxc0uFEjAqHrsZDdU", - "Euqi1B50/CtvCaCKu6kNmNN9nueAQJhJN8fXfpFFnnGDrZxm00qd2f5j/O1lQI6GH3vx136KzA3D8TPg", - "RTKe+g3bPcVNlsJ2qQn/VaPmYDr8boBE6MMet1chxJ46V7xA5BBUCqiPKptHBMMCNhtV6JCdNTSC55jg", - "NazTrZFhl0Z6ooXvPZ/gXuXDMFTumfHQOl6IoWe3+sKzkKl2WEN4NIdGwBGRdB1wSSwfi55vhL0+fots", - "txUWqJ5DO3ZRpx68X2ZSkRsPEWritBCE9eNHfBPIOjzPJQqww0SDR/l9qW59hGCdwCWbQHD/vx7iLfd+", - "Fsa2AsLRD8zpM6Foq6VNDtRPkF0vLWID+OP54/4+WmLo/Bgo2AooM+I0OIswUT9oTDOdy3EA8+8P1KCz", - "M+gwqXIAB8FT7HlWrgV6hsCoXURhXoOr/Pvg/EdUC9dbFkGvGc0ZVgr/QoIH/yrmMUIizAFcgDi1T6uF", - "frXQvQXRdpU6GOviOpJaEkT3jerooGnypboSqJAW/ugEVJpcWnBHN/OJA5LSQFElC7iEDEH6oW9wCBo5", - "mQhDPt6YIrSqhMXizfubMI3LJHrww1MdSgZ5x56pI8EOayE9L5NSG0Hia1XM+wM1NPzCSwAbFQDLNELC", - "hS+g9rVA/RKNDCAEG10ziw4cBJJv1mqWlgnl389XywucSCu2WjiF3Y7hF7eNrLZqiy6t+xHX/JOrfvyu", - "498Q2e3yA//lsN9DYTKhXE+oTOciX2RHXpb075QHL4T/XV7vPlA4ROvW+ZDIMtpzueIOdgfKVtnUW5Op", - "QOpVSjo2lcJwk03nTLx3hoM7gXBHD1+86rK/nLx5DaCUXeZZOxQoANZG8OpeFYYLxSk3eY9fcCO+Dxlb", - "uSgLPUcDDl+kvNyYbwvj4Y4zvF8DIDJ9QTKobdfawchWcY4H3n7feftmfH1c8+kHxr45Yz9AFWptwNus", - "PioPzP1eRj5fj4i5t6U0fnfbueou4kCd80Lm0VUQ3ArxqKIMCeJDhHydupKH9rJ1VMkib0lGD96tJdNs", - "X5fzbqhTpE0X/RzCdNH5kRk58n+QeqAN3lhHLYKeewEVRgLRpAPFi4JZdFO2x42vLIvRuYsc5nUBV0dp", - "EYrl+hMPQukaikl8uhc/VIddB+hzQO/cBvQM9PUp0DNp+eCa+375ZSw9J0jnthwT9UXVsKSpbH+Af2yE", - "knNAfva28NVFMZCL99eJknN77GmjEI5G1eLrCRC/ETQfhRQbjOGNCHZVUYMVu3995lWD4yxzGKKpT8b2", - "uWclB2RjOl9OKc20t3Dx1tJb5CrXE7WPZUBqErypOvPQwx0F7+MIcpzkygPA8/xTWeq6gvJfCH+9lXi5", - "H1bH414lbG6jCPdjpw2fiCOBJs3L91NeWXdLmvJtF9/fy/NaJkHhqssl0toK/Ot26ppK78+EmYj7Gvk3", - "V9nUaCV/RZP6TIiSYRABQDzPVYbhXeI9Egn6qZHULCvkmWDHU13K8bw7UIfauokRtsuOn0G6AldzZsHw", - "hbQbE/wafbZXWM3OlL5QjNtdajUOBta1O1D+8YhbAT8BrICBTeUFPRG9TM9mwpvy/gFgwT3Xbhrse8xC", - "qANzZ5WFo0eDGs3rMm5nYt5nlMsAVd6E6F3wOYPtg/iiA0ptUOKCpmNRL8LOAG+lRM8FpIDGWVuEfcFk", - "CFrd0AAfWQhZrNuBewYCsQvZJ2xQ7ew8/SM7iOH5jx/vsrcadwhdIiPhLoRQ8Lnts+OYcGMdN26gUIVT", - "c3iByTHE9BtTleiX8P1FDD3fuqcNcmZghXjaDNtY9r3DA8u2Ag2wX7Tep58e0Qaygp8JJt5DoBaQxQU3", - "YqoByAHTB5wOywLRQ/qiV3CMRmrAx9Aof9k7envw9gdcAcosQhidOgjFbxqF8+pzYQqsZY2BcrY/UMdU", - "ehWu4XEV9w4Pwl1Gy925Ety8gZN8Ve2BArrQMP9VQmZsEuHnaQ9uREY6J9iXfWyw91JlGrIs4LNuB2Jy", - "/OcFt+4U5pifSj8+GBQQFKUF+E3b3UHdgljHa1467ZsB4KrO7nfffdf/7juvG8bXnyavv/EbVL/9FF4G", - "WVdnN8VZ/KS8eDDC2hD0C6MLLrr7MvaP3Q0le7Lfd4QZ1BiBrYrWOyL4mfmvAdHgbuzkJxvoCYd8Xmie", - "n2j9mvszBN9toP/9pKLr9I3IJT+5bwkFqQAFZgdcNkrLIChrBpnKyEeMygtQwCQemk/3AOIt6CeVWwgX", - "qDefPv0339NNljsAoTmuCuioPr3dDULhN4pYvz0esDCT0EsLI4A3WKPKQhIX+kKURmQxW3LBqf9qn333", - "7dffsBQet/7AU7XoszCUIKfFTDoodS4tw57osj4ZREzpheq3eBeD7Yp8oIaQtHiKkl3YIYYTfO/tI61k", - "xov6fgcwf8PjnpfRIW0ZdCNbmTHAV+iZdEwuxZLWEdX//uTbP3377c5Tzxe7EbR9t/Of/06w7207fp+q", - "SsA+N8zZTbgvfNXGgtEW3uDjnxRdV0RX/NNNR3vEnXiNqSQbc274cKVduumgn/P8B+7EBZ9vbM8ic6oz", - "qhu2rG/j6w3boI5P5Ezoyl272MHznkCYLZrBwM+vIDqwFuK2tLYS9zVN7TUUZwy5xVS9MRcjl96mUJAt", - "zlzHmt9C5aWWyg1UCGYS7wEmJ7lgt936lhn/RD8DROR26coZK3sLT430zE2FwbTnZjwT2yScCeCzQjwT", - "I/7BxrJwdIEMd8dQzIxSZaKtQDM1AtIVBkFTsmzo/AF3fw6TQcyJsG5xilC2jQnloCQa1HIIX1KGxzhg", - "eUYACq5yvJtfFWAckA24NAdISzcj5JMufLefnBL3CZB8C12vxvWkRYMzBUb3b7iO5c3B/qWraD9dNyYG", - "d/fICVcALWgsQIArwEr/4CZRk14oe0cvXWhz1h+oAFpA4AKx2L9nVzzLqllVcAQTqBQyGkSwCJVxqfDq", - "QGW8KGySFRTYKJPKOsFzv0rBbacry4amUskhGVJ9FxxdjRUTbsE1aY0hG0IJ8Jk5M4eZh3GvBCFIuro5", - "2IFmJ5cDDezcUM9tjOb/6FFi/ueVCaU+sEJaEXby2ks4Xj60o5psQ5gQuISppi8bVQ7SvxpVJh+440ZO", - "B0ACX80nroVLbgJT0HL+7uwoJPRG1bp/jzgE/GrEcK8CA5IdvPUS/y2Ef0/L+N/wKq2H13Nspq1bWUU/", - "ROxBuef9+jomgicF8G4/ejBbkkIiqys8ryy8f0/5T11yOAg3Vk65vWMpbMQMKq5QuW5twiI/SN5PKJJ/", - "ozL3fla8vyvOE9zAjfrd3CVgn7QNsaY3ewUOIT+WvAfGxkD9Q4/qmpjCYKaaM/xcGJuW2WNS1ZBu3MJ9", - "NxUsG6iSV/7lGXfZFG5uKqdn3MmMhoAeIW7PLNSMzgD8DcrVMafZHDIGa/MsnEjleQExiko5WWAKIA2O", - "F1HFt11mNeMDNYwF24d1pfyZ4MpCQCBwHwB4R6c+OJ9kUYDCHQMCQr1qrmLVvQAnh6kMULI/FMoP8er0", - "uSVrA4t/xKF2w/L6mTmh4LRgo5ZJVy8YL4r593UavB5AVBQYgE4HgUFwu8IvBlTyW64Dv7oS/z0VDrdb", - "av/yoS0U10eKlCpxGviTRqKilyIpQGndxOYM5khy7BgcmJo8fsfV9q9dYJhK3fOoMl1ZwN9wYlZiuRia", - "PqkgxG/Bs9dvcVvHZD1hB3UgQOKklyq3wS9f8LmNqTQIDOIZjqy9WqP5QA2hqtmwUb8OKtxBonIKvkkA", - "mOS0VxorIVPXwaVFsVS2rv1RzyOUSUIUTtCZ4WSQS92PBrLUqLbHlOf6IoCLz4RyXZYFMSMdq8qQKR5U", - "ZJorDKUAGP5/VkJlgmQQlIWGCnn2gqKfJCHvWwzU/J5NBS/cdI7wzu8FXTicSUi6qBTcrg7H2mTizwjn", - "L228j6Bu1JyV2gLKab26cOcBiXnchaol6dxxTSh3vgZ6IQyWSlE8Zo63O6GOWDGPFy61I7LtzmWgrphD", - "vpCQ1/Bh3ugFxlGlbt+XCJ1eDjndYFjA96On8cFg2DjPr1KJnrq0pJ8jBMC5/2nRPvQxcINbCfo5wg5D", - "2M/aER/jMM/EnAJHNIR4QsysM3OsNQ0Fmcd4KRFmA14HqCAJbCzTpcgD00wYgNfrjVSZLHmB/CSAvZCd", - "EfTiNLglCyiySlxAR2HZ8Pd64WKEbjbvYaXEFQXB/9jtzKQKfz9ZDDzpdt73JrrnH/Y8R+6Fkqc94HvC", - "dHbHvLCCClbcBIOCVb1S+OPT6+59hc76IsYAxJ2vc5EboU+v9cq4J+F13nPBfjp6HdPfW9o1lFjRX19X", - "EQq2zXt7Y9iaJZquJhMqPamLAuuZe5ktFbPeTMtts/mZVHJWzVK68Js+EeZzIpLuY2lprwIgYlBdEzhv", - "KzcNusSt1Jy+p7kvdCSiKQGRphEma6UYWZv+Eg7S/ov9BEjgJrJgkqL1d2qwBLChheRz0GnBFDh6tc++", - "ffbdHzHQBiOsDwHaYkH/xl3A3vrsR6jXOVAzMRuF0B2McmfGE4GV516Egc7MVFUUVGHTiJk+F0Te+HF/", - "oCjKsi6F4b87x+Kdi/i0QSujoSSaf5tyC1M5xiW6VBAbrSah2ZcnfEImE2bOcFYacS51ZcMrMzoMXUJy", - "gLBLNhzQlvR2Bp1hn+2xmbTgQYsW2Nc736URBfpcmAsjqephktZRQXrNatk77r2B/W2P/kzH0Qz/jL9g", - "GOig0xYI+u4GwwtwQ2BvlkKVgYp6QLj/9tlN32r88gnxwSWmj2NiMKfFtIWG+PZE1yJNkTCBIscRFSfA", - "dhIx0mnpX0YMT4AY7nXU772sD4A8kUdsGht4yhqUhG6nrFo9/OD0sGEjkZ01+WtdKBnCvGOBvgQ6aqC4", - "CQw1x6Clw72T/b8gWoe0UPabYkUBEYwXy0yeWEwL58TcvgfWeS9Z531jbbjcD6zti2RtxI+uyNwS9bcu", - "l76+HFMCKncsEIfrRu+plrrD2OnVvsA9vMFMJsRsGOj9ixduG+WGDrZkx7ZHYoL5rSvwP/3PyVLeVNhp", - "3QP0uNZP/OQGu11NHSfLC87gcviTPcT3hqBg7p4BLE/xU0gqKwRXBIXdXp4GX2hnCAuqRiv6ReX0eHyq", - "1qN/vLtV5kJzWgvhiq+QNfulEw3MhlUlE+9Lr4ZcG0dCDSClnlYkXI73nnBt9+P+Ptv6sXRyJq2TWR2N", - "l82hvoDRxaN0fKG2gjZn40JfPH68O1BP+gzgYWoAIsxXNmJSFdwEnJZwF2e7LOOliwEBA8UYGy6CkiQB", - "K+RbB42Xg07L84F62mf7elZWTiBCm2WFxovMEbciD9h7AEMCPhQ7UM/67Lga+TVAdyCuR8g/jahuAbqE", - "bQGi179Fl8ojGGu4XERgOHhvAWDaslIY1Aoe+QU7qeOJ0rKlHIv94RCD1wbBhlF7D1o5hjcC6BFX8/Au", - "m/JYQgRjllJCkpbxEd6EI/IW2ABBkRqouLzkT0VvM1QJ8x9Wjl1MZTaFC5YaLxj23w+jKOohw2C7YRu8", - "dSXei6yCS/j6fvlccvb0cB9aqCuWhNAe1JH8LKRh+kLFTcAoLVy9Hr6VY9XVHmDIzHu4DxdEkrYbw0db", - "zlVNha0RQUAQtyq0scs7qh7RMo7NxHhiayzUkrjWqKTPH17AEIUjsAHa952DfN/BagU2kVdw6xu4S3ZV", - "m+v2QLBPFvCtL7jdCNT6e7oblggXdv2SnQTsEhzWp0jzD8lfp/KSDJslrWpZH2zbvPqVlHwO8s4t64Av", - "UPBc1YpoFOl8cFWsz/lxaxbwGshzG9jIatNlz//85dIplItdR580n8BNH2jy0iQJv1DXZzsv0eOyNXSJ", - "snedFHm7OuPH2zXYL9UuwlmIetgNaoafPqjfqXL4iQt2dX3wVnwpwaVxU1ykrlPfzkWOHZ+IZHWxfuf9", - "5yEwbhzs3Rufl0tXHCmzftQPsnWT1P+JWDgVeaDMzz8URvB88yNx5N/+Mg6EH+q9OA5hKOtQdnh+N+fh", - "zgTSzwueCShvA+nh2ZdyCFNf77UcRcvPBTo012i6EJWernj46EuxvsJ416or4SWKwn8QE5drTyFdoaE9", - "JcRxnfS5/SH+G4WILooRz85W0+0RvXETlNv90BYwnI7wqjHD9+Yg+GUTOcR8PxyBS4N4dFHAUrXbEIge", - "cc1HwsuFzdWnL9ctcW+tieMHM+KTNZjmpfm1HAi4wN38QPwCr38ZBgWM9Us4Er9gMMPDwfjkg3FBZLn+", - "PIj3TigIm/DnQirruJfWl8Bx0lsv48e3BM252O8mCJ3xK1bPFdO+FgIxZeuL9QLGhtYv3fYHFUAlV0MK", - "Lk2kPVyuqQ2qyzLH1qYyhq4OYhGkG9YT27Zro+1Z3B2oEKja9mfV9qxK6KZ37seyXz+3X5zeHTH6T954", - "qMxj5j1TKYqtXCQF+ghSYbIzYHoWK0FdQhGXH9jtTKuxhOQGSvdZvJ/zP1dG3LPTewOQkGGmXwYhHYks", - "DHgNEwnZES1chIXvKSX4kykol4g0sVJ1fIEvPPD/5CYFl2TNztEbK0RArHsBqMtSTaBwXV6X4PzUvTR6", - "TeA7wkL/xvmAn2QbC7jJI7/Q55obOKPLFUKixoDeWGHYgCCw9NJqknipHo52c4twRdacbHyBcZYHJnC9", - "W6ahuoG9ojHzI331he7bRqZU7PoN1G7YxI6Kn1C9B0ar22pL0W/Ei0fz6z+OmKq6+jhi3vNvnkfjNG+b", - "S292/n+ibOIr6Pf4yWdTCxkH609+HPthePs2Th519oYrOSZH1WUnby8GkgejZ0Zftx8+vvh+w40RZ7tu", - "4TZwYiwu329Rzi1t1vLmHIYtEY7n3PF+4nq8nhpUYdYri1GFISjt2FhXKm91oCwTURzxpxHHdki42v5A", - "/7oSvfwcow7uhGzaL1zrUIj7cdd6BQIMeTIJIS4TQQ29J2ezyjXoIanPsIIcZoUng5nORZGy1kVzHEt8", - "uKlgJ4bnEgsAsjevWWlELjOnDcu444XGYhFDapd+HfYH6jC8iOl7heZ5qJwyHFQ7O8+yWXGaSwP/Ftv4", - "yG8hPcBKueJ9qa03BWMlVoITh+LgBwwmEoayChquHkrnRgXqWBihMlF3t874SdbHCGekOG+FUX5y/eNb", - "yYT2VtWqxSJgvJB5Uu/OsgthkGOFyrzJhcotDTnczIRcWLFQSO+WhnEyFaxSni/lAac2LfZv4nmSFrwa", - "lb2kOjCAVElnIY5AqNz/hCX0eT5v1RhWnVPbSc49PV2dz41VYx0fQZp1fdi3nBGCCWXFbFSILsFGA6b+", - "8c9vHjENxWYxZXk8UGPBXWU8P4Ih9NkbYDifwQkGKiI/gDHif8dKS8MGTxvG4jUFGDHjWmoOVJnwJMy4", - "loaFsdJut5auo2W7KZd/k3Hcmad2cRiXMq9QjeXrWz7zyIpCkv3WjBdjbWaeMHQ+74Yd7QHIYCxc9egL", - "ZajXqYhePuQolmpVNEF7vqVBPKeql5pQG7eQE7AnOzs7O8zoC/vo98bjiTGvYPI43msIR4AJoQ5fmaKz", - "29mGuBZSJj8sgb/L7IxJ5YzOCezFaSrVAQoaLtPe4QGUaPjDH9g+1GZgr+XIcCOF9Y/prdLoc5kLC0gt", - "vYlQngOLnB2/+KutK+79WAq1d3gQlWBY1F3fTI89fvyDfvx4lw0n0k2rUT/Ts20sQJuP6B/bE71dnk22", - "bX42xE9O5qU4hjnBp/9O7/kX2JYqZ4/wtcO5m2oFr1BNW3zjcH54QK8cVdYtv5AZQC2WGsp27BUFTshL", - "QiyxzceIRXYmSgdlmecqqwuH+Lm6qdHVZMoynQtGCxMKpwDekMmm0onMVYYXUIjvXIqLyAehsCE30mpF", - "wNGVFSzjVrBJJXMOyA1WYIGLv8+4VLHuCvTzbmvqXGl3t2kN+1Jv5zqzj0BSktU1Ec5JNTkNVZi6nfe9", - "XNqy4PO3+MYP+AY7Tt/QmeXjHvBxrFn2984vUCPk+MVfmZ3qqsjZAYzXi/UZUMk/ROb+d6fb+Yu+YLlm", - "B40S642V82/9MvVMNeOKHbCxVDmuItWss/+7824ZbuqNVtJzQL9UVERkrisTyDkrKuuE+coSigW8hye+", - "JnN4hf0FXgj1WujD8FlEkoiUbwQvek7ORGAgGsuiEJou9DPlJre7SHGEouGJbg8wlvLGW3X1GF7gB7mY", - "GK99+S+Ote8HX2xUOqhcHOm4Uln6faWSLveNBDglNsaKDJbx8VhkWAESm4PkR/wUNHP4DNvuXchcxE9p", - "3WjlAd3Jf4brx7KpyM7ieu2y4Q8vT9g2DuVXOMeHRs+Em4rKeuPZyMyGt+jPoVdRS20c+/rpzo5vHaGF", - "ha0hPQAoBqqdm0hJszgiZufWiRlhGRfCwESlGhuOWFeVoeqdoTIWTjSt1bd8KgKlvFl4qeVgRHrH5ZBj", - "fx7CTslAjXMkeu78u1BaXo5BtLlAd0hbwgJsV+Mg1XMnUoclqNe2/bRQmR2ovO0ZWMtZ6bMT/BkLicMb", - "dW0noFgE2WGzqnCyLCJSczhRWN0SCv2gxGa5GEuFTCuuP6r181LA0cm543FAND4UTo8fQ/lVbOrxY5xn", - "VlmnZ2HkSQDZQP0ylYVgAQihi30TfA5VFA5n2Ipz4dnwSCgxls5G2eQFDTvmY+Hg9LxUFs4MDDLTykoL", - "2A0BETsuD36NBbV+FTmD6u9STXwjr6qiYCfivcOnDLbVz9/TpTYzVDmg8GxZGl0a6Tf3eSHOBZshNg+1", - "/1w4T0bHwgsT37Qfbo9fcENbAcQ+89OESfrXmBGFIFhClbNSGOgT8Lt9m3+rhJkzGjkMBSeOO4HSL/iy", - "oKpcUu6Ktj3ZJnYcdjlRGmDC1nGVc5M33k63tN5L2I3hcOiVw4H6MFCMDTphqU9JGRx0dhn85H+ErO1C", - "JM/8U3yz8dA/9nPyDwcdvD0bdLrpz2UsJbzwJXwr3UI3y62iw7LRKr4SK+0D8fuX/z6AYjiDTpcNOmdi", - "fqFNPui8Sz/82G2OwFtP1zqAqZsVl/TpNctL+oRLlZYu4a4Gv20Z4cfNhrjZygSH7CUjbdn0SzeeqKxy", - "U21WT6b5RePP5I+PDWrjebAVDhvdA3pL4824MlJlRZWLnlQ9XhSBjJAwu5FC4lJR1/C/jwP1EY4WlvN7", - "MVd8JrOEW7GtYduIho9AmRleMt4h67EfVTEnHK+xFEUeZEDumfywXuMh2zIi07OZAOwjz/4S3vToku4Q", - "b6xHHNWrX6EznBJUntsaF+K9DOqSLfSFMNDwwRgNeORrIu+GzxIuamOwBK7VWwRooaAFKIDYQ9dWG3EN", - "kbUp/KgxbT/TKAtzEC2+KWo/XOjnWn3lLY2pMNKx4YrNH6LNVXKvOvhW9vwxBLlKDe0uj5GOKg1xuOqM", - "xnPC+v3+xyHxemLpdUQ6Pv9DMkQ4ucOBgrH4odA3IPLwsFha+7lf6qm+QJ0OthI3ss/eBB0DNQVCWIqa", - "B7wVxWOU38eIJue/AStvLdN95ykIcyET+eR/w+axyeZALm+1ycrTLobQqtcph2zLv/wIRHL9uE/fkdX7", - "i98flOI9bntzXfU2m5gfAn53yu3pXFenuK2XjKabDuX06cTw2bD57Bk9Wxj2KezdaWnEWL4n3gKa1L6e", - "jQgqhh1VhQjnZuj7w8ujoZc/QwREqVwFB1e89yqpPBdsK5tqbQXAI3LLSiNn3CApPUKTon5g2Ra226VG", - "H4FFOQLbegQsCCu+cCO5V2m3hmG9u2y4vF5DZEItP7B/Y/FTz3z0BbWtK9cYI9sCPwnPc0tzBgfD4lmJ", - "wKoJimUNd8Az0rh7RO7bOKSBslNt3JSrvM9evvev2YwX3IDfHbR8rDUK5xDtgZGA40anjEk7UNiYZ4Ke", - "S+YaWKMuEVOXXlMEbuOHCwVfewhP4y0FPhEDFTzgbATWf3A0DbXJhTkdzYf9ZT4BbQ/RxktBhhtHG9gl", - "8XaYTazQ4wCLWeC8tXH9gToWjg39vz3fDkICbj1ot7pMVTNhZLatqtlImG2CzeoO1EjrQnC17f/f9VxZ", - "eBt/2/9j2//LOj4r4S6lkOosDK8fmBvCbVrEO5bhwiuuF5Wi/Z4Nc52d4h+AlwkwyCjhijksPWdlNSpk", - "5ncGnDEIhBnWA8hqVllHkGawLyRf5l/ZhnoN/QyU/8RTCK1veDlyT+b4mW9DZCIXgKt8LkwUiU7MygLc", - "Y97EEqrmwUElp9OXi6wAj5k39vjM/45AHLCfJXfTLt7mDJTyYr4AS6mczi04KOJoYHIjQSjiGS/6bfYA", - "JfufctewBFZqwYMO2lr4o99WcOIkvzcJc0ke+k+ItRKFBUUEVasujmvRQlg7pLYu1bLKSly9/gkPxJJl", - "E8XOwtDj85WjX6soHnubH2BVySHH+Eifk+9lWG8EMXRYgSjKulhyWABUQDH3B4ZxxYu533ySA4EZMSWA", - "8qQRmSvmLAy1P1B7OSDhLp1up5F3SAuenoAZ7E/RAmENFNZM7rMfPY3XlZwBRRywdtNiYCNujBQmBVen", - "msUDNeNzxCVOGFwcdWQJdcnnTBcJoUNVMT7xQm2gxtJYB75Tv3ZjKMOcFRBOARcTiFEsVRjG98HraAT3", - "6i6w64G64Gbmu6lmClHgYaFOCzkW2TwrxCmE6Q+ZHzkKGzak85oPE1T5gaJRQy58lWVC5LaFdyN68UA9", - "R67pt4HU5VqFRigd2NvdxvltOxFNAycX76NRUdPh4hAamnDwOfUKcS4KEnp6TLSFEhG0TPgKhaNANZQX", - "bHiK2jS87AcO/h08lqSC7S7zoCuaZP5fiVsObLQwOzCb/GnoYRnzVoWmyy7RWLps6KXT8BFDlxCMirCY", - "NfQAwgIVFJzclpc5w37SbqL61RqfNv6vhp4XFalH/aZxcAIKKW0YOL7oSVT8euAU64GqjdMA0sFrAX0m", - "FLmiCESaF3MrLUYSAFWhu8d2WbKcnu7rpui+sQ+dooq5rtO/nLx5zRyfgH1Ckj30Br812ou6Xw/1rh6W", - "bQ+2DSxp4HGP6oYOXtiu7wQq4Tsx0Ub6OYxl4YSx/ZXaZo88fqkRUItLWJDK6cAh+i0a/mdo87EjHB+Q", - "F+uxn45eb4MiRFRESxKmbR5FpvEWNS7oJbCLmh5IH/NNvq0QG3GLlDL0OY8LzT2JxVWEgAHb9WSc+eWz", - "mTaCRkdKnG/txFRiGxgI6V5xQC+8SnciZyIZRtD34Mug7ZHemXSNom47wDcBd+8ycS6wsDy2/4PQtuSA", - "H1Z3MBEari98B6+5k67KxXah1QT+xeC3uqOCcGk9pYR612GOE6HtlCNd7MOev2cToQGbU2YMfqtbAiBU", - "joRmxIRc5TjQY2J+9SjFbCRA0PrGf4a4IBafEWGjzx/cOeCjTrbGypksuPHiE+mYdqXQI9/gc6k83wFn", - "etpU0oKc8QmdCf+/EX5RHz5gM2BaxlgAQiZYsqnooj5x8iBPRc44FsaEAKdUZR7CkQvermVBNWT/+r//", - "jw0TH8LSq2ha+Fc90xwEaN70W6L61s+JhtPX46N30Q1TOT3jTmbsFaqCtbedh5/AogDTE2/QghWlwV8W", - "lXfyp9Nt5fA0GjvDx4934bDAZWctGmu1P7xpw7cYCiGFhW/RVxY4xIyXXh6jmyfKYwqeYOgO8c3vHSRx", - "BbFBmjfMlr3lM69g7Gt1jsYT/Nw0TurNTiggXHgRg6y9iwv+JBQKthqP5XsRluYwlZvgKkVh5ncZ/Q3g", - "bLNMGzmRdaMQJQct/JWYJAlOcOSscv4ct/t9rstB8xJV+Ss5s9a4lWhFybnjFR9yKtUCof4pzrRFbCRv", - "0QTZ1tMe6ATMTtG71/ryM3r52SYvNyXclsgngqmeb4Hhw/QzILznwjp2aHjmvNRBdyN4VsmCAaySRe0j", - "0dcCcGPmEDYFDJIeOcgCtYKvI/is4bJNqsluwm0WfYuCrulG2k3ZOGo3GJQNjhHyFPXYXp63axeL+kMS", - "T0BxATBP1KFA9VhUiuAce70JlBuWI8RPsAN8C8fo7V3nxwbmX3rrTjXUs3Hkb3/4Azusbwj86bcyT68i", - "2/yKxPSm/FywqZxMhakNlExbF9Avalq10jaVPGrCs5Ixh3v86K1Gj5n6ysWr8YAcCptMUrVt1YEvp3QX", - "R+WtwylAHTUFMkwKRWkyoia3B8dpDNyFOaXbsOBTL4WJDj5/gC0Zb8zKX9EejYMCVwnuwYvA/k9OXrMt", - "ryz1TnTvtTxHzKMQdYfrYesBIuxVrdZDXVpIVhunYQZjv8C8ZtpQrZsiqV7Ury1NnYBdET91xLOziQEM", - "eMJWQ7D+SkFAAGFEHfGxYwWgTEWdCHHM99NCBf6nWKoBZg2WObA8DC9ArwNGLXgVwhVDVupCZvPWq2wQ", - "CKhVEOoXOXcGHecK8t6EaeOLT7+ehtvSS67CqcmbvwqvrDCnMt/wbhf8Z3zlJVN6P7rBxeVf9AWTDrCf", - "7O7jx562a9pAeDWipGjkN2gphipKU2sxxOosS3UgOiCjOcvFmFeFQ5d+Q8ECvCsmlfVrpRVImufLFGgq", - "Zb2tYObs2Q6zItMqty206D9/SRBxeYPi0VIMx9sLCCksk7OZyCXk3bAtgsChPmEoL+igYRsjQtXeerKz", - "s5O2D9XU5UzgpdREx4hGGFqmlRXKVtGu38eoDX8cjgSp4qiehbgtHiI7XNOgSgGJ06W+5KyglbV8Uoju", - "m+fl2zx1/AbnatNrnHpt1x4p6PnmD9SiR3uFm2yFG3vxuMGgT0MTlwUsNIMSwm0OqYLpwDYJKzjCz4Gu", - "6HQm1EKaNjn5SyOsFyZSNawSi+KKLgIUO3q1/+zZs+8YTp5tif6k32XDpztPv+ntPOntPDl58nR3Z2d3", - "Z+c/8NquZgfhXq6+zrqQBcZDstrZGZ0DgUO8wmVWA+UHHajLgqsWZdxXeHszEaYXY9UTDgPf99lxuNIb", - "qEpJOshDZb3iXsF/Z/Bf/Ce4+oL+npMd+mzHevO5Zhrw9JuZf/gNm0lVOdI0nn499Q+ffs2mujL47Fvw", - "U33Lcj63kCLE6X7xyR//NEWt2P/Lv/SEXQhxFpbCn+F9blHb9QYJVkatgyBhbxcP7CoJ9mRK5FsfxoJb", - "d8ohmc7TWkJFAd008PInMB9i6cPGd8M+o2R2uKCMrEYrRI3E97zKBgFmCOFIQpJQLU+EVzy9abfPQTO/", - "0sRQNNdj920IJpQD5kwTCDtCU4iygkbwWk/YkcZI7it1/mynuXCv9aRNOUIMgJx6f7YDxBB1noZWPeXe", - "yBEGcEFp6+u4xmO88KVT7U9G7QTAuNE0DZRtDXfdkOxodibmFk38H7eePCJ4UKDRtxp9MLmwwkheBO+v", - "EiJHqQ3qOslRm3EFn/3Xk52dnhdj76NiPuUKfgc1zxtEtbdipHOJx+SNVHLGi3B7HTVutvVfz3bYaO5I", - "Nw6fPsJVwEjJVyCDA5H0GEWcnIl5ADx1OsRC1wrv1n/NZGY0nd9HNGmJl/taUWRlwSG6FJrNeIEekpks", - "CjgLqZ6MIwr4uM/FlJ9LbWhIR62KxlasvoXwpD32GtSNHlwMxKBOtlUaFLUsrzBRB0E24ZNvetgcmxie", - "Cb9KUudMUJhsggQK2bHMCGohR50IVY88KiRRCbGPGHULu3ExFcUsJKhgSHcg1mb0uafABH4VUxcnkxDq", - "EUgmhJvTKTl4++pHhjkOvhPfSGomOFechmP256dfx2ZO4V7/nBd/frZjQyuwByJPcX1rpQp81X/++ilL", - "mvtm5rfV8eKU3v/zk2fPvm1ccr0BgNMwtCVj5PFjug2F86cXbj+JCBKmaJNg26Z5Ao4ByGLDBGfN6Zp2", - "0UtomzKVF5D/FGcM0TvcnFEgX9RI41YHxWCmz2ng4fgIRyNymg1VBTd5PciUSI5PabTn4hDg7zQyjUMj", - "zqWubDFvWXi846RTMuVqAjC3tIbNFfK998NPQwzFu6jlOKflSbXstsV5WY/WiIwXWVWAoEWrsN6ixGeK", - "m/1azqRLnRj+XOvsDHKMpkbHu7CglT1+jEF9b08OwfMyVxnL/Bc2hKWHfAPINQlNIn0Ti4EY76VVC2LC", - "U4pU7L8a3IMaCGeA8qkgZYDuXBvl/SA8/eR1XL/FNQs3sBiLgG3VdgIoT76JoAMGBVCb8M+3XOk0kwMa", - "Wp/HAYPdJIvjhWYHIH3gYhtzGGIWAZlYJMM9XYU5NROe8nBzzDJeYppNWAicdUgC+co2ckAywUbCXQih", - "WPTmBT8pOG5aMzxIiOPtSsIS0dHmh4mZ3zhsSwBXNmSJs5kwEwHE1B2oEc/OqnLbCBDp3eALkucyr3iR", - "CLuY+oHZoD82UgNejscyk0I5iAj23UJhIoR0WooiAorgzKI8pbxhjH2CkQ9UzMXCyCdye9KE4gXDmZhT", - "dFrJpbGPameTsGwrxgWANhJG/0J6m2gE0NVJ+ez2zEeAtqYAAUjNojiStNh5dAYlcLV4sz9QT3vlFMFr", - "oKL61tPDfS/9tNOZLv4/3r68uXEb2/eroHrm1pUcLV66M3lKpe51t93dnngby5nMVJSSIBGWGFMEA1C2", - "FZffZ3+FswAgRdnuZPL+SdoiifXg4Ky/0xPgxvB5+v5atoXMbVg1zAljj8Yor9sBoQ63FWkJwK0iiSa4", - "WdZ9w5yy1xNnFE0f8mfBKams+Hz6IZKzIS9IZRC0HNfmhQFSufaoYi9KCNXSODOIzItK75EkQcFxWMD9", - "EA2DWca/cnhKp9ItLmqlh1H+ticu49YlZnCjTUgyn8W7pNTCKKuzu6j7dz1xxcUAM60LL/BgI+Sr9aFC", - "4BAmWTseG+XR4Zp/xLxPVpB3drxLz7G9c+1DUw4vTzzrF13xy4rihUo+GD41jtoB+uR0QxbJ8GKAPEXM", - "JiTq0wYrl4pSz1W5QOuTu6kxSmmagdvpw+YscBUXMk8ylUCRdxMvEV07ngW4Vv7v/u7S8g0URdngISpr", - "525nJ9JLKsI2fYCWLDsQCEYh3oXGweSwpe3aOCAfAN0YsO8rkBC4zegGvNfGll3IyG1tTJdUhE8raWRe", - "Kr+t0RYQucKKs/2hhcwkLddt9AZtLLNILfcEl1crV/dkJwRot2WdCjHBIXEqbW1SohUE+irhgpdU3iia", - "hVf8Y0HJsTOjUKbBI+upiiiKjDIrq4xjZzdppsRX4E/DWw2HBj6ILsXpEeuGwF6wCPEixdT4fpXdkm/I", - "0kkA3pplXW26uXayylxYtZQ5q6wgVsG1dgbXWuvIcbLhOp/BFIdesuKsblWwb7iS3K0eKJAXOaFemZmy", - "IktvlRgudJHerDuj/FLbcm7cVTo8gEApmWPool8p+rInDjOrxW2u73Mh7YBarYp5nVHufnZ3Bd7G6JyB", - "RO+MflFdSJwxM3wFuPh7dxdiPzAnmg7b2Wg803UIz71V6x7r/m4PjFLde7lGSWAwynl3HMXRjncEhr5Y", - "gbGUSXjAN6ynjSnY9cBSTdPfftmgau1Xm6YBW1DIOWcQtMIkbpU7Mfs9MVTu/sHfC0hx1VWJJjDHAx+Z", - "CU/sQKwKnCEbITcmQ+83zamAuPe37mYolCyBneVOXINRdEVOkjkEXgYGzpKde8s2HTeKNSCa6p+th/84", - "Bc//8Pj0+MO12BEfry7OOMjeiouro+Mr8f7fIk3E6cnZyTUAcIiLjx+Hx9didzLKhegC6ceB+Ufv3ULR", - "9NzQF+upSRPvL3bfXK1yUvA5n2qhVyZb9xOZZus2en2lPzE+BQcPhru3YNyQ4C2TZZr3ZZH293f333Z3", - "9/o8gd4vVuf/A2G036WJY7v7X2dOJfpu/11l9BHRM6oZpY8aC5V7mU7wG5rbVM300ifDIg4b9ozOTT/q", - "AwE84lTeKky8B5ICsxQEI4nWxJ3L/u7u7h6MedIR/pd9/qXX67WxfxSIkW0hxcBpv5dGLbS7v9UDcDR8", - "+xIVXOoqzQUsSEdYBWAIJPEAaU3XRHs4cDfqqUlnt7aBShKVlXKMzNYTipPeA6VU1rg2RErH9pSCbxKa", - "cKbvu3yXk54DeQPhc0xtcyT+nvOsWdgZulORKWtJ2MFNtI57gu2MUv277voqJa+gY6w4d9dGuORgw+SN", - "QuENYE/TG9RUzaog61NNGBmWRsllBLvRcUfWyDxxHaHRGO2POIUfD6/OT84/4XhLuC4pHDGfrQyk7iNb", - "Qap0S5FRyoZxrNJpSiS+YBw5sgcn3ek8Wwc9ymlewMGvUPsC9zNmv6FaBhtyJ02qV9bbMAmehiJwHBkN", - "+v1+IctFv9R9/BBiDzWAQTgyI7NaV0zswaDfn65mt6qET9yLh0v5m87F8MD1T0OJ9UoOmScagf3BboTN", - "ZWEX2odEiu8dxbG5d5QfQaR8pKiRLgnrSTh7FEHCGmdQEvEGS41Y5emvK7zFYjNAvfzOphUAzvlF7Y3t", - "UA5bVCcQCG/dUcWzUdH9rT9MsC7MzQGJDdnjy7p/TQHEaNj4aqv0OA10Q1p7sDI0WAuOUUH0PmSS5Cr4", - "Cze1aOkOIcVthnl2Rnm4QgDEQDrJnPLuwPbgpBM4GFdqlhYGqPBK5rfi4wq8Sq2rq49eHUdxoBJGDqHb", - "EEs+pCsKXPXE5dF8btd5KR8wlVjfK3OzykQ0/jSfw+kAJ3UV1CB10sBkqpP1wDGEVakMWAI5Xhl3QRv3", - "1uH5kWP9F1fuv+cX1/DilTviUVNrJc0AAbH2d/d3Ef5kYZxQF14avcEgrQIejN54+++QZFk/13OJsEGZ", - "zOcrGXqindrYlwGaViGxL4jpPgzCRxUxgAdMgbCxWJCGK5+arJo6hxzb460RoUEE+OOJfEaSwJHgHo9y", - "inmztYA17pcvE5xaM7k05GK4xsausTHJMOguA8rgwNpoe0M0DfcbfYeAEu6ykGhuyLJ0DrHGHH3A6xZl", - "eoxd++Pc6XOVzA6KWyWvEUz1GPNb0Sx+heZFxOmm28OT7433NvmDAL+ISwhTpLwRhDWLmOT9QltgjZDX", - "bEq2/XuixxCAWloOdkZhkbgOTpkbgIcR5sI0Ai4jaB50IXpvb/8Afbzw19t3X4/e1EYNB3uUHxZFBlD1", - "leOLFYYOz4/cnYqBuc0D9Hta3d+vKKliPSjVbJHrTM/X4qvaSRy9eeLJBHcGKEmYwwGMqoR8y5TdcpCG", - "gBOAH7zj1O8jT+wY4++ineBgz43Zts4vrsNE27WZKm5522T9XBNVGAUuNnFxRclhA2B3d+BUjwMzYkfv", - "uS4VG4koAwUdxwhqyFZuO9OebI3Mb9N83iGIGfczziUCbmAWfnX4SbSu8C6XWfdwNXfroRLxyUOjtTEO", - "uXYR5Qmjp6kQ/U3M4PT0DAWcTQ5kuCtuCV4DGc/6syWtwOqA3aETwI7RzdkaDo/bMQdcSnObOB099A/n", - "J83d9StmwV3jA3hDH3Dhhyh0uFB5TfwYD+eA1zDKTzxzKWktjV6BrQVPrA+exIcBVw6W4UMmrXUs3Yb1", - "swLoxGKMgJM4OGjbncYur78TRooS3eDIbxtFgXhZQlS+hzcVLXfZeqmpLeL0gJMjek4Z5mj/4T2Z0djR", - "BtVhD4ft8EA6LMy4ZaUVPPGER1fKFYMbNVnqGeUJ2IETFjEDEPGRAjyS60D8uoJwZ89m93r1y4tDtYMk", - "8pW/qNsY8QbMY1K/iSh0t3bPTLyNq+ckfaRPo1C0I7ehY7jbpSU3manM4DSzfyhiN/GN6gUDz7r2Xa90", - "oD1DJnusymc6UYZwu1GngiwrT+SYo8hxC12/5R5rqueG7ppntrIEiXQ2Wxn0Sst8Q2iIUFoghZWM3EvI", - "kyWPc9eHQPJQyOpY6kK8g2iQdmO6eOMtP8c8kkwW4FTmyx3MD+6NvV36xdBcKmGBRGeGJAbUkEPUIUwK", - "ny3TB5VM3QS6Mu0vH6Yy7WKT4BPq3u01RytWkkX9CzOZJ5BYPIbwAhzo7kYcntvlA6YtcWlWOWw13cR6", - "VYIGT3QfdnYtElUSQCBu+lwWoLkQHIFYpnm6XC3BYmgXOkuaAbsaV3wJAVZKZEqaPOTCu8Vc5bX1Xab5", - "GEYwBpbnnu323oX1lQ/0eC6LcaHMjKI0D3Z7m6vRFZNae5OB+F6pAmUZnr6/7iAY15biX/8l9A06qn0d", - "LGytqfvJQAzde4gpi0Gzs1WZ3qmwlkI9QFZ1pWmw7eIRc5oE79/bHhmwh6U7N3O41z4gJMpC32+ToE3E", - "Txj7BA0DxtxMRItCmduDbdpYlxlL0kFDrbhX4HkTS21LgCXF4EJjbyaulUzCJIcwBd+Ih3nAe/1epfMF", - "LbKyAfMwW/N837Ep2MvLyFgnseQ34eymSFqF8JHCKOIPNeklklhGOTDpSGKha4ZB8qaUSgDakaHfy0ii", - "U9KQuWZnp5IVhI4yyucJV467AfmqpY0ZgNF7GCTzivVVtMK2fuU3Fc3dyKhZXiu1yKUx+h5tCBpjXg4w", - "nZJ5FkSYe9q1YLPGZvDMIYNfuitRr8osVcbG9hRY8ucNKgwh+KJF5XmDx7PqYDB2KFtbLjQrXH0ESq1C", - "Zdavetrkl40vTnxFG1xFZGu0pZBxLgKiJe0Q87leb0XBJpzouNQJAEVU0l4R68xbReqJNuW99qn9mMYD", - "wSJeoglyiU+Fj+wp2Poof3+2/44gCOojJyEckn57IQMV7AQI8YjxMXVUy2pqJZ2cD7VYoYaQVx+v7gfu", - "b4s4gwZyq9LkwT9cquXYMXWPIFGNTr9AtABGouK3J6LF0KHtgTi5AXyRDqP64oKmUJNNmzVeGq2VVaCk", - "ayNKBfFd7apznzpxi8rak5DZXJu0XCy326JECxMVgzGq3WiNEq1Na1R7wxwlWhvmqPamPUq0Nu1RbZ8S", - "g1v733YzzTrsMKdMktIAmC/2hZS/TZgNT64h7zxKRh+GQ0MEe1gURj+kS3cEb7u5ksbx5NxdNVNYosPv", - "z8/bjEgIYsyGVFwl8Lt63ntvU3uubfBnaZJ7aVRXzmYqIy0pSW0JL3M0YBQYNDw5O4I8GLOaeV3y4Zuv", - "+4dnRwNx+M9/7aMT95//6r7b28f7gxzEyFZokDEWa1ccXp0NxPnxxTl8PDw7Fq3hTGYU9Vaa9CEA97WF", - "HysYohzRvE/LfzhNKC855hAt8slqphIm/Ruty8JAGBYg1kCkYcgzPdfi0+UPccSK9tHjrrUPlz8Qf0lU", - "kel1FM7ckHv3AmcIe9TAGvzDCnd4Vrzmj56V9HWWyaVslvRllnWdfJwto+crQ08XZVkM+v3MyVsLbcvB", - "3t7bg7eUiVTlUsjsz3RCNOZ+w8DFhK8XSEFnnzkjcHY4Tw3OLt0R+j63kboVrAg+y8Zygi6CskFOOwN/", - "TTbf4lVC27eCNP+lzFeApBMfXIp9wXIvMvK57uwcs7e/eToA7wST+DekfoSiEa+YTgB+8FPqRBNCOJvK", - "JI5UDj7Z6pgoKIVbStIlHh74BFkSStlunqXKA3QVJ8+hYEbqcXVtbjhZLuws7jpCOvAFhcMfEPRBqXHm", - "xEf+Prw4v5Tlgn0fnBhVI+4J/OQrlpFgwb//tbeyyvSmTsci0EO/VAPxGUK8ptLJm/Qrsm5M6SqMXhZl", - "bRPC2g7EMf2zGh4MTfBHnukSGItf5oG4KBigAUJHc+ulq29rKB4AUMZ9pTdCL92WJNVLHvQzRsuu3ec1", - "smzaixp5boz2irnexmjRz1pIYxm6DIO48Seevq8p4+mnQjK1ESLNEpLDH+GUeHTHMOAKs+T5MlYcZ0vy", - "lEHd/uZtlXX58Q1xbv+5AeJivWaE+Cb/WB3fjyYFkjtDhnXsO6ZdPnuGkbmzDrIfhiDTTtfpIt503CTe", - "XwB6A4xe8P+N41kPxGPjXgzET7u9vY7Y7e27/xz8LJ4mCKpQIR0wl+lblXdRz3YS90u9xAuKD/4G5qTe", - "u44YvXm77/7Y7+2KJ+rSY0hSEDeI/Zi18SXzcR9QixfulAKyX/TtTxh9iZcFSK0/TxDUL9ddXXwr3Ag0", - "Rj7yULDEACSg4YqQ1uTZzyVd4iBsVK5SSuUHxDtSpqg8jOOPeN3DvwqVy5SzM6cqMXp2OxnlMa8vVqbQ", - "NsLMnwAf4VGgcDPpiYtygWGSOCbKhhjlU6MhtnNS+wJ9M77+B+ZsuQGbGzlT7PEzqxzAemYyH+UUBA42", - "VG/3JYkgQk+qH8lnzJoVUef94eFJfzpXXbt0Uo/Ku3d7vXdOmKnaIC9g/UQLI0+wxlz7pW4rMtbzEtar", - "5Cse0qUuoGgeWpwHYhKaczsM3tsuEGKXkffQXIu/QaWrMLNC5YcnL04FqKZpKgD356m+e4Ar6d+URTq+", - "VeTvs7fdXq8XT+WMp9DczES09t4dfJ20Ow1v4DxE62D3b/tNb8hEdnd3930bnnp+HIr3SPcvTZuOR+MW", - "QmhRr0xLmUeLHczgozcIVcY+6K6StuzuNc5+e2Nu87Y+3XdPZ3qhjOrh7yqfZ6lddO8ONh6BTSZL8/lK", - "Zu45F8IJppoTj7IyyifhRoGf+QR7jkXAvVJMyMH4mzKTnvjohOR73bUlhLhFhiCGYekE16iEyBYIjQqC", - "HgI1yyBwkojpM5YIzUViUCEhmqC3O+aHci6dbooiO+Gxn9wgg4oFcU4DBIEc8+vdvU83UodgTbdcj6Mc", - "bMKYOrNQy2aRx2O+kejZfQ+64yVIncDKEVog/Q0t4lXAjJnO7xQEcjLEDy1F0B5Qjm4Qcpu9Gs8KLHW5", - "6HUqJ3eIb1xidOpAPD6CDejpaTTKP7C3Xzw+sucfHhyF9tyzqPmnpz9Dp23QVHlnxBACCkOIJ2gkOztN", - "GoRbcl+wbOEf/2KhHNx8lSaqTwmGlJIP6YS8Mw0Z7jMJqcUYERwlOiNKEenEgwCXwK07TeihrLZu1Eyl", - "d3gwqjnk0qJR9qGMLRY4U5r45PHR4649PU0G4gQz39DGRYjc8NpfUkKnfXrq9XqPj/30Bj74wEEhMhOZ", - "nqczfh/xDJz0yF+4X7CTktgCyJb8wSrPlLUhyoQ/w99pdHfKQDKa75S+/l84pU9Pjhs+Pv7vrVr7fwNo", - "sf8rk/DHQJxqXSC2HQc97Oy8X6VZ2U1z8VllhTIc477XEzs7dmZW08/lMtvZEV1xhT4IJOC+LdcQPjG3", - "hGZWGjkrMRObSjQ4pe/z9dkpkOxkMgl0BL88Pvr2xaJcZmPSep+e+AP4P3dsuc4tDgDFTnI10QM3JP4d", - "SuLg94dJAkkSGcSBYNrXFDJ4VYYgKKJVdESS3nXEYq+7+LojsrQjVDnDWHERQiyKTKY0PYwDMwrS7xKA", - "e/al2sAVtLOjfoWFO2Yfbohl5shk3lK7dY0cEbbUr4wqPXqDGeCjN+2np0P4JxFm7X1HD6BnQZIavM5w", - "okDT8VfEKg7cmCGJGob9SeXfgxxfkgkBHpEFxR0/KCa44Q3ZOhP8fGWy7wBE9EiW8oerEz/w8LhcpLYH", - "74xXJmt4AQVJx5oI8By4EnzR+6WYY1DVxjcQRM3ZJhxNjR8V+baP6oHam53Q4gnhWKovD/HD1SkillLW", - "CFARpCMMAGlVWvX1W4GBHFhjRvxwdeKjJfBN6Kz/S6Hm307hg06v15swTU5Imp6IPv7bwh9d8aOauv5t", - "PUTJowGuC8VBBJiNIUJIelMMOtQG4NOEwegAenwAuDqyBDhEDnRvoXNpQC/WF/xWrbHGBixYgEn9QIO7", - "9EgCYd12dk4AdhbwTvV9jsWvO8IoC2brVnpDqQDtTlWk8AvrW7o8+mjxNnkomWth0h8ivUMKh1F5AkEs", - "0iLirf/c8TP3+RUcd8cBq+2gtHKmf0uzTNJbnikgjVB9WZix0ZmnD54Z3p6F0W6DMJuKd45r03KohxOB", - "kMVRjROnVIn7RVqqLLUlPbw06Z27e04uke3B5e5BRIbDq49ClqWc3VomLR4KAg1CuI2NLum93d2z9/zu", - "5+vrS5H4sYN6q1dlgLLh/MwA0fat+E0ZD0hPPl4lE8egqVXYcOHltc0x7O++/aZ46GCdWyIEpquhUgPR", - "XG61XzFy/oUXtNudaTQ8vAXGDefyWuscGTj86SshXl9cnHugq2uw6FyYFKMnCVL3nLCC2lu5YegCih33", - "EDLS86DmxyJT+bxcnElzq8x3CJ7tpIC8/O7tS58mCtZQme9Gb0ajspF/wcw+ErZDiNPb2TnY7X69+19o", - "vkJ/Ezp/6ErDw/b34cU5E+QHqotDZm88FpMBOhVrCABMw5D5jzNEsBII53cX+U9/Ofh5IGTaIUf8MmPZ", - "4JqK7NN2uNZXeRqAFnwhL+7louLsOj09E4U0ljRDIcie7Q/WJF7xiWhNtc7aAwBr/QtD0TqVDsaOyOcR", - "kTptyrNr3KgJwKe3B6CRUm1gYQs5g1MZyNt/5vdtIloYFd5mXwMBOWmD/htaChQxwfktJrTXE37D8hkh", - "o687qsWqHFRELvRti5NaSWp28foURIEF9wbi7zJX4kjj4W3esA1a86CE0QUJ9DesFKX25zgqR11qnXdx", - "x+Hf9PUnLU7cnHzN5caPZVbcKnub5v25xo9H+Y8EQSe9cgmVcLy3DYk1BRtu4n1I4KUqEUssNZEbKbWC", - "cbJU0qsgYx/jTeFWFG5n9g9xdWPu0W3sUoHnlk2edVcSAhZjc/mcQPRR6qaydgFC3J+CL9eZI+XY+6de", - "dsxuU1+3a641rdXPlgjVu3h83Q4oBaALwrbxBeEmmN5A3jLhC/cRyjncF4VRhXJ30uSvvYlIsdogJRlz", - "ZcO/9hrccSIysYeto/ehTN9Puz/30C8y8TwtjYxP11Cmh8UFJCzOAZeU9tFj5Gry8ZG2gKUEIMXNB/Bg", - "RqZP46qW3wuVtCuePV5Rn82UAicAicibxn4/lfjV+tOtG+e6VIPYwedOHrkVMaAxzTIEqKw7ImO7MsVR", - "tw6+eRuqYlRXAw8rg2SyPegPGJ/23h183Whe8lalP2Yuim3ar7RqNwYoX4YbAK3pYqjKVfEfmnpENVSB", - "50+doX+3Yrn/6+PF5fH54cn48PJk/P3xv58al6IRYH5nh2J9g6B6n1qVAXzXZ0Qz94+sD3fEMPvZGgLq", - "PbQ3KVB9iikD9HNCaYEq69781VCgNA5trfEAtDkvlXQM4maVUUAAtYyq3p0VHzK9AuQycgehg/xOZbpw", - "V0i/cErEbN0RM/di5BiDbE5PJX07kwwXRkJKfJ1DYnctlAVisYO91yNM3XA8hbzTaSLuFzpTIZOhAkQZ", - "sk/w9nftHMaQ+UMfunYNCSb4WgVWv79ZyLOKtE9BPns9cYQZgTFs3yaEPKjZjVihz2IZ01L++WjG7Lz/", - "EwplN9ag2FZt+RV4xftcMikKeYU7tXF5o9TPCHQ9RKFuqeEXBbFuH8kBx7/DpjtloEtKABcALWeLxlE1", - "p78y73Ifj+Hj2viijMJMFpUag9WKND4rtLEK4U+83z9vmxsIn02FEICWuSBLKNqRsPZjV9O4/CdmaXKV", - "hTQvVuUolyA5GyxXj440stx1eGJckAH/LnWBpJTJwnF1tMb6ND3vKDsiBIsfXdcLWRQqt8hQ1nrlAQIN", - "fvA/7FxHZdzy74hjir4tABsTKjfpbOHOaKeO3EawClFuNgMlhpoGPfFZGfXfbiSyRADQQs1KHFjoFIGE", - "RItKKjD7y9U9ufkAbIOwMSt2i7YPGj/kr87VPUVElVocM5rlNYI3+mD0hplXp7dtHQLzc6ov1lCNDuBU", - "2sUov7wYXos+Jib2H+H/4GXp00r1H+Ef8NvrBIeKINDr9TYZw7Efot//yozI6+NXpOp/xAlisazqHjLO", - "aBRXFF4OUKN1mN4ICHULTC/khQL8MNXyQMO+u6MROde1MTfKWs9yzmoPqqsO+D2vXXRaulOtb4UsxQSg", - "wcY4BqqBExaU+wOwZXgzZFYbow06DrmU5CGQTNgPX/KpKRJUVHuCupqd+vlUrhOLUMaAVzxgmNrSrOug", - "yh9lmqkkarS5CElpUnZDqgfEhHeMym28vrl5ZjsDpGBtQxn+GttGrNkwM6cNpbnPM4KooihX6U4BFI7V", - "SwWLQOGmHkMZphXBEDZQTmuySQH13X+aILA3rCjZqRCjVy9VuVAryxDMsCpW1TdnBZlvBGNPS11H840x", - "nO5SWb+swUzvoxqSNueZLJc652mK4Uzl0qS64ocMatjh5Yn4IfdhVt5wFC04bUNt44H1WmXu0plivD7r", - "rd21eXAFvRaXJgicqd2YM0QNgdbuTsEYbBKDmBg5AY6l+5hOuSYIID5CRWm6ww9FhH1JUrYK0IMGArtt", - "KJXb8+RC/h0PKS+pRS8/LyRDTGLCeAkTLylLCpDBtSnFxL8zNqqQqRlz4sBEWE2NhnyXmXQXnHuPjPom", - "vVMJYqz1vF/0yh0dgFWGwAG2/3pfEB1GSvwrzdoblbGiFJmTIYyM9RWvJ4CbHd0FbPI1QclyBPTrSpck", - "tgfW5LVLdoRCABPyOSh5CaUoUKDpxH6Rtp/AaQRsHiXZI+J2fqdsmc5l5HK7Ikq1pS5om5byQcjSqXpl", - "nTp9DXp3VfnCqECdFZYLtV9YakCJYAOg/FoTPiDHV8YvYPLjkdEF1ZSmmxMvfDE5Oj49vj5+keHAXl8p", - "lBxxUZxkg10NxKRBWGhq5KAX3d3ggLEbPL3LNzOyFDAL7exE1WRZUmtFOXZtFoIqyOE6D8GmUNs27mqU", - "M6gWshPU/3AlKYteJqrHhvLQaxQv1abtqsPabgqRoeIVIEe9RyZxJn/RJhKB3VtOnoaIQVghq0rbYbwp", - "Yi1B3hu8Rl7Dj72Ihn9SLanRG2yzS212qcZLyM6n2qikQx4M+v3lmry/BJDmwQXr0ajik5EF5TpSUvJx", - "MocgIfcCPg3rCjVVYUnhQAD8aapzu0gLNOpxMF1ppGP4AX2kJ07T2wgHpiOgLaj4Zkc5V4trLI7GMnKh", - "DJbCxgIF8RkKtWbDfMTFnbt/1P3mTBAPDP0KToZxg/CprtEYYZ4M65IS0ilhZuUJga66keSKMNAQls+3", - "AUVBcC28ubi70AV7+LsREFK8mgjr5iT1KKxqHubQXBPFS/bwZlNyqJ6lMhvT47rtkXf+uWpv0Qa5t+OS", - "BMrEI2QXZFw7YvKCN2TbsI2aqbzEahMMiByVvRpvlrp6sTraMWz5F1ZG85yqUiLtj1Y2kzM8BiAiyAhJ", - "/FZVC8a03bHCcAIuhX/vpAqsYBRXSUOC3qyQtnEs66rVc6XRqNEvLIvmierOViol4s7Ev+zsiFYZqie0", - "EdHHy68g4oIrIarXUSmaQBdEXMPjxSILm/XtoEg/5XJBKQw0m1IqD83GjRW58faxVknf9RwfD2Rf8Vix", - "Hr2mdYaByjIglEUf+4ED2WxSpWhR4keaC6AfRAfuiiNGmIhbUyhGQkVQD0EhKkW+nikG5vcXUl88lrIH", - "j36hHpiQNshu19en/z8KfD1bzQsuCvs7+SvGCo6bqliON6t/BSZ17c61wK8RL5yb+JbIocKpqPoXFI+G", - "jxjICHDq3Uoe0q/io1LJ75zMH+G6AUjGUUQmbQmV1NyQYXpxo98KnTklg/hLU52ujbpkVwqwj3OsVfd7", - "t6s0Cm7xcXSDPzNFKGxW3zRuw8eMRW19K4y6McouBCBEuzs9I8wgnSWVneXtizW130uERTp2q/el5OdU", - "NfgOrnK3Tc8RHzjFspSctXmi7yNNEFXqVxVT8/yD784qHwm58gyj2sAxaiXZqNwZ3fQACAkl0pBdVu5X", - "W71gKV/fN1+vv+brQLp2W3ttWClolepQGAXxhFUB4VVl2aCVABD95bXVquJYXNaNQalDdbUNOWB7hTUS", - "0L+8uhpsHxdHqm/a9nJr24uqIfSCT7Z5roAakq3bHBBh2s210jzpNRdMi8r2IVx5leiaCqepOjH/oqd8", - "yY6dSj4uy+y7gy+tn4bTwdppe/t/C8XTvvk/G7XT3r092Pei7xnZGEFuVEldRofL9hqwtuKDngibutNK", - "VTWmaua2kuLg3b45ds4z9HwFKSh3Q7+zlaF7+2e9mFusam6t4VbXerriA8KLgykCzqMVidEILY4mLF+K", - "2jtOgsxPtW6iVAIUptE0QU3hEb/MZC7kbAZFSOZUGCAqGrAB2NFQzw3sOtGo2ATExqSqlOhLvZFGOtNL", - "oP+lzMnctL1823M9gSlosytv6VHNS/NMSbZLr4xvVDkDjSaWdZ1ik5ZW6PsctpeiqQmEPplzvbRQC45K", - "ujVJuKIFSc58GMM3lMF0rsFQoLykPohWFCpqxUOzC0kZf8CvYqmXQPXhNSapWtW5DzJ3g1kCRSMryRsp", - "95kydhjdTnR8fn0JROaBIeMa8Y217MhmtREs4yRa5F8V9K2KgG9f7c/3JDiuWCm2Ove9hSGupexFSpZZ", - "X/o8dr5vETjr7cvMEcSYa2P/gfb3FtsDE5zu4MN+qqpU1Rkelju2lDjVNd3YEIjtcFduZYl7FQ+VZ7uZ", - "nrMp40eIAICaJel8EXPcqDgQfQf8JQFRJSZ1kd74lrlAiBRTXZaZytXslrRbdAbYhTZQyatWAJFQbdKl", - "6lqV25SVGUZHRAne+pTTwEhpIv9Uxh2jumFpSuKGj1hA735AeYJGeTQxkCFwPqMo0BQRGSsXUlccg5ex", - "wTBCuIn1K9hgKHIMCAg7/3ydRQIvernO4suAgGMPCBiM3Ex8COOXzpr8LG7bVlkiTtyWVEEBySgv52QB", - "Rqcp2IXcix9kTtUbmcaDvYzKVnkjVCMe4EVcJAMLZpUaA1TCKv5glWlYOfj5tU06LpWi3hwavvQ/NjQf", - "PXxtJ05NcgJ4ZcOpr8Mi/V6tG/pxH7knr+3E6Hu6E30KEmFchs6u9D3qBw39Xel7Uh5e3eUHaadpLoym", - "cKO50VAoeDlVxnGiuONMNfaZqYbehuk8B1OULWWGYB1QTEDOSkulv3w9nV60bXJ26yiyadP4UUPRTyws", - "MvzHaRcTZqlblWC1c8ptulGz9SxTUXcn/F7j6eU2PFicffP0s3uxlPNPbp3c4X30VABzedN5U8o5nGq6", - "sqs8oqE862aplgaw0Q12AyeOu16Vi6hjOlSVI+BpNNBP3IIHQgnNxI+jFQjdRHsV1vHnp5+f/l8AAAD/", - "/w==", + "7L37ciS3sTf4KoheR8yMTneTMyPJNhWO/SgOJdGam8kZ67PN2Sa6Ct0NsQooAagmW4rZ2L/2ATb2Cb8n", + "2UAmgEJVo/pCcsb2Hp+IYw2rcUciM5HI/OVvg0yWlRRMGD04+m1QUUVLZpiCv35kq7P8LTWLt/6z/Zoz", + "nSleGS7F4GjwbsHI8dszcs1W5OzFeDAccPu5omYxGA4ELdngaHBtGxoMB4r9UnPF8sGRUTUbDnS2YCW1", + "jbJbWlaFLUu/zV6w7+Y/8D9fvyxfy7e/nOt3g+HArCr7qzaKi/ng48fh4KKe/swys2V8J1RPuSAaCxM7", + "oCHRdbYgVBMlC3ZkmKDCTBSjOVNEKjJXsq6OmJj3zMa1teN81vtIzuadokLTzI76LLctJDo2TZkJ37ye", + "6x2810y9piXbYT9rzZTtsmf+tWtpxwX4WS5ELlli1h9tA7qSQjOgt+M5E+YdKyupqOLF6r2gS8oLOrXt", + "WFIVhglj/0mrquAZtWM++Fnbgf8W9S0FezMbHP3jt8HvFJsNjgb/20FD5gdYTh/8pWZqle7tVCmpBh+H", + "mxs4EzOmmMjYCa1oxs3KVfsAM2svK0yN/GK7JDmrmMiZyDjTlt64b4dkriFCFSOmGRqpm7HZTVkAHcGa", + "nTOjVqPjWXIzX3HBy7okyhYiOSvoitjDwDIpcj3euIEl1h0cPQ0bx4Vhc6bszn0cDr6l2XVdvWKG5tTQ", + "u+7VpgXu7cEt9PoyYw2SybrIiZCGMG3otOB6QcyCET9bUro2Ca3NQiq75FM2k4oRbagyXMyJ5jkjbDZj", + "mdF3WXNRl1PLT2Z+uUlWcDs9ohcwvBvKje8VNogjx7njlgwH/3N0LMysWI38io1eSzN6iTxnbZxvFdOW", + "Jm+4WZAlLWpGrmyHV0SKYkVuFkyEYRE6p1xoA4uY1UrZimENC+ScJV1ZxpoxlttpRKxA2DH/Y2BbH3xI", + "sAEgpvyc/VIzbR6MejYQSQ6UYDv7OBycSDErePYZOnYzJJnrUePix4uqmJa1yoAODbPDe8GEZuesolxZ", + "6q4U07pWD3fIetrvncRPihum1zhUwUtuWE5uFrxghJLcNjviIme3RLFpzYucZNRkC6ZJXe1zoC7q+Zxp", + "23aXgaVOzx4M7AfOFFXZYnVSKy3VhaEPyLtSjfeu6QtWKZZRO0cvEElo006PcsFyMrPKCRNM2ZIjZCfE", + "0YstTg2f8oKb1TdO0jCRV5JbniPkjRXrBIReoPiPw8GZ3aFXtYE53vEUUrHaQd6+F7quKqkMy6FTKzVx", + "uLuJ2w3S9UwsacFzguRmjxef1wqGawWspcaqkKvSrlcWuiUl16UlybV1eHCWkGy9lxysDoZTKV2VLsvw", + "7AK4xBCUCKMtDZTcEFmbTJYskoJTRioll0x0zl2QGOd0ZkZ+fKM32MBuMqMW10LeiNHyaSw5LE8Lg7eS", + "gRZW+V2RBV0yN1BL7lTkpKy18cOcFlzkxQqONO8TJE2XfeKktd4XTC15xu6qoexG3Vt6fAASt/QAZI6L", + "ynWfhuhIntsVmdbAVGRRWDFeV3NFc0YWVMOCrxgIoyVTc1zrT6PkBIqYUV7UitmxA8/2+uweTDso3XBh", + "sn157fsBD2tar09typrm3r8t/wY6JC6wsbe/wtIwUzj1T64Z+U6Jhl4Jc8dl8IqZhcxfS3NcFPKG5Z9+", + "KNgjHA+KfYLcNQsgWpS2dmSvpflO1iL/HGqjk/F2TDPo8+Nw8JauCknzd1K+pGrOPp/2OpX5irBbq+dr", + "1F2dvGU50fxXhqqgHSIoG9/S/Htq2A19uAMK7b7wt+j+43ns1aByyvLcMkCpiGKKimv7hxWJ3N5cFDO1", + "EiCNSC1qDUzUK2JhIg+uFLRa3agMwDQeaVJxYYe58LqlnY+B0RpZyULOVyRbUDEPerhlvlQTVQsBwtFN", + "xW3IO14yWZvPvi9/wV25ZVndqGgd28iKGF6y3CozGxnnPTgdDOO+7G43W9Nu4v+d3coLIxWds/fC6kuR", + "6vChdyGdVB0TV5XANMGo4X7SoPtTopFYhBSjIH+R15IK2ck40Min4i97Ekk4ulxUtWnxnXB+pfJyo816", + "zqlhL/Fe+q/GeyI6t3KFGuZv0J+U2h/AuHo/k2r/QkUrAitFc1JRs9ik59rLgzfrWcnENdF0ZpkhKkP/", + "JhoXrNp7USmZMa0ffkNaTW+XM3YZ8To9re11WbirGfJrlhN2SzMDW2EPn92AnGUFtfLfDlpRtCzc89p1", + "Jx3F9blJC7cjs7fmgmkdPbp8h5zywYc4HBh2aw6qgvJO7cRrSI9SHL37NMz+bOZo39mLNF1ZtkjRSOtt", + "AFxfCnddHrbOypY7uS2q/TJdCl/r1fuLd+T1m3eWHvwVnbwH2RIPUjOt7X/h4kdd05cCLvpTBofU3fFR", + "BE1XrepnL8aXAjcKBJpXgE9vF7TWD8nN+zroJzAnYv1XqdZNoSwM8xOxctAT8KnjwZXSRNvbXlvOXpBM", + "FgW3Wz4kGTW0kPMRFxlVwlmvQCkFOxUVgTZxBaw0VJbXZrzgWDxmr6CQfIa7/loX/16X/vtJoMg0+4rl", + "nL6DQp+aWUe9NmzJdm/pxHVNmMhk7t+wXZPhrfgFy4DqwHVByYopw/EpmQp9k1rk95pZGrQ/DonOaEFR", + "fTSqzgxcYVEPAYUNjVYwMEuW1xxv3rQCY2qebB2tBXAlVuWBYkvObog2rNKWG1pOYeWlojNkwVNGaJax", + "CviuHnHdPJNPpSwYBT7ohzDhefrFPoxxyuC5DiYIDGjdEaChk3+0Gm5MqRL9Gz4OcZX/4golVhmfR9fH", + "9AOfL0YFW7LCCqi6MJZJUk2wAsuB4y+4tpoD7uBwwA0rdUIwhmFRpehqAGQ0o3VhJn2bHL8YQUm3II0h", + "0l0XFnZIpFJsxhSoLwvJs4SnwnCQWvgLvE2Ftec5E4bPOFNIA3BqRxlVVkwCQXNRwwkaDCP/CKs68dlq", + "ImltFpMlU2491sYA9LflsLU27EdHsBJGnNimN/APWpCCa2M5Di6AdhYv9ATZa2t+iWhlnUzdVC2JhlUz", + "klB9Hfe21oklnVSTPy1WrknHmSwjFozlO1A++M/AikaD3noEfnRb0Dn2Z0TZu4vt6MCR2oKKHGzuCy4M", + "rCclU2lV5pxQ7wyiPTG4hw3HNgbDgeZiXrBJIMiyLgxv/pwpxiZWsbT/5qzIJ5UseLZKvIe4GVidt06Q", + "wMUCFHc3tBEOrTHNaKwWDTFe70lY0SE4khUMtR4uJpWSc8W0hr/8b3a0lBcs3zROVq2P8hhYKOHu9G4e", + "s72B2K46vCpL0+VPVmV2iucN1cTQa5Y8fjkzlBfYVp5zPDlvoz5Q7HYGHkoSV5/QqayNU7BZNUiQXO7e", + "LidlasOAdY3QPaWZM7fKviAlt4oY6g2DdWk/HIC1Z1La2+A88cIHwjoMFfglkoA9WpdY+3KwK4d8L/gv", + "Nc7TaokoGZViBZ5WuGgomqH+cnd+Z2nG8zr0Ueth1TAQW8Kymnx9PJmSWpM/X7x5DWO7uDglulYzmjGd", + "GqAOZ2qnIboj2OVEAl3pHH32sqAwyd2OMEw1o4bNJRhCghOOlMUko0VhD0hB0S48tMda63CuB8OB8zPA", + "P5pnRyzacIDNB7mP6bgHZitxqD/NzbCj0YI/ETARfBEaDvQ1r6peBjJXbI6eDHV2zcy6xjKdTzJZo0q7", + "fpmZK7uGBEqQx5Y8NJ8LmKwwE8NUqZ+sn6rh4HY0lyP7dWSHN5JOrI7A+cJqKDNaaGZPtsz6+n8d1P5c", + "ZnUJCj/wO67JFGeTOs8zJcv1xl5KK4OAIoDKlb1/uWbsas6kKqkZHA1mhaRRw3j3iGfky3z95T7ztKOa", + "UD1xe7M2vu+gf6uhFc1I147YHh1es1ViS2G+4Jr82G7e0K0DOgHn4DrBTDZ+MibfgYAGKTsCgUpgtwlt", + "SAq3gluNETgEDhL0OsrtMfKaJacF/5XlzkECVKNvwABvq40rqjR7jFsxvmarJ2iuzOTSGdOgwjjJENlq", + "t0W1M7bbnnNdFXRFHrPxfDwks1DAzj0m5f0XXGdSpRhtOC8ZI1Bmwzn61FSo6+kk2sANonsbBw9tnMNt", + "ZpB6o60Lo+3xFXj10PV01Oo8wdiNTMjMqvqnHl0jd6Oxuhnn3emoIwrtKY6ZZFoYhjV9QQ07t8uzzufT", + "bPHCUGWA+Mnjs4s35A9fHz7F1warASwZKfg1I5cDIW9Gv88vB/c6IWlN5DVFuQdmADuQeH9Thz5FI6ci", + "320W95pCSk/ZsiN77UYspB5zkRW15kv2ORjD9q25y67EJ/cxu/1s87nLPjVenp1bUjFnU0V5NvlZ4oPF", + "RsboS0dN/9nW+zi0LUnFzaLEZosdXslj4XEcqn/8MNwkZEI/JClnSIcH70wj06APTma8cBbbTaM/pzfw", + "3GYnn9GCiZyqCTS6pMXui3Diqp75muvz90VG9IYqfO63JVHkU8MmC66NnCta3nn20AwcgoT6/iLwLd30", + "iSem018wHO0oXhuOnjL7cW3slvePyxWIxzZncuIr3ml0vtHeYYG6mJCUqEXK0CkjUozJuQ9BqUXBtCZX", + "UF1fweN1XVUFdxfTLdpoUkHEthL3LZUz2ye2BcY+28XGDvSYnNJs4XggqJR3032D1QbYqx3JN90XDdCJ", + "SaMTE4q38PFetsf4rHX34pbl7XPSe0Q+tewpuZhsuAb6VyB/CXSXUWdAxL0wkkztsc+KGk2dwZj89D6X", + "01LmbB8+pXIuaMHN6pWtmGDTrGBoV7MtuylkTbUWOZMrWht5RR47k/0Te2Oy9FZS05DVD6uKqZdy/lLO", + "ib5mJlugjUoKRuCFylYRuaVU5yI/JKzQjMxoUWhieTocSYFODOAkzUVmiM6oHQN8vSK0uKErDR+1HVlV", + "KXnL7UiufAyZHZobge0QzBOa8BkR0VjG5GwupHInWpqFveKtKgYBZTvvi1R8zkXqVNvvnsstqF5M5orn", + "rTvrYyTnI3I5KKgZFlJcDp5cilMkGPv5+e/Hv//9l38cjp4+ezb+8ukfv7wcXIr76L2Vil7l2iP+HodJ", + "QhHy+Ono6bMnG+eg72Vx6ZMV542IeBDJ1SseNP81oWu+oredx153w3POS7XCvXOfMQavGeCT1qE/vM8C", + "fZJLcgiu62jYe1yN3RP0jl3Ci/XH4aAW3OzOw7xsf29rrTOwoE/YVrdqE3fT1mGeW7V1MDqsK+sNX0re", + "1PuYrR5aroncaMGcrLbCXZCoScK0wX/Y6xuhKeb72KiaPXF+HR2W6q2odtZPxuR4CqFDdhWFFKO+oY3X", + "H8D3IGa6TNgsjpcMfHbgfmCo0Z9b5HvjTY+BErlQ+qCj4VI7y+WwUVuc5fIOnMqZxhOsqkclebGuiiSX", + "8R6aB73tZ5FInv+0zStTotdraf/ksXmLz4R5v/V1HtA51PG5g9F7t0BnO7K8LafKaTTeHOJZwZj4EMAm", + "1C9unWtiOcL4cxh3TT7J2RJdxpKmPpxHKPPP2yZdp2yRdflPHdFEzib6l5qqlHZkBydnxP3+zxsnUGjq", + "lUHMCxadvpIZxbMO59R1OSR0OR+SkoshKentEFnYMD4FTz7LNBS3KsP6TP7qfvlnLfHHzZqHdwTsuPKs", + "Kny7jR/HJISg14YdXYoReQUboo/Ihk3QdekIbIhTb+2LbeVbLxOPSFsMNhanlkhsG79sC98zedS6WQxb", + "OpwtcixosTI42K718FK0HqPLAaoYKBVQbnmxCYfKHyj0DbD/jWZk9842OnC3kkFsZhsMB2Hg/of4QzwH", + "/DNMAtyFOgNPv4/3mWvXNEuAQJponid239ZAdBTwNndGK42wSVZ/cVYKOJ16k2NHh6rQg7YgP9sOgskA", + "nkT9wCNrAO34Ttl798DrWoPh4IaLXN6ELxP394d73GxLRnWt2J7r4nhT/3KkXyPCZrn1aE8cXGQbJAPn", + "Q7LZ5c0Viva2M6XkNcQPAzEbINQzwZBBzRzlTPEly6P9su1mVLUxGcbkbT0teOa9fjW4wsrKEC6MJKiz", + "EmgMyg9dNGNnDTRRrKRcECbmXLCRvBEYxd6mZRzUBFwSGo/lrhMYE6C5DLFb1iG7aKn9jOBSZEs51yJo", + "eUxC0AbaYhHgoLN3OZtxwXEKVGFcb4Urcvz27G43IDv4NDdvbeAFsKV1lDRZVvby5kZhmZe9xmUo+q0K", + "ubajAFAxJi/AVYzlRNXC8JINCc1pZVXJYXLuimVS5WHruF+unNO5kNDnOOm7t2TBp3BS8BnLVhlGFO19", + "nGGphmuNKnmjJy5SCp27g+ituUCpuv8Vp6czQ9WcGejzoXtyyz+Z0ux6xgH4oTE2f4qOMkujdTWxW1uW", + "TOQs/7QdhlX8pL1AbNSn7UIbWrBP04VeiWwCZ9T5cWzyWW0zjd178SQ2QfJO8tVgZgqMQxMsDiFggAzF", + "xXxIFKsKuoJ/gnAz2QIxS+7BEKMhgpmkfb63O+CF4iSv8cXLAe/4ZmMF/f67FgYbvKeTboq2CPFFCMQw", + "gGBFmXQ4PrTa+NPxYTy4XNYYjtR7fdh1bIhMloglcP6itIDoXS7s4KBwg7dpj9TQwxEMQ/SSlbqATJPD", + "3qOP+PgujxBusI28bmTFRPe5wPcpL5HUD62AFx1xtvfmkQlO8hC9cICkg2v8/aZhssWkrqLzdeeD0LSF", + "j8wTbZUvdyl9QBoO/VQLqu8loENLTlh+qgEvWHYNHyfIgiZWowm97eCvi4QzKuSc+Fqa0Jnx0dG1AhWx", + "6cjdEIw9x4JB+JvtePyw3CTn+npSp/38L/ivwdcJoca4INOVYfqBx7Cfw/OSa45wJrHIeOB1Yfn80wjd", + "HissxlS4kAvCZ87O04Jnw2DqJcvvwS3YbQXBdRO44T20XjlTDMwPn6DpBaOFWazuxd6icOOUbKK/1MwF", + "6JlV8L9kGlxx/AEILYzJiXNQsRdTgFZUAKYDvP+XGs3n09o02HUwmkox45D4IHpV34f5w6gmJmmBe8Ht", + "XyUX1EgVzCQ4D6QuNDmacWQhCYLtTkYQN6qCaoPPDZMmLO+ujZW8YNpIsf11GeH1muJQWWt7t/kkJClk", + "/ml4BLg6uQX8JO0XVAimJjNaFFZl+6SdADHkkbtHMJWC6w9wMz+Q+xBdqz//FmVVLF0XZtL7vPqSGqZN", + "pNC5dlxM8sgZKpt3bneObDfETwEVUpxDCFd+YInUMz+dURFMBDtOzdYZKXnzrzarQI/35xqh3aDZTwuZ", + "XVte8Ckah0vLvWSTbzOcigc+jUr+jI52k0iXTan560H6lohS+iuBeCQfqN/osE1fkTr70HSTnA9akCcz", + "LuZW0PKUSvk2Gl0LBDhIfqq1zDggLQfs7XX9HCyQ1BimbKv/xz8OR3+ko9mH355+/fF395Do6YlFkZab", + "JtQU22MSn3xX+u7ULzbRiwspPiJgORy2rEAtQ4ACWPIHuUpXSpbSL/Ie8dvK8BlYxn19d1frGqzvakID", + "i7tegDRbBjn6gPsWbDE7qVjnvrQPVrYN+N3Z2b7nzCItA999jHjBsoxh8VsvlHH/wSYdv2EsqHPCfdCF", + "tsS64yrbovESA5172r+XsHFtca3rT6PBrnfQqCwPMXLQejrD7wZb2oIEymhExLVyit1WBc+4cWxjZBsi", + "NINGuJg3ML4NU/okJDDRdVlStXoAzcEZpD6xyc71EuxtzaPx/Yf+aW13iiF+1Ke5BCqmZVHfQ2g0DTyw", + "1ICGl0w5i+V+w/te0WpBfBMPPTJ8Bp6Aueh+BBS35AXYXcW/b6xC98UHGZjDAr/HqFAoBMynh6VeHN6E", + "OmLcj0YuEHHcVybaKEZLhE0xMg7KIXMgJ5bP2b0IB99CP8k59o/tn0bBMkyVn0TQGmlogW+4u75XKjZz", + "93sXdLfmt/GwAs9Bi04UK6X5JLvXhe1qjKJJbyVBi5XLJtd2Iakymjj0dpW1SD2srfvS7OEo5ofRF7jh", + "BhM89+/k45lw4PdzeeCG77cU+oYpQASKgs7zkPAozX5enL49Pz05fnf64gjgZc/xeYQWcUOEC20YzceX", + "PuKocRhEEDbAFBqTY0BYRNBwWTRgk7g6hIr8UnhHpALy5EG8rvbPUMr37hoEr9COh5L9YWIPQ2E5YSqI", + "oqRcjHTFMj7jGQlFmzHbJlL+gdmCpmIQTuxne/ydZUCqnePWoOpLLq5TRMRcCO1GTM8lLZwv4MeAFSW3", + "Xn6+9wWbuiW9ncBrya2ZGHnNUjCFPhIEf8eQ8yYmRTgcvHVspF9qpnjKvf0v+IPdX4dq7YKdEQDbbRLT", + "8JYjbwT6+I13XV4En2+C4BIgiWqVxiZ9pImgpla0IAUV85rOHSp3GhcRcCxb6wbBs4OjLw8PD9cQvnH5", + "IMAQKmJU6pKpBaN5cgUBuHRrbFFzxC+g/Mfh4IabxQSVhmA4cEPb5bw7dUOKrSf/VICdCaDafHdk6UDc", + "PDj2OI5ojXi/HaWs101wbqTAwzYN9U3JEbJMjyPz3A6jPps5b1PLPeH0R/Vtg+kRr6O3AmF4Ok9LxIj/", + "eml0X/YLiK4x94UPwSO2xX1TzLIHt7Xb7eY17Qzie58Yz3cPTtPqOrfnFzn9+DIJ7Ijl0cCcp03l/SOL", + "qu0wyJOmsFPQ5p1h4xjX5HaJLBAi/AOc82FXnn/sAvlNAFYdW3PUvVEutCq/a9VtkK88u1asYMt0WMu5", + "/8nKp+DbECmqTrJ6Cr77fO1au4fAXYRWM4uZLAp5U1cTj8OqNwEJY+lRXZGm+J64uGAQApSw7fS1AMGz", + "lZwwR4prFQMw3JMNyKtLsafE8muzRf1rs5ULLyPux1Wgme5RCQ84KcUOEiJVrOCCOSa8kdfsoJhwKeww", + "Gu2kA4u51/HptNTiLhtbCSXbLXiC3Vb/O1curp3eQjMrVieLWlwz1QR27Ij+Yau9cbjSH4frkel8Uqsi", + "jQL9/vyl98BpsPSP356FdKUer/DRwpjq6OCgkBktFlKboz8c/uHw0ZMxOYGQBA04KJoZsuSUHL9+993L", + "v03OXn93en76+uR0YvthYsmVFKAqgkuM0+Ma+O1kH61bsuIpqVHKnBUtdWEw47fg8pAAvrbLZTUTqGW5", + "X63ZmLzAqsAOH0HtRw7FCrxLQa8bTakGaHJs4RsHI4LhRO49n1DXMIhg+Kc+yHBn9cFvgpbs48GYXCCa", + "6qwuSKYYgDj59OseTQajfQGlnS8ZNtVeLz/J9VijDoWtw1YlTzSSYZwE3a+Vz+80vhSXYq0YgBFlTBjl", + "sWHevXsLwO48cyFYrgHdNAnPpQh7ZDhTJKPgZD6+FHaXcA0BH7ikFezKm9ev/6f7nnPFMiMVRltp8ljz", + "khdU2WILeUPeFAUtKbmR6lo/gTF/8cWJ7/gV7MnRF19cihGBJTwiF7jL8NcIMueFcVoWbjffgyhMa16Y", + "ERdDItyYvB74BEMYV44sYPA3vCgINYaVFYAFFZLmG0mjmZsbNi6LG+7Lp0fkFSvtzO16MVzFZ6OSi9ow", + "8u7dSyj17Ii8ZUpzDXkb3rKp1ctzaqglYFsCo3ZnBZ8vDMlZXoeMEj5ngveux+f6DDzJMULssoWZHzGY", + "3vMLF0s7Tz04+uownNdAv566rJwD2hpgqh5I6rZkqqBVuFjZ+ppV1F1yB3aVBsGu2JQ6hCwXSKqnkPgr", + "5qx3Og1N/rCtxwG/PdLEUwt59TIcCNsBrFHToia1tg03NG4l6Fl0bprOI1W85wyRxyXSyH+RKpCBPwff", + "MXupZf4AvISB2H4dd0t1hKRHyjXKC64dlsAC9UW9dokPK9+fBO1MHHJSc6DJdM5gFiMmRsun46/I499/", + "/QeS81I/GRJaFKNXXPCXr0Yvvx4tn5HHz//wJf5o28NWIAjQHlKWN5eoK3dWmaOjcFivmtM63udQPBs0", + "h6A75uR5+Dh8UKn+yaX2s/2kdncCDQ1GcuCxFwR9u9FsxpP2CBNLvDaaZs0b10m3+B+S9h6nMG+96fip", + "BBU7UgRjC0IYgF+ZD72KYtd2d1eG5g/4kpHjs5ipDf7lyG0rMdHbrTbLBuDLBPObv/OPkza3DSQqIoTa", + "YHRNKWlSiFv3THnwPStLOno+suz2AYjQahW24zoF/30ihVGy0ERRkcsSQsB4y3ft8eH4cPRsfPhkvPHa", + "/2zLtd80qWs7To1WCwxJ8VwxOwafTMsSZptiMloU8HS5KU/WcGBkNblO7IusRtdE25UHKU0VLZlhKr25", + "tpEq9ZaXFazWPc3c1Tpy98N+DolK7628rOda7kgsL49Kq5RNFaP5iPKD8nZK+QgrIw9dPu3R2DbpgEn5", + "lUwUEPIgTRlVTAFVeAx9fNpcm5u9TUHyMnsHyxQzBLJIudgvEVJPwYFf4zrHb88mP57+bUyclphHi5ZL", + "htn5nYcGkQqyB8jaILZ0phgoJ7TQ4z1kXJhksyuNoBsTgECQmAJyuL6VeNHU7ZsmhydnJJXuLWlMLphB", + "fyvvmOURo0oJedAo4naiVhqa4WgEokUyBHR/fvUvLEH6zmf/ufR4PE2cBVXZYkL1ZCVrfBaHNBE3UuX+", + "PjMcLExZDIaWLTDFM4cuY1njYBi5vxRcXCOyDMwe/wmekvbMeo3CVinkNI0vU/EfU8fLoX2UzFCrkntM", + "CX/KHuMBclgYbNnKyA5RlGCt4FI8WYeGQENGPqGmD9VjwUJPN1Q7y0fe2h27HiO3IGuzYrcVV0zv1oMr", + "PCav66IgJaNC45R6+xN14ZLjtmLx21k/NiRX6mSdiwbTVgnot9kL9t38B/7n65fla/n2l3P9bnc0mh/q", + "koqRT0mOisjG7k7OGuMspnVYP8hMQQzX5tR0TSmiM1lxMYfct6yszGpIuFgw5V5p4fV0Ztc9annnR9u3", + "oQ6sQnpXgj1fyZsIFr4HL3UN/zSe1AhxY5S8IdjQmPzIVi6bbFhkTR5LRR59gXZBWhT4m34yJn9FC6At", + "77g1vl7DyycyDj32m6P9SqEThFusxtDXDEN/Y8uPkCu7T9CJc0IFvIEmq5mgSskbtKH1U3PDx2rNVJrE", + "3thBeY6cpKmf5ULkcjvQkTsxQ494FPocxuwiyWWBg/3EzeICeNLuBnHH+9ZN4ZDANOW/9VbZuyH8GIl0", + "xEhozlZtFlJ58BxMJXtErE709ZePcZ5H9j/IQjtX0L+9//PP59//9fu/Hy6qi9K8mr7/avn+h+8usqd/", + "+frvp7nMvn/91V/Etz+z93+WF+8Xb6bfXf96Ud5e/+Uvf/pTX/4hHdaloxEWjCorb7w61OUO5MJYsa9Z", + "VitWrMj/+r/+X8Lj/OLNMyGdUy7aU8m/X1TZ6lv7/4uzH14Xf3v+5yI/3okS3IiHYSc+bDdVe3buBU9z", + "o0A7uUe8z7qzfqwX8sbqNRl7AqmLvS8woi8Fj9E0x0g6HLymJcvjd+KOT2Ymha5LfPejwuMyvVUyrzN4", + "Bq9qo8mUFbLJaVt6dBUmFM8WVm9JoC5FnqPJdHr+Okr7R+euVS+ZmJtFfLHq2bDQZ/J0uh/R/z+dDM9+", + "BehKA7jrYThG0ey6iWdDT33LMRGt1CtUsZ5DtQa6ASsyAKfRamHFeF0UPvnmhpjqtQGf96Qxxe+AsuVH", + "e4NMFp1tN43XB0GHdRsOMqlUXZn4Uy285I6/Nns/cVk50bEc3IYLXnIzYbcZplPdNDukan0S0FIu0ggx", + "PljID2E0xS3BsPWyNrSBRAtJkYW8GbaSa8+snh7weCjgTSs+rSGLtAS4K1JXEEgDMWRWF6ACo4HqKON0", + "k4G4KuTKn4BwoYgSeLvWJi4vNKxnaCe9MnXOZeuJtEe/0SQ4BBpJqK0WHoHWD6R/ltiYH/QNFiJ5iH1s", + "ZwUlU2ZuGBNxbzqkXTgiaJkJFoY/fP3l4SE4kyUMDZFhA+EVNw/tJyjTPzIrMvA1DFvrLEg0yOd2SHcZ", + "aPIhvDaLixr/XHPVTWNVnlhZoJYOMhnrkrazAN6dlTfpWx0kgN9RkYdalWJ2zjqmPpcIWckiADYCviiO", + "8UMsGF2Z9dykzYS6yYP0lIvQPVz+B90WjwwTFABxaJ7Kydzh2r4zB+2Z4t3f0uy6ro4LUG9Ob7k2+kSK", + "WcE3RSY4edi58bk0JX6xptDyhGLTEwZtJ49lDz7LSzan2Yos2lccxL+2bH4M3ipXtt8rhGpXVOC7cdL8", + "0uTWTVrwV6gfhwnAJD9sdfaDaTd5UH0vcZv9634mZjKVJN9eG+Lwk44WhNcK9zu+qTsFGoQSrnyLeJZP", + "x4cAPLY2dbdNfRns8efoNtvJkV5rw9QIS42eHT77anT4dPQ0+arir9kp6LS68l7ftcbUOR4+Fs6yZblS", + "Gb+azQDC10R/9qrUB0IuIfOAL+FvN4mF08+PDg7KFUJSHODOuOnqAzf9g83zxpthYnnhe8gX5D0P1weB", + "jEej3MnrDI/R7k5uhpdMG1pWGwwlbp8jS0xrGZoZvnt6ePT88Ojw8O87Wmo656Wht3hgYZWibes/Ni+5", + "NudO80/kEsbdSYhgwUhl993qxl6RcDOH9xCN+nNJBZ8xbUb2qgGZsXa2VkTHGh+izrDWU5CA3Y0R7NZM", + "slrpFPtz4EiZFIaLGukUy47BwRlgBuz2CRlf/P10EMh1vON+6A2r/crZB9+LsGinaY59TALTC1bFkauE", + "eH1Woa0V6M1O3WMze/105IfZ08H6YRWiOU4gj1GV4HmCVpVzX/fdTCJowboZaN8++Tk1yvHa7IZN2wWI", + "233afQk11tpsnj5Wr8HYguLj43AgBdvBmrHDsD8Od2uib4Qfwr67bODH5ZTPa1nfVS/w14rJBknjaD3c", + "s6KHScUM5SJco6kfjXekGpN3Lv8zvgplUmS84A6gWkjjgDhZVdCMwUtAIefgQeKI7uzFeE/B+LLbALlZ", + "SHwKgvzpgD4GqeA0Qr95MvfAmskOexQo1+gkzPy/pQ4V70c/q4pC2fag0A073WfM59ptPixgjsE7/rEq", + "kqdIk2D0INIjserxpThZSEsxFN4iuJjPaufZAtqc0wk0ZE448IoeF8Hrf9x2vBusKWCj5TO8hQUTz7M/", + "dE0+MZTO8ejvdPTr4eiPH5p/jiejD1/8Lk2qQiBSw/qSnb1A25N3B2M5uWK3GMU34fKKNJWjtJuA4Ldk", + "anUpBDM3Ul2DhdIto72YuWUckzd4Q/J6gibMrkvmE1Vi3CBe3+BpgulvyIwXTK+0YSV5f35GKmoWOjzk", + "Tplg1OH1NIMjSkoXmhIt45eHf/x6i+msrel6J+RIU92k/qILMuTquELd9+qInCKQfrBJVIuVBgakBa30", + "QkI6L20SqzWE9J5o0RE5ZAcCz9QrP5yrI3KipNahSIc5ckGOv/vWj+6xLiA5tCfpvAZMULQUOUyZIQAw", + "ZlSg8dgZ/IB7r/yLiOsMfO3eiGawdgdcV1wTWhs5yplBFgrX9RnmG5jzDFFRx61MIJ/nsuAt6J4BBIcD", + "cL6snErJRK5hGy8sCeYxBTY0dkSu7Pejg4MDt+4HljSvwHm4pL9KQS6eH5EruIUgeY8sl4BSB0Ye4Big", + "/PdSzgtGTgpZ58QN+ohczfW2quhkHfbbcZ7wpgXumkPcY6Zd9oD4yRhAhlxoZh6HuOx4i4LLDT67tW9S", + "+x27DfeMsLktzpUWIyZbnMgS9evvUFtNKLogXJaM5JBmQfskqr4imdp2kJcLxnLtmL8t5TjfuwW7FLJi", + "oknamzOnxtxQlY8A3tQA7C6418oiZ4pcvPhRd/P0Xgpjr2zERf1DS4A/LqQV+l4jh8vCVfgbUhuDPEw5", + "+XllJPnAUIJrMWsUCq/W21rDACt+BZb5CSZamXQyTkwU+xkO9tU2TaOz9LB+0GtY03nNcyoyNk5H46bN", + "++HR1936sBystbuJh1lRw+ZSraKZIZb4tM7nrLHGX/X0H+lFact7gNhHyDgfQ+1N75CDms7YmATK9A8P", + "TjlE4zxu5mBXdWt3U5XJFi2QgLb4spTOhWbKDCFG37gUIyHGMdJ8rDyhRGOeMOfB7XzRj40secbN6osv", + "gG9+8YVLJ6YXVOVffHFE3jTNwFsEVPAbBqUwyz9VnGls4pVnyfCztq2811avil4nno0AEt2dXvL42dsT", + "zHzrOshAPmL7N4ob5lzOf5A3rXZgvni8NYSbwESejom/coHTPwN3AcuQmCY/vDwhwQiC5mfnz5VJCQmq", + "LHlD15fimd3/5isOJe7V2bGHpLIXqYxX1PIIKAc4yMLoS/F8TI7BeccyePfV0rUl4WGrW1yPVg+X4ssx", + "eRu37vLkIO+H7BYLJYWsdbFyEe2gYoXuv7I6H4BYrkghZUWY0LVCz7pgo9eA7WwpyeqEwnsbRWNzRxM3", + "4i1ToDSIjDnaQcpxewa8mOkjclkfHj7PyFelJgU1TGSQzOwk2t54C4/I//nssF30DGYRwUeFNp8fBkfW", + "G6m0GWWWoh4vOYVNCdN9giP+vqaKCsOYdgM+Lgq/oW4viPMrsTMlj6k/HIQCOcJPSNOgz52sr43lGq5v", + "4G+PBbshaMmA0NSyu+jQ0FnOykp2pkke+4gM1tknu/2WNbl5fcsEm3HjZ3XO8jqz3TiN3mMaeOju3CmP", + "fMnzuhXSMSKvrH7JZjOecYj7cClTyOO6yuHsUDAb2a11wVa1kfZylG04lOBL2XARXVHHOLRXg5CRYet1", + "hf8eEXi8sL1fe1cgOxW7YYaJIbErG37xjmzd0Chkjdqbc49+/4c/Oh+YI1nkEweIB5ZdHIOtBD8/ffbc", + "YZkMjp4fDgespLxwfjf/w/UwzmTpPWuOBn+WC0FeoEsOnUOfWa2NLMF+XylW8rocfHB+P0dffvV16ODZ", + "V1EHVLCeDqhg5KLkZpHq4gPEW3XTfbnprwl0pegqhksiZy/AOxdrjMmLEJ1OwYRSyqW/DiAPA8UUpMhr", + "aRhq3a+lGMGu2QbD3mhe4Iszx/zvtuQL2w2Qg6HXDHKgM4cbsAQ/DSSCoOnaS/o1W1lJJoUjQdtO8BKL", + "X7Zbo9XuJgGZ8HHUbSLZThk72/wjGurLGt6TTRgFP+qcVE25UVStfNq04KCykrVqp1hLZQvvxl5UyX1u", + "96wd5gusMYID1b0ulb4u7P63TBtSKXvaM0cG70GuC90QAhGIRQKjZtrHNF+Go3Y5sH+BjCuYPRqQmR/2", + "lxRw+yDUWajjDa0aIURG5JoxwA8pnVpJp8XKauvKtEhFoxG8YLc8k6At2ztgYTVNSTI0EfmH6CZ9oVTG", + "JcfXGRVrMZj/tjwjQUCQLqtgS7Y1tupiJbKXUNC241XPTfwmqZ9awuJiBMbihjK9zMHAzFdSzOWLb0fa", + "rKJbCHKgd6lGrcJ3Y4+Mi1jjs1WEueHQbsDDcIS/jlBpU9TTcVf19T6foBYAOpbzIEMVEwSoV32jgVQs", + "gmiKG4JAHIQCBf3JmzScb7pTTc2o4NetBh//jotsSH4HSWR/B1lkf0fz/J28YGZIflfVRQGqgb24SAHx", + "ElYzwkmdiUyxJoM5U9pqTOxGD4ntRw/J0vJzOH3vYQcaZdl2rZlByZ/nxEWELBkBFujj8B8nRhMaEw5D", + "BFmb3wYv2f11zPIc1dBCh2P/NoDIlcAv7LkbDpoVgiKyGhwN7EohfvnC/jWGmQ5CLuWnH4e+ILq3hYIF", + "1eavnN3Ak6wrPXh2+Ozr0eHvR8/+8O7ps6NDfJD9+ME2ggMKh7RnNH5d4p7gUDZ9LHllm/yw49tnIP3d", + "oErctbLvFRel/474ijogKDicOMPSwFmzxqKz+Rk3ZQYKYvVOwwpV04Bem/Ha3d3UXffH5CqYmrz725WL", + "anA6NMsxaRM+u0p1KaJLIbFLTeeNcQEyXrk7F3jyR+13wN6vLgV2ZM9GpXhptQNkV6EOmoApJIfm/lF6", + "riBKqmMGuhT+utKxkzkDh2U8gJvQ8kZG/QmytbJiNSZn+NB2Ka5ca1eo6ztY2W/Cw1vbwOKuxprUQjC7", + "U1St2hblMCUwHXaWvPWts0zpGE1/Qu5EQHHt3TzovmW5ktn1HSEPfrogroE+tAMwpfiyka88ICMeJaOv", + "9JDcsGlI/DCE61fL014PyenJBTFUXxMlC29yPj15hgFoEMBov4/7Iv8zuWCKjWHQo+WXR4dDQsGYPjbc", + "UDHCHwy7NaPls6NDbOaFzKDywphKHx0c5DLTY3qjx65qJsuDKS7HQQFZXsBwPa95zg4wCn0UvE/HLrQr", + "EVK5Nrh2BKXrAoxyc9yeWo8Y1Wb0dIBmPKbNJDjeYHOT5ZfrYZVwM5lo/itrvUQ9HSbdvEy2IAB2Ai9x", + "Yb9DKmgjiawML20Rs1Cyni+q2qSjaXNeMpH2XXtTm6o2JJSA7tbJDBe0rfK28CVCA3rLEFIaYMEpanqy", + "OxrgGg5+o2Li+Cw2w7dzl6DakO4dnPt7coSdwI6FOUNRYos25mUXPuj5wNB/gECiYcehFdOyoida77t9", + "f+y6HweGiZ69GEbBpZWS8OBmv0pFjs9fo8FmKa9ZAINaI+hHQ/Jo03l7BI1Rgcpn7JXb6vT4/HUnbiZ1", + "dHYKQ/WHKg3eME8S6jvH1vD3wBX9cnn8Fr8K4ZA+6gw6Pr2JztvHOX4wprWR64/Frns/y4CZ6/OcX9lq", + "V2D4mwv+K5i6lb3PQzoGWOuR32o9vBRrn4/PXzuO614rw86MGnLQB7ZYMpTYNqk2b60e4vjtiQOXVc9N", + "hgSvg+74D/EpHxGUqD20TDtPG0uDrnlE+2nCs52VYOUsBzlTxSqwlFYsAS4w0KiPG8E/4CG3lDktQMQj", + "h30e/fvLT4UDMoTmqolgN5OCC6a3A5+GXC8SXhIrsEZC3Qa0xjEZdmu8J1JgteMkAqtRtciS0SId9uUL", + "kilb0CWXarx7PPUmvINvpSy+sxz2Lx6UtyPcpCzSwOFTKfVWzMlvoZC9ENg+Eha17muy7a5vnIyK/lHu", + "NRYfuLoVExKYer297RMpfq4FmL9b9SZCbq37guu1uvjgvH/Nj+ml02azanJMAFqDizkimHvYFp/vIGeZ", + "YlQDFgMX7t8Yl+WBTnUmFdPokgTyc5xMv90TINsgm3yr6JJdgBzeF5TxJzZtVUzjMiYRMKBXgrWbwHyp", + "APbg2/PjvwbMCqttWy37SdL1Ps7V0e7hO/+TCyo+IlX+p5yuhqS6+dMNY9dDUpV/KqUwiyGpVn9aMUhW", + "4JloBZhtN/Z/Svs/abAHXbGigERjWyGnHXB0U4NoRHt12K4JXsVuIdulDEaObSzzzIMqWoYY1SSPp7LI", + "h4QbCsAIVF0zpZ+kX9T3h1XsbiYareOPAeGtUnxJs9VoJjOgdVQA0Sjvwc/tFdjeRMPzmLsQXTBTV4Do", + "9nRMLvhckLoi1ISbzdT2CPcZbPWAVvwAnpe/Z6bBWLGyI6d6MZXw+py6I9GKj7GNcdMoraoDr7m6K37q", + "aeDblWHnVMx7EiNMV4b1YDs5l3qEImr+6L6AnNCCiZyqM2GYckK5G3KFJUb0BvIIu4IIZE8Nmyy4NnKu", + "aEnofK7YnHoibMI8RQ3jXMhaAYDHajAc2FMDwk2AqfuXmioD4g7OTup8NP7c6zLER+VvZPOhPriVQwob", + "6SBJgiaKiH10NoZ/cKa3SmroOiX1Ov0lVjZk6G8ebcYk+owgCiVdoU1YFlKMcgbRrSzHXyua4XM43IkU", + "o/lRwPtPuq5OxkejD/+VdFg9ofA8brt+lfRsOr2lmRkt9YhWlZK3mKIVnTH88aUka1qJKQKM2VabPILJ", + "0DgNT05+WFVMvZTzl3JO9DWz92x8hxbe/I6Bh1xfCuJT5A0JK2xTgjA7MHjS5iIz8OITAi6fjG3PUOKI", + "0OKGrtBzojapqmBkhxrRHI88YikOGx/3cJjfOD9xPiMiGmzbQOaUaOhqMBxELacJPc9OWr7DHVAG7yds", + "0g5TGNI8gjARfHy0ohDj7RFi1a5sE1yyclbKBuRq3fGTzicY7drXnTe+FnTeDY0dpjt0EG87ZbFzQ3Ah", + "qL1j8CGqU7bgIn+ofrWZZAvLhEMqO2rSIcK0KEZZARp/8FdyUXOQwNg/JGm3ZNDqsLtee43MGT/vPKTG", + "eEpOXpyQShaFFU7F6s7DAj+xBFAALjv2Aq5kwZDj/LOHTRKMYeQYPyTOXWcHSK0Og7VdeRuArTcmJyG5", + "5uWgktrMFdOXg2/IrDa1coNzNZDx0jyHuyI+lbWtFr6B5KgANgqdONOQMW/9yKKi0UmFwGnLZ5oj6+wF", + "jruv64+FNNv6UqwxOWCcHHRZSLNHPwiygL5UCY3h70xJB43ryniKi3t3ZyDyJDW0sR+uEdvzZ2lIQsCF", + "SE/audF7x9WM8SW4EMOpC/d+rmHXcTjJ+eK79n6n/gZDRn3j8HAFcaNw6FyLdzxi/ZaCaDXWtimpoSwo", + "FydSoOtL27ImxcSHGq2p7FgezX3Ni5GwF4UG2ZML98QBwhTk7hE5Rvlr1AqLW+1A5YW9WcmZf7mzFXz3", + "R+SdLwswIysnb4N0d4UdWGaruHvwdzCaUE+74spqrqBIJWrYHwn8GCrFwhymMBjGK9QMAP9omk8L+JDC", + "KhEd6vx+Cy6u0RG4WVJYT+fo3gAi2sW0ulEWNnLdTz3a4q25tRqCuGd6LBjatnrntlAPwm/TeQ/1mld9", + "HvDrDxPOk7u1plyKgwyQAaRYMqWpz6u7tn7GJd7csnp+QCeuRi8mXXCsN1IWESSd/cuNVHvASci6wMoK", + "XtTgCZhwk3ajlwXbY5TntjgAu8piktGiSIb3YawacGg7ugyCXixr8yuK+FI5PH89hkMEEX2yYH+yNZ4k", + "hxr6TAX9+36sDM6Z19yo1lwbKky3l/DDk51j0G0XJ7Qokm4YMQ3Ckm4hv5OGPjrOfG6FHAFFYVDmRrog", + "LhcAhfkUcH2OyOXgB1YUcgjJGey9eyXr//1yAI483jXLNQqO7PqI/OO3Sxjk5cBWN+zWoJMe/is0eTn4", + "+AH4WRNM3dGlXf4OyxH9uBMb2Osx1rwLtEe489a41XxLVSKfz4d22kdt8p+1FONzevMqRGo0+R15aVe7", + "MRD48o0/D+CqcTE/gM8f2/t6LlP3K/vVnwh/BDQTOabt2sxb1nFyAu1akQ2PSQM8HH2Sw1jSTVsUjgNe", + "Q+AqPuK7lQRS4wU3zyfenHmcB/OZh018DFZ/cAw1inJh0MeM6utJVlAVnlaPyLG+RoggSEMQ/2Yr3LDp", + "BJs+8hY8u3I3bEoeu4Omo0KTKLhUqs4PMz6HQczs1fuIfAeGgvfnLwOlPfbQQEYiWh83K/hRyQLHr1lJ", + "heFZGNMpJlEMPxwsAQPYLwfg+dm7m0Ong+49iNpaI/YXeMQl37569tWmNoxibK26/dgyY07tVcB9EHTJ", + "52FZMWys2wDmMzaKAsUVrirsmzPFsKZwbK9LjDHWeQKpAMV2KACseX6brAprt8WSc3upY/S55lO0Dh6m", + "rvkzDHrjYdC7++x0DgKcE/SROZuRKwZm9XwCXyH0D5BaDwKydPcceaBsAEAriuh3aOKyAY4Ib+M+nouc", + "Nhn/faVK2i/MhyyCoayziPhy7Eg01IQZfOfh9J1LSC7Fo8YxBMN+GxEOYSL4Lo/Jf54MLwW1t9+yMu76", + "NqNFMaXZNXh/2Q9Ak9qoOjMQs+5cQyqqdEBR7SKIRkuagF7g2mB+AVlgSlMoH6Pk7rzmlwJb6V/zbiBB", + "i6zXqTUi6g+7J6dt+HMi4gCOhuNkW3OhMRO9hrmEDbKYcMNSjzhfDXfI4uD1t6YNcHc2tRLjS/FWsSUs", + "MRfoNgjhWBCPCFUx32xAVmgSHhxuyz+wxsa3TX79OXCQlBEJF8EA7BmBKzQV9Bij0XgO/wXttTFFkGsu", + "8kjoNOmZR+Bgj+sgHf5hSVdEQfgWYiiEqH4hDWG3FQR4LtilaLp/pFvD8rZ+e+ilQgawSqb6/JhUQTeh", + "NL5lauQ9X7I1ThiQGiGszrl3Q9CWv5mMyK9MSc/8vLM4vERgCq6QJiABv1rnXG7NvbuGNPmxnfpqe14S", + "hwAZ5SVpue+3QBZDePwGJEifRGvj1YHdmu6ozUIxvZBF3iOEcod54krBDgCv7mTIwxQjT7emGNmaOyNN", + "K/vnSkwlWnzolIm75yp4mOyHdyfORn8BHak/cCz1aL+ujTTaYojcbGiaC/Kt5TnjS/FT5KZNHkO0L3fy", + "8cnQlwcHcLt+iCUC3Owom5kjouvZjN9GEXV5aPyRJu60IQvwndGphgc295s99EdZqyVgFfYWDopBpJQ0", + "KdrGiUftHbLv27V7cEZA/s4UoMagL3aHg+3NJ/6TQHPDK8zW3KtM+QcJdOaTirW2epNbSsjSdONc+qAi", + "4K9YNdUn0svR1Q9CD11+GGiuMRwH4raU3GCAc+Gz9lGHBRQnHYR4RDhgrlHMhD6VZhGaE3lPjXUXmd1P", + "w4MIpofOxbQLGD2pBeIerKsgNJEuNR0PcNe8lz1JLrtEmBRba2PzBzd2bcEohA0ZfHrTKu8YYNLBZYb0", + "+gjmZ/9JqKDF6lfH1NDz/VJ4eAzIllEZXtL4ZqqNJfT5Cv2XPeHrcNEk7bTresOtbpfE2SfOAo8eLfqG", + "m2wBwD44AXvHxsaiNzMXXW2Ps49Nslooxm7aWkaSnGu447Us4ZHDG6CdTcKlrnQOLRujY11hcMd85TAn", + "XTtu0XbyO73whQFa3CwmGN7swDh387qrFBs1exYaAB9yyuGfmDEFKEBzh8vtem6ccnZyy+uSaTJ5/hrB", + "YgL3DoG2MG1C+gl7Xc9kOcWB2wsoDp6JBRUIOKmj7BTr03CxtY5CXYwN7XvsipKVdxN9tAbbFAW925KV", + "Vb3vrnnvzs95adkKyzFcJTlShXwTVwqRBiCrAtg6b43LRAUp5eMchW0Yy3VP1z00sLowfFItFNVJTIoC", + "/JBAVYBCIO+cpo+jhjApuAcDGpMnZ8RbAVv9jmANuw+6c9a6SWse4ky55yevkAPfgnPePgvNsY+43H32", + "Q8nasBAwtfFR1ZZ8hxnVdm39nqzyDh31kP4bF0GXR0SEd4uDwAGcdYa4tyKCKAPo6quJFZTxbI6ISyuH", + "ryGPFIOg1kdDslhVVqPGZMnuMEGR+IdH99gzK64A1Ldvtt8qCWhDUfRvM21/1BMEF8iSa/Io9PLoyb3G", + "ehcht0f79XQC5qi0veoFg77Ak7uejkLJrbPPfUWY/QNzlLXn4HAG15j4Gml3D1W0xMNYTiWflzFY8b85", + "ZO+m3An/DNzeAMW7A3AvymxIh+7WFLjOVWP1XY3/g5v7H9zc/+DmpnFz98LHTWYZ2fdoNf6Tm5J6Fv7N", + "ElOTGBk262zWJCFu0l06ED5IF1ZXw0tRVz4zmhu1Xx/0OZQzYkfraiP8N6AL0KlcOkYKJREWxAer2hNg", + "omQdlsjTMHtbUqXsu2pRtNFXX8fxRk/XRC5ilLmfU+L1boDEHVm5OefJ58ji04dU88bee4rA/1yxFpJK", + "5XF5Gs988FZShkOkdch210p1EMf/N03snGXnAh2U5Ywwmi1aZLSzQ52tgzuArW31rGslufGJbdySbNjl", + "F9TQ75UM3axtc1Ybe7+E/F+rdPhzzvX1BHh8yyy6Iaohl9kEEQb3qOABXCYFn7FslRXppBDwkJMeKEQB", + "OIrdpdcFo4VZrCZLaWAFQHasVe5xpv/ZKr6TzB4eoWudHpFL8wIpX/rG1RMnHMeuYCsAxZ3ux+6exj2c", + "ALTtxO1rX/GqoKtJRuv5wkzqamOphgp/S4b1U6FBxw4YRslyuMZpouhZ4ahK/9y7TtCeBLacidcyZ31H", + "gsKDDtwQZ9y53O4y3uhdd/0V1bvZ7HeQrPKb2TOx3/HbRG6J4opyMQkoX+n9cyhYk1yWlIvkHPE4oV0n", + "WaBgVLOJvZpqh5i2y7IWVpFMjknInO0xT0VnpneD7NmZFJLmuw7Le48nxVn6F7jR7NFJh7b9tjYT30Lk", + "EAzdR+Wfm6sDvOg+VBlXBkTlPSpvEBJM5B4YYWtC/n0FysOw+31XShuqjJ/THuSICvzu/bTDx9aa25vD", + "97HuaAWiQW6jdYyW66X2/Vjivhu/JyeqGFMTV0en0Qo27UPHTdIFCu7cf/+6r3OYVutbtuCiR49/t7K3", + "7r/zuQ/tW3J2g3f3cKslRlaykHN0JEAo4pBebd1jDsbcXrmNfhRpZTixlnbed2o3UihSWwRRlHdpN+bh", + "/Xt/t6ZbR2YnhFVX+weQ9P03NtQE/Ha7Ry53J4yDOjC7zCBo4nDpbf6dM0jHnYdEbmmfAWj2HA1Xn8H2", + "28rs6jJSgAlupmT5/1OTrM+UZrWk+5pk97Dv3QAwWLTaXHv3oM9u7nJLEB78Il82yosJn/l81F2r8g/y", + "xlLJgoq8YE3eB7zEo3m53cLVETmeSmUIn0V2Kpf1GhvQ7ZDeK33Nq1YDF9e86vY1bB4hzIJhdO+Vx6Zm", + "ttcG+9slt+w0ET2da5dCRRs6BzcFAU7PPG8AevwxX1uf9mgHw0EYRNs6s1buwWyQIfNfjxGylTjZJYlA", + "wktaJP26pk2SL6maB7OlDqk8fYuXgouAduhTy4BPviGl1IY8+yq09TlNk5/VGhnYd585cpuR0G/BRiuh", + "UXw+Zwo9T2vls/s24MtrlsNPZC10o93LXOiWaEd74R42wkZv2+Se3Q1ZMItJy7Gt7ZeVc0xH5f1ObXkr", + "QbOAEO4d2RrfPa8eJNw+7Rn9lSnp5DfkAakKubLaYcSRUxAqTcGgY0bUgb6nSBBNjh9cNpHTQgp4iIYU", + "CAXT6Uy5i6AO7aB5Od0JIH1zdjuJo1q2NXFma7iJncT1NqW3C2zQh9+ii6Zl2j7XIJ3KGh0GW5pbMmge", + "e98xMMmPFQo3lKtZppiZ4BnY6jxjy160Sd/5Km+ti8XcKHz1zklxu7fhgLzzdPNvdkSooXtcB5rV/c/Z", + "+s/ZetCz5WgxecQA/Pdu+QQccHBPKgEPO7nklFw5ONQrDNwhUpGrkzc/nJ4HsNOrZG6B3oQAiBjOxLzg", + "ejFaPh8fBo18SJ4ePvuS5LzUT4auIPgjFFzMa1pA6Q0ZAhwmeSbLg+BNdADNtN39Y0T4QQffPQAhHw3W", + "R9oODcDu1rH+ewFk3zUr75Y4HZLWXt/08ib87+Ae8EYUq8b7twN9v9mF3uWnhyCICAo/wMkZiVnBI7IJ", + "PlkuTMXfCC6Fc5LJmeJLpslVjJqP2e07TpYOefGqsx9XGInuA8pCOo4xeQt7jIHXsog8i1yQKcCNY45d", + "8MS9iiMq1nc9HqBVvls+tvDBo/onuW0iUitJP+sUIVwgcfJctkK5Wvf2ZOs7IfA7uv1kqOltvHK/Gqev", + "X2y54APJaVJIMYd9pXiVLOktBqm5jG/xNr5+8/p0MBxcvDs+fzcYQh8fHgTyHDfinCnajmbdg7kqqPx5", + "mSv2meauT4b+53vzVGwnHUOVGMK/MttMH930JHY6u82295/ddPP3O7y7B88No1jx/SLFokAWqjXTGmNe", + "T5e0qDEPLLq2/1I7XF6RX4oAfBBw4HH7HDwvLFJGCz7FwEgQDZlUaDrT1/+egWEf+1Z9Xit2emswxU1k", + "c08EF80nP+suXOJvH7cCRcd1eza/nRDgnpkSMtfe7i8pIYPBRjtM027PLAT41L7ypze1grVK3SDfqZpF", + "u40kCBmaHYNDV9/Y0u9uZGrYwCR6dmo/eqCLFq5xyxGsPwPSaaNNJTMgwUtPX2IlDY5Gaey/H1oRWm6a", + "DfxqjMbs0Is1kSLNI9M9vMKVa154ABt8Z8DnnRHOW3v9LpnJCYcCicoQgvcVB0xNl3LFxxYoNrcX6hUx", + "9FYKWa6+IZcDMOpfDgD1O8AMOIgjTHET2m1N6ZEGA7ndOiuYfNabS+GUx07+Ok9DgxjucjjwVGR1TKSh", + "SP1kWAJUe/wndRisigo7/6nzHzWKZtgezCb94LcB7bt74V8HuwEM+Jz4/N2QDFCq61khbzzgAdfxI5iD", + "hNJNLi1cU0waNQyofwBWNAwZh+0U8UTlGfwxQaji8WCfnMtZnm01fbTwz7cep2CxcMUaNM33Zzp5aPDp", + "dT3Lh0uyiHYOH2gEtm6uySVWuxykm2weHLeq6K7omWzPMvVWexGM6yiqUwE7zc5CPNyMz0d4u2vBHhG9", + "EmbBNKam41HeI49GDlVJRVGTX5thOKvbzU6uYHt611zkO0A/uho/2tK9/K3DQSFndASi2qF3nxFxT/jw", + "t03cv73kOD3LN2qn44DWFnRpNSbfkkOK8snPoiyO7rRF8FL9+NoppGn3dtye3Q3VLbR0Np7ba4gTtEee", + "uekDWTFB+UiXtCguB/YgX+KbypG9UBzATf5oKvPVZMmyywHyyN0PdvOctdsON+a5CGdtD1SwQFodaYXO", + "PoiBDTQXhjZsM9MNEs0T4NoW2K/w8OXOcLwRkVYinQYjZN7Ci2jOUAcyMWYfyCM3y4k+x6STiMtDkTG5", + "9O+VLL8cuPy3lBQQta3kFLJDeGkJMQcsB0NR4HckTmVrK+BDInmsmUOMAURuuCk/GZLLSMXzlS8FGPuj", + "Ixk0eAjosjQspMH2cwwDuhzUIuQKbRoS0o2CryP9tQ8GQLlJ1Sg65DupyMXzYcBjY7lfDxG/VbCc/MBo", + "/i14XjTJA/3aYFT7kqnVpWhS+mH+8gKxZ9BpY0zODFkyiOPX7psVUhD+viIVUyXXVp8cXgJiXFljLqDo", + "lya1y9vavAEy7ebedTMJHkXDWMUexou4haZ0/1N1JEd2d8tqcf9tN4rQfM+pDIjACcT0GGYYVcQGghjN", + "pY8NuzVDwks6BzyyIeZCK7hgpGQ5pwC31iAib0XZaUEUbxGHttP35y/3qfPKjqlV4YNdBojmOi7mbKoo", + "z/B5J/IO2wnQDRuBuphvfutg2h32YsB5a3aDk++qbbeJwK87gAnh2Al1iTyDmhO6cim+Bs1iVfxHtooW", + "ad13zJ5tIz0mFoVsH96ctW7fuK24YnrCRToNORzfWhheECjqMDlc/tPfPztcIMbX80OS05V+MianZWVW", + "jg0Jy1XaFilbZfd7X0cvijQiFs+paf7kjFS8YnAQrtkqqSA1zGiD7t2UAihMAJY8pdki/sG7CVGi66lm", + "JrhR2qWX9tYW9bUzXPjbUAfWJe2L3niXyptJk0Iy/fT928fhhmmOnEoubxxU9pj8yFYY5WnCsmtIQPjo", + "C9zuxg/ryZj8FcHbbHn3GIOAD3++ePM6XLb8dmnCxYIpZ+OSNwJutw1GWzMM/Y0tPwI0xADibTvxz0RU", + "k2a+RFCl5E1A7e1ZN38Ud7YIwCkKlhP9uXhUp8edmVQD1PYJuFTOBGbd1BVVmsWocF0+9V1dFFawfK71", + "avW382oF+MtPsFgdLMxobb5XtFp8roVpOtt5VQCz/BOsCGK5d1cjnsy6BQNfWPWCOlgEfBdG2TZCJAV4", + "chEGTe34CeQEh4SZUZ40Jxu9QaATfRL322+RaT57Zo89Ahap0aSqVSV1EjSBCcWzBbwpp3yNUHejgs5Z", + "TpqyJGcAXQ26s19+nBIPx25HyXIaWo3eUO+GSuWgTlpvBofdnf+rw0NprRSAgYY0oW3U0MOE9XkPXJse", + "7t09auuhLyNNZywJ6GiVJ1zkfcnL3osUL7nwyYZKWlUu9rZRJTee87RmPIyZ/Mb6PWIrgv3dXD/JxF1O", + "g80111lc8wrrsofibn3c9ZayYUTbWODGxditcmo+u9RL7yAmfXEREDq/HtEGeS5hMn3xIwGsGNB/Ci0J", + "zTJWoQJlSTEnEX8HRBifPmvJlCHcoC8810Cbo2ByzZvcDQDAydFuEKHgYa6UGs0Px2/PhmDUsIqAfwol", + "N1w1+QgGwwHOfydPvpAkK5xU50Td81a5G4tuwwWmOLYL1mhz7PGleK/ZrHaJgONUxogByWhJMmmVSpc4", + "uhvf8l4zsM0Adg0G5kQKUsIRqR3h8tXhYdJIDUAyG3T7XfWF6CB2sachg2bE2sLt0UnYFmNERljSCpxL", + "5A1a0i5FU/8bkksgFMUqRg25sh+vCBeYwXXBug2e4STd+/ilyPkMzIrGXSRiKydE43zX1q1cUqAoQ40t", + "9FeE745LNBvAS15Qxc3KlnwVm1ea0mBd0QeArn6w5DmTl+JS/E3WgNdE87zBMPLVjAz5SbqTcMFi3QCR", + "3xpu3mCwh4fZwdHzP3w5bB70IvcXWhSjkgtelG2vFwnJTyLRHkmLj8H7LHTlCjUSISlIRV1igHfqyS5g", + "lmOJyPjgr+qYwZm88JjWVBkgZXjlU1JrX9Vl2RRAWxiR6fOb81+5mJPva57D7V5DrvMRuSjtXTSnhmpm", + "NHmMyTCeHh7+aA+xfnJEno6eu+Zhr1nO6zKqYIuOnr7ypZ+Pnh5GxSEwqNs8a4o/PfyvUPpSvILwFpwK", + "EjOZMrvtdsq0KFjBdQkGYi7sEtnb1JKpBaP5mDikubAE7LZCky4E38I7Fw1gihE12dVB6G4Jaa9gEBcw", + "CFij4CLpn0nJF19IVDdhqEQxWvhooC++AM4la0NyeYOJhaHVMprazYJhFswzkfMlz2tahFnfZozlxL3K", + "OahuTR6HEz8tGMTDIdhuQQ0T2Sosh/YgaQWsO47PNmarvLYtJ9bTUhm8W48WjC5X8FhcSIpb8l4zp3y6", + "B46rgzBfdhVSNoCIxNAjQFIrKbAJ7L8quDFczI8uxdXV1ZTqxaV4++biHTnwrR4sn0bNQjGECgOvn19q", + "BgaS1koHhRGe3CB/MTy5WxGSMW3XguqVyBZKClnrYjVsBnIpovXRDsIuOPCDa7ulGb+T4eXOjlbVlZNe", + "lnCO8yUVBpIzWkokpwXVhmcuDRceseO+9SCPhbRXlhrQZ0PKB39LgX1+g5QW/HehjictKAHgzBVVGKLg", + "k/U6pEywGClWUi5A7dEacl/mtcKemvV80uarzzuZZ9MAIU3+3knvK+VbzIp88ZeXkPt2PeevHpMLcBzQ", + "5Oz1xen5u4P3b18cvzs9eHH68vTdaStB76WImguRjF4/a2X4XXIa8AWjTmHjXnnB40YAEmkGiTRtY9Qn", + "XcWWnLn2CkIRr8h/kSudScX0Fflf//f/4zt1vz4ZXwqwc7pXdHC4uJJigjl+rw7sv3NWMHt6AjA2Mn3M", + "ZAApHt9+T1zqJaspGNn4aTvNKkbZJOc+bd7VDS0m8Nz8Jzf1K3x7ZCRaN5fbuP2euzkna1g+zKyefO/d", + "SXMFFfUCi+4eiYG1sOwFg8OjBxuurFaX7NWDwRxAi4nPrrZDWK2r0nqI88LZ7vsnMk575MN9TNPHtVkE", + "yMToRdSlv7C3FnkzQntwSDDXSbNgmKDCAFjAgGaYuWyLQXg4qKjWN1Ll+CzSRK1n6rl5+z+0vnmj8hhJ", + "PZRPpdjWViykXjTeu1/C8gt2g1tAzmb+aTf30c32wjZ07i2YvRKRQn0OK3gb1IaU1Lg0lLYGSMeSme7b", + "y89yIXLJtvrHhZn1W8Tz9s12Xwtm/u/5Xvfa7n0BUN7Ni8Xmt7uOVUkxUyvQeyE0H/RkSNHcrGzHUPFJ", + "lzbd5T/1tSG5xtteHu69yr2u6YqBuYQWo5lijIhodM0YduudXIBQHjnJujwcP4vz1IH/iEISkmIOl+WM", + "CikQvNgJegSqpjnJYnfzRIrvRS2uJ1Z1btlonx4++zJK+PD1l0lIO+cdulvqo8aW7NMgoGTaCrv+Ak/M", + "iav1FivFTsxw392YhzHn6Ao3KZlRfKsn5gtX/BWW/ti+Wu9wUrwPXTPp5vqeZvjnDDD1ctxv3SIB2Egg", + "gWG0/c5pmjauOmDTc9i3jSkdohta5BF888CWV2jZ0KJZKFnPF0FldS6+TRRf0gfUpcfcyZx36ktHa+O8", + "wFJpYhL4hXYhkq55JSsnEkaagosruZjcMD5fmO5zxFrWk04eqOGgqqfhBrAb1cJc3zbVGrLFpZ1QZfiM", + "Ziaih82BIy8aayrQSPh1GXxZNxCKvdXDc3jb4gi9e1udyRaQF6R5VULA8IVU3ICpdzs9DAe3o2YmI8wF", + "ggqNi57zvYQwg6a/R5qkVucbwlND2eQz2gQx+JaIdpcvH7pp5NyF6+O3iokcYrPQIFjaAard/TGOXT+w", + "8c31oYFi+frLfUGihwOUY7udC8PKquhFCJTV5LrD4jcnr+2/deRrD4TbMLp2F5YN6W0Ulk6aB4GdFmsu", + "NVpLPBz2SrIgBXcoy1SIhUqBdhkmmjDnu/PLwCob5uGtfZOcO2igjYPt55WJlJ7bSWwXX/aGOLq+7D0e", + "3+uccJ16N5E2ntcJhINsuONq00QdHWCSSajSDhfSTJmA0wGJ0Lt8BIDSNsEvpXyG3NJtuCi1niw/hzL/", + "L+MKk9TkO24x+yvuqemtq+1BMWr6u5OSHr+GxtqYdohjzuk7rWC1V/yu6k0kCneSV36BPoHc2iA64IXc", + "d/kWsM5ayZnvLkXQbShI+8q1vSvlJNLpfTou7vlosEFozUzSTd2zxC2Ab7j520aSXHzc8vDI3s+8Qi8f", + "dtzeVst7bm44mB31zeqPTHHHL8CKjRvP8nnqNPnqO6yhSzq21yKeYJ3oKpPP99uG03zOXqHLT9NKKteR", + "C6nEBHuFRwEJJpb0r83XPm89y1YElLBD7yf3AKe788xey3x9ZhVFCKjNoiPs2jZS+2zS8p/vH5kUk5Gv", + "5P4icm1O+5i1sOc7Mtdg/JxUBRU+a+V2yvLV3rpaDV3F53zX7ewRQ+4Uw1HYXZzac/xuVbEel0mX2B24", + "1KRiqkH62SHJew5hapkhS47vmtBMmxXGWd89oCvqJvYuzhxIhsgb20/O8rp5LuwkikcsN6LpjJmVQ0N9", + "nNVKMWGKFXk6PDw8tP//BN1y/N2ZzueKzTHJeiZFxguOpDGt8zkzmkwh0+ONVNfepcNd0iqIw8MRl1Tw", + "GdNGt/LTPz2E/9uaoh5sdnd2hQLKQLvf2lY2PEgxLYslU7vTB7R77qr1E8m+WtxWuXt/E0RdllTxX+9h", + "+9xwfdugKp4199KWGaxxmGV5zJE28iIqvF9xkx3sYbxvu09cu/vdrj3h7Oxx27kt7uRrGwvMh/Gy7Yzi", + "bo9I+wvgXWt09iUKJcx3CSYIBBTCXolgN8UqGNidcT2Ey2dBWOKtD32NIIer5bv2EwS6hZb/5aILIhWD", + "f45Agz5z4t3jDdJvK68jVKzW7qVW6F8yZGGPILQu/73frToA8uxjjiVvLNELKUYIYdq0op1TS9csklAR", + "mxRcjYOW4qkdA6D1dettz0PgHm+GMUbS5rIzMPqIDBDIaYEZi7Y/K7XxMDfmV0hhvE/AjLnD8CJEuRSg", + "mlUf/Q4hKhM8GTkXu40wdW1NZStYBm54xvZbpkpJS+Muq8ceUCapCbZEfnJiis37Ft3+q5oIdjMB5+Ke", + "RHqsBKe1bma03vkZXjJZm+37GF5x0j9VO3YX4WGuuyTtcub2wvZrxGIqQV/GtJ5cs1UyZcvxTxfOnRM8", + "vc9eRImBkbWMMFdAQErTY/IdLQpM5w2uzj9dTI5PTk4vLiY/nv5tcvYCaVtIY5lR2/FpJWs1wv5G12w1", + "4nlahKIvcCIE5/nIu1vYS5r3GfZh8Pr5mEL+YXqjx5ksHxGpyCMIXF5IbY7+eHh4iHHTr7g4e/Okkxal", + "XXmwNQmEg49ulji9vo5FN8t8tzW+OD05P30XLfW2dXZtN8udxARi4ESIjG6DSoQTwXD7BrbIHkSpKEaC", + "eiLcYXqpwULbIxxH2pNvonXbgyH1iH+K7v4XFy8P3r28gFFePG/hVXl41CNi60OJ458uhgTENfyJoe6B", + "RnaBegRU+AWt2AuWJcAJd3nXC010n/W2YX6BDD1rsgsHH9o+OMn2ikXwAISJTOYupJ/g4jvUCfBpBG8L", + "ou0gSUHFvKZzCMCccYFq7XhXhW/wdHDfd732ekV2wQjjWcmbBnyP5eEl1CEjwcq1rb3IUibOCdmyfVn4", + "v1LG3RfUMJd4zH56EIDNnBoG6PbgnKDSryOincLO1hnZSoOUw8gWLdinQeh/GoMM65ov2cR13OPGG6dZ", + "2LHTpmnIzviwjYcmH3KhkgyACc1ewwOl8y9/u6CaJVUmzdQSnzKjZA9EyaKQtSGVrQd8yL13/nT8cuTS", + "vIP/pYsGdC7vY1ds0nLeQXRTB8wJV2SQFg7O3Uc0YRzJgurGe8mZIIMr0IqwJSIAx3hjBSDHAx4aJgCu", + "eeHyArkvPsFV/K01wvRpsvM7ZxXlkHTd5789TeMqHrfWQzEYBc7cUC4wMgWQGxfcrhbaCcDZW+YM5q0Y", + "zRYMb/UYVIMGWTSnpl4t89YbBAxgomDEkAY5pOxNvwSF5BhJJHafkbpMJUeq53NIOkxyZqfksvNDNYBO", + "XjiLyHgwTGQU3az3Qit44sLc7NH7MNyWRRoWpJlZ3NTanD70nRsMEY2c+bblWUperU9vaWYcSTgfs8it", + "kGRU5SCnzCo4/cupc+FzBgaRUfAySD9YY0qrJH4i+LlhtBbkWcP+NWF2TMWKsF9qWmhiqJoz439NQwev", + "t5KIhVqsNLze+n6alwP/hGEkhCi6oIzdktbGg0u4/OH6YvCg9+LDSk3eHcc+4oPZWdW9s652xpVcoWGz", + "PWkqa7k4v5k6FizFmfAVz50n5TqXwdh7uOWiuyW5WaxcADU2SNwNGukMQPemjBhnCqOazAs5hWyBvreY", + "m/pkNrUIYTaDsB2yGSscY625mE8gqys8KABadPhbsVIa21L7sza0YM1f8OKw6vTnYMRLrsGFFL7UAo6u", + "/5Rm2Snv8ZauZwuDzrWeDKLJkjjSmaysTMInKlmbTJZMk5wZpkoumEstkodgeFxIwbRuJfrwnTUJ6qZM", + "mwmbzaQyu0xhR+azL3eINhLNc77KMEipjkBCCkDUSpTcgV+5VbI1qPCL5QApIihu2Iw0m3HbPeNizlSl", + "kvdtZ4W1/fmI/Q44S1M7vjtARpnRDc/NgizYLc1ZxktajMlrMFm6poKDDzE1CGuPp00ovL87pP04u+vh", + "6I90NPvw29OvP/blc41pGCm+SbTdeYdQTC+KeFFBA8CEyU57AsfECENg+zrsym2BdaTA6s/pDUGSb+IJ", + "wmnAIxk2OIKGdXuNECIlXYUQcPQBNdLQwmuCvGAxOU48JcJdDq7B4MTePf+7Ti0kI+49C7RoImnsUdDM", + "mALwTU3DUzHcWJOQ59hl+ElOfEjgHgGepGH1Zgi8rQGtFYqNFPvZ6b7XvOqRwD6tcu/wwxABGNatnY/y", + "Xus93Ulq/ft7TCruEOjrbg+UzCw5D9uEGZvLog5xe0FFD7wjUo+2D5gHqTlBiZiMEKiMu3b00ZpVMb/B", + "FLs3XEeewtpNGKSYF7oOKPjnBuq3GcbOT2J7aAIp87pLf7quneM5DdmqIgBvRGBBrg45ij19BK4tcjiw", + "2vCisHqDQx4ArMvXdVGsLSIemvbs1w9mzy16j9cxf4u+UxwZitHGJ9M5jOY7LB6QJw2qpq8ZVix4j6Mq", + "0CgRu3Iox292HopYhcF0t+8oDG8IwA4CF7FYAYepGKY28LWKlUO53nmk2MgOI213joOy3FBIXCYYPWYc", + "2bnzSHbsMIIgSJ3nEXCkAG4CJuEG4goAvuFGYVxceSSg9uEZGLlECnnDlHN/AneAojlhIMhcdCdqha3T", + "tPNyOL0ZsotvFm5tncKBpwax4Vx+mCY3mEGK5oQ2GcURNQynXPRwY09QE5d9eQdKhsQmzVnxcOtCipG/", + "tK92XYpa9OovySNkBeMGRTUS582Vyik0/wIcsPschzytR4ZvlZRJ3btLXZt12c7RjJhrwzHWaaRROhsO", + "GG9lI9+i63SjDkWK3Q537Bb7jyxmW+5nUbBq8qrG9YPmsspde/fOZQUWtoSvKZrd2r6m2COqCA1sxJi8", + "Kbnx6UCWnpeHCgB0464f+htIgmckObT/U9JrSBQCSWai4p6et5gEDx/sZDTLmaSQdtR3y0ZQPJvoX2rq", + "Es53HRKhGkHP02BrWkbAcuSxgyYQxerJmLzXDPJFaC7Y5SBO72QU5cJH6WKBCIUOn5LJycuzt0PypmLi", + "+Cy0xoVgaoIHzWxoNJc+OMZELftWmnluaOK0zgqeMyqIj6vHCGK3XJg1KG6pZZuPV7I1aDjWdsbrR2t3", + "DzC/G/DotX78ZkqW/cfAT4c8Dq8+TwZb48OjwfkywN531l+3e62FgQHYncupkbIwGNnvUN7Mjt1+ttnt", + "6rzm9+294Cbp9ItDr4XjQXMmg7e5FbgANFceEcCzAUy+6/KIXPNCNl9KfkRKXjD4Y2aOyIwxY/+9yo/I", + "CtH5IkotB8PBNTpa2GWyS7NKJxN5IbOz/EEYPs/38qhaywHUw9mcshPClha8yMMJ6djuqdJWz7FFRkhu", + "voBUmD2LeltnFHHgw7S8735/KFYPAIYfG6YRgMhcJ/iDdymMJvmGD6PtM6Q1yJNQDEITVK63Nh/pk0zk", + "4KQUDk7q+cH9RJjIGxxB7PKareCexcDo4CUE+CVsmFVBtUk77px59kRsmURve6wdJFdw6lvKCeuN/R3N", + "jlG+Iego7Dq0PnRug1rzuei7Qm7Ree1FvKAZS4fGnLjnm1DEXVWaJe2fZqgzae4YCeoPLcc3Eakc44W1", + "0gteBeixu84S2ptsyn0HHUbB6Vb0OryvtdPWsTGsv95Cb9cbc3vBliom0qTTpHEDFuwTjKZ7k7UBd4l0", + "ri63ibYUBP1gqkFQLIGq4JlYI7ZZTLk9ixp5i1UFNxO4Y1OVODUvHUyiL0FCcs0Gq3LLYcEuWMHn3D1H", + "py/ZTTNoO7PVCDXwbhLdJvG7H8+GScYPsIaqrWxhxtV9+QKAQ0xXJgWvclxVSt7ykhpGXJApywnEz0ON", + "dDeOP/hHvfEOB6Uj4MKpaZH0sCNb1ighZirxAqYYe8R223JlF8H6yp3H1NboylmGl/8fe/++3EiO5AvC", + "r4KP05+VlIeklJeq7lZa24zyVqXpvE0qq3tninlIMAIkUQoC7ECEJFau1vYR9o99jX2p8yRrcHcgEEEE", + "GaSUPdNrY+dMl5KBOxwOd4f7z6W4obwB7feoNyrSjYsK9Uj5LDwoctsRpQ4eDPCfMCc05ock5giGeJ5g", + "7ouRIiyr1HOPE3sEBSbiCZDGnEjHcn7jkoOCFS8V6UjBBNKlVCepmJZzJqsJGu3D7xFyVFzr7Brwf2Re", + "EaVlJCOFIpC3wgAKanUQLSepxh/JvO13fgsr9SGPIE1LU61y9WLjdiB6FHaILjVJ3eH7BXAhlTOp62Vs", + "527G14/bhRmk750iDblZOdEGIWSBr7mjF44kBmfjOzPbhUHWFAZNJQ74Ba3fQ92ePNqF01geXYDq2bks", + "lUkZKhi/Lz79K+xAy5o0sB8agl4F6egzyEIePsjHbERGcNyIPVyF4EcpyxNEZ/L13W6l2G0yzjutdKGV", + "TJoyTughE+Au8Db75ebKAeuEfIxj54rWuIKtuFqtCSCT+jBmHAmm1bTV+1Vi8I1hbRiKO0kIwfgCZ7et", + "yeSyriMmbtvvfsuFwTzIiQOX6LqfO/9bSQzai3xuxwJmZqULrjDYRSg0zHykfKPwdIGSrOfmOWW+jKeL", + "9oOqeUhvCYPTOEw/whVfZ5qnBH3alYaWIp+LsT2n2yX1XFxLXUZ5T8AAK9KBdq3qoPYbiUeg3s11sIsK", + "sppUzRBQajs3bvY4z7kq0UK32e+P1UecMBCe5bAoWENjAzvhEEU7utPQ7S6BvUn2AJIH+PKYcre95UJv", + "3crP6KQX20gvBt17M3WSj3lRWA1cdDAVrADsHnIGLGSyYB9efoJ0vb6JYWsvyI+6doHvXrb53EpWcGkA", + "eGui5xhnGzC4rR3a9sZotYvs4s8rVmj29AnTSgwwe4StQKZ/yNaQ4ziq7vz9vWmdj/hnNm7qxsjM2MWd", + "bXO8qRI10vNffYx2iA7Uukrkjoy3xS0kycfoby3SsZN8d++N32a/S7ZrLzmDpoMOQBBbKGYibwRKNvbI", + "yQXdycJtB/RlSc+LFpCCoTaWeL+oh8elGAd6QiK2b9vdKnXZpi7FrtJZewL37S57bwKPvLo4g3oj+BLi", + "ZasbQcdbksa7SOHoS+/Pn94CJHQIfQsGFeCZhSbRsJU9h9GfvBCtVzTRrQ7uatSrIEmQSnKMDc9q/jOd", + "b+edIoS9/XcTV6Wse5VziyAcJNEe5y3+x+iNFEpIQKvOucRLcLT+ICRLw4KmO0yv+Y4XSM2byn+oEkak", + "mJpwXKOgKAk31IHaqaqtel03aWhWMa2uce+2ST7xS3S7rNAmu+xjw3grY3aMc59pYfcbgCErR0RhiSvx", + "3Z+5Ww0vkYvo2+hYnUiy23PMJ0Es4Q26ZG4+Im1TqWgGldpUvS/gFeo3KHf9uOstyuWilsWG1wyYEsFB", + "wPXhMu0EzG3nmpEJrprbXov1r3oaCSg/3FLDpoKmwO0A44YgXLcoJVVqdeU5TuXDmORAiM1LQDFjK246", + "O3klVqvLxgQyuk2WwpIYOwqe+FMhfDopuul5fQzsU/AvAYGz0iwYLxhHIzWEmDgi8mZ0ivWyU/tVTzHd", + "DyHEFNqNo7aioaM/RqGMeTFeyiyTMdFVyVsmVjpZMCwiEq0owZjv1d46AbJTl7V0We0P6BpxT1A1wLSu", + "KGLCUOhFYwlhcFZM1NfdB9XmMPeyXJYZr6G96xsTO4TONyqlbRy29lPmMaPfG+eW7iXspTaWeySIyx22", + "frCVb4PnRRg33IRRdvQhl3N4PPV2+5qDpTtov+pp9Bj/qqfxywBQ7wbuBdM71SJPgL2t5Mf9bWIt5rDK", + "ZTens2PX2XYGtQ4yKEGnmIvrYY4WmMGowc4zl0u57RVX5AP0vnW8xVIUgg3GXcI2u1DituUtzAE0+1AE", + "qUrkhfbuInAKF6lQ5kbnQX4VvPF3rjL5I3bNeen9bKvBGHKrp5biR3UVD9V+K2ciWSeZoJhsutcqYg3d", + "riC5ILybIYsHj84kESKFXysXTMeto44unvm3Got+NiIfzHgSMCJ4EkKWuGGFx5FjZsIlBUXTboSjl2q8", + "yvU8F8bUnT9NocGbdOtg92CnaI3xIsENhLz5ZuLbYxKu1B59UHk2XXdk1UA8Y1qWiCrvz1GNzB1RF9pq", + "kqUVfBSlErstGE+vwaHKawj35+SQTPQl9BqFtmwLFWhbJizffZlADm4R/D7jW5BWLvI+IuSbLhJgoXfc", + "SN5zkJWrVexGijgCCfTA8DlwT9pe5RsCNN1jNbm0tg6byjHykshJDq5bP0vHwisar5+q0Jvbc5AGUwwk", + "jSYlR6TZmDzYepvFJLh9VYjLgudFa9b/Ty75qs9VXIX9UH5w7wLl5WLivA3FBM/bToAgTNFrPzC5XIpU", + "8kJkayumG/IZy01RvzArqIUy97Tt5FARl7rropXzcu71W53aQvHKbCqDcf+16v73SWdO+y3eqTWhVgOT", + "BFDj8Oakqw11lShgMfXUVXwIT3PrIlxET3RkEe55srvT7SeyDG7qv54KYx4oqUx4IYxLGlap6viyQUEv", + "4b1LuY6jF2zTNcf3vNcRDC+NTW/tFl0okKvoLm1TiZwTWUhFTm/ZJId2J0iPaIGnHr0hoSP/QoV+kTAT", + "EtkhPPRQB8EW4TnwPrAnBJG5a/4esaneU5TGxQX3MZgfpRc05dJ76rQd096+glmwt53lsq1z30Ngi/cd", + "yGv7dNIm7nQgXqVZptVc5GzBU0gITlNFt64G59mHxpvu+G7P97rhkTS7HHMQuwK21XLNvvC2JZ86p86f", + "+Ka1joI4w7WTqNvpPEXtfDP040GvPOZy2q1yiecBmDJzi+qSXUtVin/Q+/F+modHCZ6uGW+s0pCBpEPa", + "dr8ONYbTQK2lAikqFsKIuiZNHUllCsHhpkVkfqQZDPBFbBugDLPi317P+ceSKZontE2wuI9hHXhauyod", + "JKW7933vZeFE55i2jVIExCNUtptFv5lh87C7XdTE/m91rXsLQdgZCTRLWFu75ku+JovNf46hjAzwcO8h", + "u3ujc582F1kSzzKRG0JiMOwxRWmrbM0mblUmHsaFq5SdVi5ULmo7Bkb3dxOxuxjcXtaNbAH6RI1a2CSw", + "oU3YUnBSKHGZmFnoMoM0MkZahXchKvaN1HFkjmGRLOGnpVV9FcAcQPvP2cStmmtc6dit4Na7g3Xvy99D", + "dPwmEmLznD70xWq3ePPCNEP2imwTU55cWR1FpYE52DQ3Gc2uYBrEuzVIg5utw1s10VnGV0DHbqcw+Rqn", + "M+duWRzhf54t8aGF6w4ydMUPola1jdCUfeXsfSxq22RyyGvyLshvsw90pUhK1HxXThGmTDn+fuKV2OJy", + "y3xn2OR2wFUxy9YDCCyZMEAfpDQRmHUamwGgAeSPwHzAf+oSY0OueVai59CQvRJJxp17rXdWs9JHqgu0", + "EBcLpxz4EUEuL+zDZwyyUjRX9KAHOlNjan3ypUYPLRgEkVCFoehR5UDoIOwuK+3kshAQHwPA4UNGC29q", + "qFjTNZtwtf4wm1gpbwLJdyaRYfo5utWSDis3SDz63Dbt3Sft0NO14g60xVJf6uqb2ngjxkrFs/VvsYwK", + "5/QF2iOQ9dtioHOJ6U1dB3HxDmKLDs3M5ej4spzOQlK+60dk0RTcXDNxLTK2LDN4sc1Sw0TdedRKDxWx", + "whajJEHOjn3G0f+ykEUmMJbpdgX3CTM6xyNxJdY3Ok+ZoYEFc69OIcK/ibFUY55FM923BBJqQo4TjII/", + "bX0M08KYSUvpeLoI4TVq8JXxbFq1blJxSzZmOwvE+yhAh4yokCDt52XMvrwH9IEDDLUSZJBZv9KcXAFH", + "rBua1DsscGJbgD0JoAYrVxuCmACXcvhM1G8YdWAPIDSh3EtU7s58KKpsDOdL1CkVKWOPXYZc0wA3ZRKe", + "8Zw22BLcVCCPlQotG+NplaJBpCPl6dAxB2R29oQTZfaZKpcil8kJOlKf0FXWHynashP73z5zyO4n9o8T", + "+5cp+HKFjDCT6mrIiAGg23F/pJDkxtyM17oEr8g+mwvdrzJn99k009M+WxTLrE9I/n1KiZXz9UgFIYeW", + "q6YyF6Dvu4k9BwqEoxcskDttEJc8UtzgGR260whhvQgwTpkCVnwuCeeXncOlRKoqCMIj5bNZYVBlqhPH", + "9BGcHFI3pBiEGAvL1bnY61xj3gl/4Oiac/kfEOnyOI6uRS7dW/M7BzzyM2R322YOwJs2ArwhZlI5E5LD", + "GQYH+frlul8qs/NYDrNae3H/az/I+P0Re087JxzJUJ7AFGb0UOa6/c5UExyOlMOBc07iUs1coit4Iy4z", + "8lbHvYMzVyMNWmPLC0MJaECndFBNpsulVxPe7rZvZeN+3E/Uo1pwfLQSTm7wV2j9BoVscxwhtZvS35C9", + "Cy5exEMLIQBt63hBp0KsSBnhCIBPbQZ3M+wfnpCcE8/kinGm0JsRvvuu/14SzX/f6v99q+9zq/tLq/Vi", + "37yYWMd76f9Lt1KuVz51UPBq1Lhi8vU4L1VsdpuDW3rfeUeDufBYfx7pvvop4SbhqYi/e+8acJsRPc21", + "M1sckDVpj/k2gxOo36qNqKEA1cXPQbbePS6Oc7gTETOO7pBQ/0xFQQFmcB8MmeuGwOYXAjLIuLMwUi4X", + "Cjq8AAQhgmdjte9MpQzjLR1DhwjS+W5bcDeWNw29EtMKbAbeZnrKCPcdJownBS6QI3CUBSOEwGygx8OR", + "8inHHsE99+gRu5FZmvDcrsRr1DXP2Kj3CJL/jno4X2HYqAdSLf3cZ6PeVKdr+udIxcO2i2Qxppm3RP+9", + "kVkhcjYN9gUuUBLlQjBMzJDjsOCqy8CqCAAmCuQTY5jxlyIfKBdE4jk7jttcdlQaMSszNHSKaTmfSzU/", + "jsI68cJOtts+AT3Z6yCwFQX54B49IvIlWgOpZ5XZO3IODoLDkQo2y0NAPXpU2zH/OyYUwm3zPxZ8boar", + "XC55vm7bQphUqVqm9RGIC5+p7c2HExyyixmThR+Hn6ttrF9fXum9K6NSTWvPr5udNqi/MYb2XmvraH8c", + "2yW8Aes0zCwVDokmk1fCFbIXanzNYkz5dRqDdzjHa7TPboScLwpMHE7Z99hUFDdCqMoldDPNjndPjIpd", + "KMyJdC7CgJTwqe5eCb4I5NIR036qkD95MDzfRkxj6TYMtJVH3Dm4ET88GzjbaORVO1yQ6bq4z1pgFpKd", + "g6BEQN9qEB1EH8hgbmnyMzLZrm1Xfq9dKC6M03hossMTEw3qloUApG4FeJzXNKBEG4SK02pGWdrI6H87", + "xsYc9ivemEiBtnpF1dnaCcDGib1SsV9O+4+/1J7AU11ioqAIbvAmlOk+CbeJzD2pUbN+Pb4cfoIsObwC", + "e9dGBkpdFhGoYSoKodzpXBiXR2sN+Ke6LM7Yh7KYa4AeggI+1ELpVNhCUp0xSCZRlaEnBldiqovFGXuh", + "7Y64xiC3fK1WDSsVRysViAjF4j4Ivu6IVPnL40mdK9wizsxKJHImEyS7qEHKJ49sLKljCKSXBRECyDEg", + "F9+VWB+ZY+ysLPSSF64vn/Q8EPLI3nhET6PY0LE9CGDxRNiuHN5hjryMgaWMlRbhVBPMRZ+2CB6Fpj4h", + "u70mV5lMJOZU42wMxYaRu3H/7PnduRgmntaZnjcU/LkttkG/n6msXWYDkNJF5a7gtw5IuciFOGMYxOXN", + "QSKH7eiDb8E6ISRf6OuMvddBs1Ue3eMandpme30a3sFE2hXU2K7TFp+qFrSGz5BqooJkRwJw3hjxSNZ0", + "vofLEghGMSzhzQkgyEhenUWeZR9mvbNfvvbgtRL+2kovWs8z0Wjnrr+90l9EXojbPSt9yDK+5PtWAhD1", + "Ayp90mUh8j0rvhBprpOrPWu91AuR77uEaEdqVvpiqzVsEh64QK3KEPNulQtC64kJk6/8d+aSOEmXu3LJ", + "9GykJrkocimueTaM9DEZsvfipo7w4h1USiNGCh3RwLhbNTUhGSEONmV56VKnsWww6Dcg8P0e+A0g2zPj", + "mLaVXuCp18H5Hckln1slhpep1H1m2a7us4+v3pjj/kiJa6GYnFF8tG1KGvUdeGPj2n9n2LSUWTGQigrk", + "Yi5Nka/ZWhTDkSJGP7HLO+k7yx9kZTRBs9ywamLogSVUathUKgfuKVQxUs3xiiIZHrNiketyvoDnM/ci", + "x3jKV+g+x4tqBdzEVzwHVZcGVOXupLvHOUEQKmeQk5xqwFuJlfZgtET0z0cKDeuqail3IzFkuGUQ8VSz", + "8BuZYe+pNJgEbM4kLt97XTAFEaxhqgKeQYoSZ1h3ntG09JSMf5JkcjV4NKG/koyvJsfQ6KNHTjV99Gik", + "JpPJr0arkfo6UoyNem7go57VXPFhZ9Tr40cYAH4xeikGs7IoczGodm9AJVx5/8FWsmQwUnfQZ9S+6vre", + "yd7p0H905e/6PZARv+URb3aw63zjk+t+59v1ge90XmL+BrPwPXyLaeS8EGPvArxtJz/xQry1BYl/7/EE", + "44fSjVqkmn9yNdxl0Uwr5Mjpy92XTeNvqeRMojGlKZ2HnMc1AkfNEapxzyEBMjYkGim8jbjQLClNoZfy", + "N8EW+iYI2uW5GClISQOP/VaXAfdl9AT3XYcWZ7txP3GVZmLKc8PMWhX81jKrkUOlYtc8B3RJz8IXIluJ", + "3BCPcG2xy7UpxBJ4xYA9enQJTT16dBa2T9MAnWBRFCtzdnKy8J9/NcNEL0/mpUzFyTE285KDBdK2U7eT", + "e+0Ds/Vi2j5KFkMLP80E+/z5rReAz9j3bClVWQjjWtfAiuut5yIRCIPmbaQO14muhtsC5/7CLclPuCQw", + "+ZF6PGSPHpkkL6c/Fcvs0SM2YJ8A+oQhpZyYYg16zrwG88YSy99wz0BX/enzu7eWP7LJZFKtEvzy9atv", + "H9xJxnRl3d25CvBf17Fhk1F5evo0wQHA32ICnbsPdkjudzsyqn+epoYpcZOB8wP6mk8znVw5o75hR6s+", + "S+V1ny0eDxY/9Fkm6cr1Q7DyumGrjEuaHuwT+BNC7jWEmAVmZNfviV0/8TdYuNd/KzETOLCsXBo6S4GH", + "n2ldo3+SM3Yk/uYwF0Y9DjaVUe/47u4czSulEfnXrydyRitXVfqXK7G295dVqO29dHx3d4l/o4Yc1oL1", + "HqmnduBwccPYfxTqz7JgqS5WuV6u3J2Obj2ATW/v6kqi8QJMy3SweplnfwIB5xUv+M+fLvzAq89WqhtC", + "mXGZZ5ECo547feTeBwcPagx/Xc1HvWgd89RWUCmw2BNMvUOVVqqt0kxm4uzk5GTFi8VJoSOd0OIxZrmG", + "8+YCsELwUhF4rICUJikv+NmEDRgaNb3Xqf2d/fzpwnixBkpCZye/rsT8+RQq9IfD4cQR5sQuwtnJyYSd", + "4N8G/jFgfxVT279LKuntHDV8a3wvklq51mimtoG3IGrZHwxwRTDquSM1gXW0xS6fhnlfHYD5EVryzqhg", + "c8GvxNrOgBbs3A/uJQ3uo4+0qNbt0aMLEIkto3ulb1SmeSrSPsuFkb+JlB3JGcmPx31Wu0H8wvqWrNiP", + "DPO2CFAU4fkjhxc/SAmsUkh7wA2DTfDVLVOz1T85WOmi3k4J3grv9G8yyziV8pwBaUQkJUD92hnnOvP0", + "4Wbm4Gs0PehNvW7AjKtrBGS5MOzICMHqlrRPgqT24zPHBbNM34iULbQp2M1CFiKTpqCPH3N5bS/Ai4/I", + "GeF2W+VWLSoMu7z89IbxouDJlXGE5wbK7OpjdJMJbqnHp6fvXriyP33+/JGlfmaFXEJyUGLFT0+ZAyqC", + "J0xo4Tn7TeTaqgl2gQ2FovHU8nBqFciBpXKJz+ubY3hy+uwPq9s+EP+AyMRR3aUQZ8ydF5T8h1KfpDox", + "JzWp55/ccg8GibZsbKSeAW+HU/tZa4U8Hv6JpCYV+/zhw3uGZ4AdfdZXQg0+OB+kDyBMsvfkOX/cyiur", + "LkDHHeIbmudQ8c8sE2peLN7x/Erkf8IE2xKCQP70bFfVVMAaivxPo95oVES521+ttigNzPCfiWhhtpAY", + "FVhBUvTZoo67TkuRCsLisrfGiqNXj3fUIxZYaPb27TsrmzHGLopKUX306Onp4IfT/z8B8OWC3IUePaKb", + "FY87vH1j3vEll94CvJDzhW0YmrXlc7EgyuFJUuY8WQ/9LP8s1uyN4HZUAet+ibNzIiYe88nZBKZzJdYD", + "dK+B8B136sBGjHvClrAp5oxNrHTyyz89/XLGuOyjS1h/mTmB5zOfluA2jKtmW7cSuVVxYIX8irlePqwK", + "uQQuaMu+ffvOqv3GWfohL4YpOMSdUI13tDIGd6opvsAivBBKzGRhQh78FsKozz9ewHOQIcGQMBNgW0rD", + "4UGSK0NB13Y8voZv6Q039vR7XFxo6q0wBtsBLYFCanydd1YJSJzAywbsjSwwjrGeAdbKaDAY5Al+Rvho", + "Gl7G4WmZsCOroB+fWYmR/RMgQctbiCyBXaTMvvCGPbG7NwmYjdVW/aWMB27CjqQqjs/YBfwTGbNZIZyW", + "3SjSNFG7rHEu35I/khN2hErn8Rl7Q56EK57zwkV6Es3AUMO2lFaiD0rShE71xFUw1R1MZhLK2ewWyMng", + "4LZi51FUPnqFZn/BpJiXQL9QEJ0kzti/ciXYK42sOk7s8Mm9W+MlxZhAgfiMPaEfAD/7jD37/nSDE71y", + "CSoV+3T+49mjR54PFUH6yuAMBdCpDpsd68KjmxSmOv/++gxoBVq/pLcpjsN0N8hcFotyChJoobUaYK/w", + "N9X+UbMLu8JLRwnRyjxbXQlzJdXJXGPlmn5Ku+SUNP/cBTJesJaN2+Qzbp8Vap3fiv3FcvdXvKh/SXkB", + "Hz7zubEf/knwpGobHF7u7r5+tdfG3V2fff16YgvYGiP19WugwtFWWVnJiy4VrvTmIBHDvLB9Kr7EwVVm", + "ibOauhjYK8ZWdbSFP+YyEWfsd1+/ruxfwRDeVeoJLBQIc1sH4JdnQ2GJDCsYTNDpyyB8C6nBymqbvdY6", + "s3obHqG7uxdr27j7l9fUKuUu4YWY63wNhkyxlOUSFLz/9f/8X+wj/tvJ00HlqU7XwSgfPXqtrmWuFZDR", + "X3guQdwiC8jkx9fvLt5fjM8/Xoz//Prfrbhv+bhVKu15wtcgdn4BZT98fP3+vLUsPs2EBV+cX74e//zp", + "rdOEQFuqioZaxfnHC4NV3749f3c+/unD5WdbDR+JnNOzrU+aEylFYC228u7Z48fPnj47xhlfLK16Zk//", + "x1wMMMOGSJk3nbnT9e+6BFOWhPJM3EoDtbwRyrAjsHHggPsMn3YA44Gryix+3GfTEkHhMdEWeHELs2Eg", + "m7hs1HibTIbsNf3gq1BMJWYjNhCYWAg1Uj7Mxr0Q1GIwJuNqzBN6G4eQQg0xOgREDurGclWgl8FITSjI", + "U+ds4mx3E7KYXRZihev0eMheEpiaCpcGXdWj8wLryBtwBlQknd4K08cs0wv6gU28ZD8Bq8RfIQizfs/X", + "ZnbGvrJRD01B0MR7viRz0Kh3xn4ZDof40VfBj8Ph8Au7m+x+LoCDii8C79bMMV//AOBMH1DAs2ZI6DAc", + "+lLBgG3Jr3jPjXrL9Rh3dQxjhxGfDh/32enwif2fp31mB2qL37mHBRzyK5GJQrAXYsGvpc7p7P6MF36z", + "u20rBL71dxPMogBtcnQgB/MeEiKO0XbwYUlJFsM9+CXa+JcJywR3HvD+IFFO7VJBmjiR4nTswF9y45nQ", + "OznP0dyKyfMIp4Aq20tparkHkEPkjOJaNM6to3jQq/3c0M4BNSoesQp5ROSkE2qCewKrd/yCF8kiSBER", + "jkHPZlaZdaZtqebCUDIAn5zjK75C9c4gf+PA1x48HZglzygZP70j9fRKKC7B1L/xarTpl7AQEUs+xZKE", + "4RRzsZRK9vo9MKjc9vo9DWzX/oEd4h+QSQG8ivGJEMAf7DL1+vS0FvUr9mzXDrHyQmg++1hGETx5Gstt", + "vP8N5LE2fQb51Cf1Y/NlEk19E0+PvQlOsAGzYhWrcCjIhqRKZSIMmaPRs21CsviG8xIVjqWGjuTErwBh", + "ItAJ2FOtpX2n2Mw6TYPzbUdcYTZ9mMHdP43sEps23EfRm0rPIDdKJosiEwOhUskVbuPTJ5g8Joja+z+e", + "3aKSR9YFfEr2uAJIBjjtjXjvpwN/jfBspIhrTH55PDztsyf2f54OT79MbDc++t/QoI8e01jscKNDBQev", + "nZ6nrQtmosTUWDHaEKAsCnYaqUu+FMyIJVeFTIwdMkR/YXu4T07zsbJHbfUIOPhGQmjkBnWuYGzjgEi3", + "uuLW1wUplVYNc/HCkCqq2rVUrv+KsvfovkZBYf/U2s7um6+k9bVoDm7zaHwJOZpV+2/hjYYyvu0RgPMX", + "nfFCZiQMDaRKeI7hzoxTi0N2UYDjIJhxlhp93suCKXENQD55IRO54pR+ORc8lUoghk0jnhKecsZTiDUx", + "8reIF/iFZS+B94gppxQ6isAbkavE+wLCFDqnxIAMn2Pna7/pMIoZQOl7oxOXNAEyKdIyISB+5ywSbh64", + "GsKMPVz1Ruq0XU1g3TI6DU8kXpYHAHdjZiW8BTuJ48HnZwu35qS80fmVyAdcpYNNunNZdYbsk3/BxnBs", + "8AJy0RYGTRT05u+wTvwg48kNHA5R1C8+AMXCpwP004KX7iJYt4AIYfP6PmA0yGlCkMv2+7DNpX47Atoe", + "fhse67+Tz0aNX3yEqk2GhNtXAX83Tst2+o2TZj9y/CO78qXVoTQ+8rOvXoJc5WLFaen8COwkymkmzQL/", + "ccNlgVG7BeT4lmkmtguMTReXPSMcEcI7Zd63ZpDrTEA6vlymwvjoXUwgAm4tWhFssLmi+PeRqjlxnPlL", + "eYCGTgZOVHhYJp9ef/508fov52/H//bz60//PunTMwVqfApRNByyGebNb9R79eHlz+9ev/88gdAL4+Oq", + "4AS6oY/UQt8ARidxYrA5eC+oAb+xrTovRoJEIj9A6TMEdnQsbaQhhA4H3oO/MrlqK4+ZMllYYWVCQCPu", + "M1ikRwrVKgIT9S7zCMpadzH0IdWycNaK2CJt8zvbZxpQp2UO8A1N6jT+ps//3hNA6tjHay4ek1mVYlVe", + "QuDLFdHj1AK9dLoeqU1q2XAGRf/MIfu44fxp6WikyLbkPLBmuRBgFQ9H5R1F4YZrwi1tjQZ040XYy5eW", + "9PlcXHpYxa2coOm4TxCFrcgLFPOmyyLRS2GY4YU0M8phWxnxEhqFRzf4JFYZXyPwmOV1CQlwANdFPiJ1", + "lL7pmuXiWhpYHFtvKTNhCq1qaTLChFvQ/fY0nZfEkprufEH2ThdPq6fkzoTiRnD9A0vAONHeWe9//nI6", + "+CMfzL58ffzD3e+iee/tYsQkH9wxYjeYfmFd4a67tMRNwacxki6iTirmOU9xCJGcCS3otG54CDQZJC/O", + "BfrncXAuh2sKn7YRvq/rqBaCZ8ViHR8ULr/L8biLLAsILYFbI8scdbpXSktkiFZB8HloeluhOwmCD7Yk", + "2Q0GAUFiMAxKGboHlC4CB7uj+aFq9cI3iulGY4YNQkls3x9OAgNgulHKrWC3UpHAKXqOAqDPgBVMjgFx", + "hfCweyP77yMKeuCUPRYN+dlHrLoNKdOtSh2UB0+VXSSlq5OFXjfBWtl7oARtqliIZVdCBurbmVsSVE+T", + "6BUEi6I9h9fA2BHHN0L4tiisXrcRNVV3XLSWM7WTyqOc1U26Is+Kz0UxP4P+3LkP+NJWkdqATN0WNvlz", + "q4N28wnGCrIgjjhLjKGENKgzGIi3RBsJ2saQ3x5dfnx7/uo1k86PDn4+3mgLHYx8eF7YLJoDf3rxstZy", + "9OWncuIi33GXFlzOKuOCyG2HwX37v/7P/9uKTzcSNDx083IIGJsSLObzrawbHqT/ybN+K+5sRcYrkbOV", + "xteFTJqCQXvsyNncVLY+DuOVf3h2CLeg2E1oe3dsyEss5oPIek4CGR/EbSo+4/djk/LI56PasVlzO63c", + "QxhHQJDueaVR6GV9pxGrQ/G5NyS7N09MmNhKBUe1fYfw2VWup+J4yC7mSrthVIZI6D7cqsf32KlUmsLq", + "keOllaiTnStOxd9haW8t2r3XwA42wgYDfT8O0PJJZIIbuy6nwyceUh15dAA3GybrWA7Zu7Io4QapMnKC", + "2oAVwx2GzaFE2zVwZAAIEYUtSg5ejGdGO8TVSpk2XuCyqjA51JORyyu4vkfiauOEr9BdTQoz9B1TW3+y", + "m4NB7qYC8p2uyVMgE8awVKwyvUYgGCSGt+AKFpLD3jHXPldwt910pYPtpMPSCeTLpRYLWL6V+qzKlcng", + "0dYHlQflCI+JrIpaGETrxOT5oR8A4+QwAFh66AVwGJ4crVFL6D76tBXaR5IEo4VxHDnu4PiCZbnPYdic", + "XJsRQY1mDURwfC9cluXY9rEVka8EnjJYiqXO16hgex94ZGPuZjh4wZZSjUOwDqKKzdwyyM8ICocVi1yY", + "hc5q7I9uYqGs+m82bq8db4fdxxyoux3vIjgRH6tqgdAb4S57xgoC8/Nfrz18Igoz7rxgTE4QVRgcLc/T", + "PH4fHiFJOCYI6eCS2wBkUpCVssEa6QUOfbtkQRgr3ngtVC6TBQDWY7xiJkEEoUDF1Ac/W7aNKcHNQq7I", + "CKhQsw4a6cDeqrUcoAQMZoOibfiuH1HBa0WHvzf8kNn2UuJto6bIBV86AddyjTkcyCF7zZNFmMUx0Tlc", + "AlyFhhZ3DJYC5DyPXQZ3xJGvbifbd8afK7E+HrLzrLqwYMc98CaA1sOBp8bNiieCVUcPHZLh1yF7H3It", + "9FxBrtVH0ux7XttnJAw2iRFeN2L7883vyk7qv0sCAecalULEs7u9wOpWQF5KRf963DAB9Hulkn8rBX3G", + "892ZlmDF97pDN7QhUoCOGxoQ8Yz91Z773ABFAOJYn8FmXCpG5DntHt9j4TI/7P4csksh2I44VwrYBy8m", + "vELITOcdmXo/iSzT/SA0EXJMjXr/qhcK3FbtH7xYcPX1q8iMuLv7+vU9uAGT8+r/j/27LkG++fr1fC7u", + "7tha2GnrmM14j6XVq/FVQyHcvEjQlbwCccmFKbOCsjEXZa5QRKEnn1ad8HBF424LErVna4PK/vG15db8", + "pakwfLkLWlqS2D/wYn/v7JdfPGsOxNMv/fBnFOnqv3marf/sNNv6r81B1T5GZ/Lly6ax5LLgReQCsT9L", + "A34yzQh2Z112oppj1sdxFwnnxuFRc/Z/1u1thtu/gcR2+kaJfOCfBVx/1Ru2S3K/EDwvpoIXW0zkcOVd", + "cwm2y1hOr85mS27WKhk7pPRdEIWHIqdNeXI1k1k2xqfoLRK311nsDKEwQMpNS5lheoIc3lzgTxRE4E8r", + "2TlJpVzdixn7scI1ON6SwQqhn6AYq7KbkguHa6aLbfOAwfkUXJuKFkVgBGmvGk7VgUXdGY3lb0hX9MjQ", + "J8Bzul/w6aahXtZN0S7/u+/KSikcsE0fD089cdOAbnR+ZXfY54fD1FcZXzPpvZDigIGHaix+5SDvWuxV", + "lczlgD9DrlSUlJu9rEvoVihE2JojyJHmqBPDMRF5jp6UXUvP3fxu7IoveX5lWCr5XGloCz75ZF3TsnC+", + "+/ApEZQzVs4Vz+5zHcIhGZer4CAefFCqtlC/Ghvxt1JQ2vQHpHnfj/e2uffsEb7vmw14IZIr+HGMOzsu", + "uMx8b+MWJLnAKo1220Gm58zVMj6JovBJ7auOCBwKki4qIdxpGj4s93Hm54NeufDJVhkxRpPkmOwo405O", + "VOAh/h4qXmK9j+Qi1dmSC32TT9K49f3R3URQ/IREfCebmHCJwyd/ckCjOENmhAhd2CCy7j53Eo6diOkb", + "H7daX9/2oGBXFBazyvWvCFaxe3NAI19AQDjZt8EyM6DtqpoCK3WBD0hwK5cyA5DbXCRaJTIT6ZD9G+0a", + "Jrtkic5zkRRWR57xLHOXOcYrAxJ46APvKQPt2y5/6eFpOaqHh6sxxFpHJF75m/eSdLIh+kQ/7HlPdbKb", + "WflEfM7RtTKJODPktTQQ50fItTDk52T3qPTVJV87RNcKzN01sarcdMJsjw84WZHOA9b8kA17e9k4LxW4", + "oO7yGPU1PmGFioOKPNeRaJ/X9me2FAaC8+UMBBfDEhBXKPsh+YSFoKQHPICgg8l4nutyZR54pSAF+Ldp", + "ejFNxgDA9a1UHNsBvR9/qy4CBbDVw9s5cVfpvQ1Ysze9rLxQi+cO8DXAW1HnTDhEKwguwQyX6Duar3JR", + "QCYpCoc09xFH0X4Z99J8Je2/rJpSBMmucR5I3dc8lxz5gHNFrnSU+4BaV055nV413lXFg3RK34SMlU6/", + "DZMKbt9AfI2JGg0DpZwvQMSJiKyMfGdcQnQntgbXc9XXA3Pz+Hy6eFN+DEZX877xR4sboxMJD08+CGJT", + "JN/LofK+E6u8vbZOKHAK6z6Jb74rbcnSX22jFwLMO0NEwn7NRuQ8sPoo6a24zMfOdjq839pjGNaeWQ3d", + "g5Ov7zR+sgAUMjHDe6T2CDSRTnoU+vgEj7GVfEH6kUjHNeHvIWnA9/DtmJnvwjlbf7sOtC5gIg/cA8YB", + "kOA83iKIv/K2vSp4L5C3HfLNw55jb9bqdD9+cqUrMqtOa7uSV8nN/VDXdocajGlkR0QVj9TGIavWBJ3Q", + "0ZTmdAsvRrj8ZvDq5E2izpLpowECJI0rIVao4t1bv0O21HH9bNFw8YCjbfF/33cYY2lM+W3O4mYHY2EK", + "uXTRn/cduUm4ag6/6SJnCzIoY9gMnJIBFgJTctAFMbANMZ5AIw7Ay+Ucp+vnwU8RzqBcLnm+HoNF+b5L", + "8ncwEFEv3pgavkred+jf1t7kXt++jWyeC6Oz8h7iQdXAA8sH3+qSInPGGPT2+21/2JK7WA4V01xjFPn4", + "IAOrsqcdOip0HrACLVk8IxLvx71saXS3xy1p4jYRVTo3Z7l/YA5GT/dd3wW2R/VBAlyeiW9zOomzfCOB", + "0EoP3+T6hICUMYkr28ywRDEntcAcL+o87LbjoCxDMbsTIkGx2mCanrMPPjK7G2bbYqHvGch8wdBCP7B2", + "19n7D7JUV0rfqHEulrr4RuR+w7PxlCdXmZ5vfWkM4l9Uw7vTLcxfz98+5Pw38YicCTAaL+WH0xYqdaEA", + "Zst5vwXjT8VMKonGzqodEzg9iAAprHrJ8Yhd6BSDcbp87XLMEUvN9Q3w55VlKbo02TrSmHNlbQtXshww", + "46sWGBRGn9FzGX6pplePcTm9bzRSCy4MjgMwwP8OgxD5+FcTs2Fdug1LnbtsJDQuOriDfQXQvN0OIYBO", + "MdwYUdRB98EJDX5+oKF0jjrxASeVc71/sElljCW+dgHD1YtmPfIr5vb9cJveEitCMaYzFzJiNaMqNWYz", + "Q613k65O3YqvM83vZeublVkGmbvx8t3D2ZhpxTgRY4QU+gyQ9AJmAdkFpKLX0cKKiN+ZCuC4zLIBFbmv", + "o/GVVGn3x8c/29KtqcEvYaAxngdO6lHoLzw1bYccEEAx5ZipzrurVD/wGGbqmX6MSh2mBUBcAnh9FZPm", + "EBN84964vxC5eM6SXMAPPHPpb+hkEfYDhd9D7usomQG8yweILtrTt701GqZ+bgkIs3HRUHLO6g5MtDLl", + "EuIxm9zRPbjb70Sl/g583rKkYRWkcF/lXl7arQ7wHuwk5v5enZ1VWdynf3JBgeCNLWMwSPIAjXIC8R4D", + "jAJpZMePuIfUQW8YROWLfE2cy+cT9MmTAc1hNqMkPI5MwSN9WiEDYgzn30p5zTMIJ/qsWaKXUysQ1f0z", + "A7Rl8C4E8sEnCWy5gueFUJdgRo2xDtlrZzkDN9Al/WMJcn7CFbkbLOWtSJ9X1wmlxMpu+NogrhjfwEd5", + "qEjNWMpX4n7bhcw/E4NsRN4Rm4nwOttm+AANh6LX78F5quFhRbGuWhw99oO6orrorOpfyt1jXuC2G/AF", + "RILbAhWYXAmVbto77jycmMvWMo7JFa+kWWV8jcz8ZyVvfXIXq1UsZZZJygnzHDDftKVJgvTAjG+cLbXS", + "hWXYLMl0ctUVt4KGB9MmzDN0jeoI7xepv4m9ul/9Jb+99xhMwcF0ufdACGG4fSNXTTw3gGbr96wEJdWc", + "GrC69ZVQ8jfCecMLEf52l/UG6luM5F2nDnpub/jFZgMep6RT5UNM4/u49HZqTKciGafC/ndM8EKme2Vl", + "YOmvBWLptT17vKwKel6Ap548wYxnFfRyyBDXxGMF3EiV6puNc/f0STss5j1RNX11ovb9KmPAxoE9Y+V9", + "+1V2WVsQqMBVbz/KmPGCZ+MDKnrctnHCVSKyQ2sTL96vOmBJHsxvN2r7DYzeLH/lWTaAC4ElDdxQqfCq", + "Ca8XT+NLbSBgGF/IfR9NTNGut8zGqPe6IzZq731DbLSgYtp9xlcUkkQuTIoprvTfY208f+6+Kspq28jT", + "eBHdfNhewEj1O17bbA8D5jiaY3zQKpsKfNEXmZxbUXjI/kPk2gUnFVhKqnnXiZLN9MBrwIeMjYnhmj1q", + "dnTWO7y1uKvcPgiBXTzVDh+fU9jtQI00hYjdgd6VBPWXBc9TzIQupyC8oEe6NC77Kj4Qm0oNCvzNQt85", + "lbb4Bg7bcoFvdXvbTDrt0AcPuQscFcdvJas2jafr8YKbxX7tkmPM2D/67VO34Fk2dtATlajZ6/dQJhmT", + "26xVAK28OSbpsxY8rK9FnpaiJmT6H2PCJnQbezdzhOFePcCRaMENU9oJSTqHH/Ad1SqqhanMQCc+Vs/p", + "NfGdP8SfwT9jIXtXndkCrWSFfLk5HioSiDntATZWB5zn4C6zoTgGYZrZmuWlUsQ2m1028a1JYNpcmYhs", + "3spgd5yozgyyA4/agz3uw6m6ahTRW6IuWrZxi5g02a4+RG7fgJM0aWuDkKqj1jjrDS2zoZX2m/aGqHUh", + "IlFs0yPbVMR2ObdVfo6yyxZO2KbXtak27RpTXCVp03K2WCy2GEN22Cl2mSBaBeBWbWCr0LtLC4jLuxt8", + "MmrbcwFTTVDotIulFzL7iyrPLRwmBqmAZxhoOM/5EpNbA3hKnQ1WN1FL3NZbMefJuplNF3uBzGI1kJVz", + "RZ8o0CvWkfu0GakR6yMoQlB/DiyLYjZcZu/opIhbzAp7f8eQOcr5HFOg0mshyeGpgEj/uvBeE7qdmWEL", + "tEooJrVfZI6FITiwi/GHdwxILEHhD7osGGQpc0WJ2Xa50mBno6RXy12wYZ1pvF5L9GMgsF4CcHAoAgBH", + "Mxyp8xQeRH1qhPO5UMUnHGsfAlnX/l8+eVnV5ACgorIKDpIe/WJ5gqhW1E/trTSFHV5VBp5pS9UVPeq1", + "q/ieL0UMHxoFj3GRl8ViV2M/QtnPUPSu3/u1THe7ov2IV7wOoDbxLaHT0CnPcC+O29PtzSlsp4URoYYe", + "UMSUcgBubNYuiKM/E4qdbe9f/swQzNRACng0uv/LVZ/lIuFZZv9SaTL/lyvANuK3dABPT3cdxxW3d62D", + "JayDGQ6/bw7pMsGULSGKoW3hxN6cLBXkkE9S2AZyYTWwqC9CkA2NrnpiM7VhPd1YqM/0GNFYeKlcDvuw", + "66e71uSuhSvQGY0+qquUZ7rOCALwVvbxw+VndmI/Vrx5pH42YlYSqL6gFInV2XQcjuR0xpGvxE49Jd+O", + "edFT9HDlM3biCvc38txV020e7P9mKx3YSr+Hebo3l+hH/wSJJQBRlfDd2ZFjNkAIVQ6QBkRpI9tIhPvk", + "cg6xMPD9BJ7Xw47aZAGgj7FMI1t78cogsNoGFbGj2mAdbzqOUFXV29YUhwGJfbkff/4EF29rxqrGIYWy", + "G4l16I6PCkjoKxs0ElpqQ+EoblSxHLQTYV1iSVsHg006VaKirVzs0vcf4eyG6XzOFbjwTNc+mfXG+tTt", + "gfFX746HHvpFmJnIkbGr6sLaceXY0YzLYjErMyWMsfdfJiBpVJ+JIhke9yLz9nT6rUb7qXEQ/Fjxdu4z", + "f2XjNd061tZdqyigEYcyn+dibvmIqRD1KLkIz7KAK296DRAsPYw10jLlzYHPrU3uTmzaltOl8mcOb44F", + "dy+avTZpZb/GqMYeyTKavvBBi3irdRUYqkstsriEA1i1Dn6o5gzzLTcJyjx6ZIkJ2CUL78Mh0X9hOfjx", + "2UgNSCI8Y59IMmQDNgvg5VwFy8xNxdttTU+lZ+xjJWNu1A9uA1pkTJ6E7dqGLI2fsfeWNtAf8JU0YHAR", + "KXtZLssM8X9+5FLZ4ss8P2PvBFd2zHKV64Rn7BNXV/CRr+ijo0g/Nlyrt2/fDbgZwFUfWy78gLY/t0LE", + "M87YhXF3sl8XCtJZ/zM7utH5lWFwT4TXMiu0PrYNhawobAt3CBylGMlb/2zLO/sIln+lha+BGSZ5moJ9", + "GvvHGgu0cYXN34gsG2DCrjIXKRQ0fCaKdVjK/nLCV/bQ55IXAootRLaKDLgEUZR6BCSlZokZTyj9ABX4", + "Z3ZUGmEofRECXsOiJBL/NSbGfcbOc+F/hXT59sK04wl8sJBqgUnR7vb6PUtHVnbPc5DgV2DepL3DlC5+", + "+YPELv6ftHS9fg9XBzK++PlDET9X+6/GwOPeX/WrICJyG3AARpR+zO1QHfINJrwUBU95wfeMPvQlq6aD", + "/H62vQhPsqxw++tBINQg30TnZq8dxl5Lqveh7aar21XGlbeRoLmGhK/NJ6D42r4vl8LfruzodPD4+B5a", + "ZpRt3/JLCPkL0ld2Apb9q5jWKt71m8ZLvpLjK7GOuQVzdv7xgl0JhKY0omCv/7fz8fnHi/GfX/87E+qa", + "XfM8qgsIlY4D9IeoN+7F5QdLEuIEXt/L1cpneQLt3dWGMmwms0KQ2+Zmb+g7PU71ksuo7wIW8FDMzv3f", + "CObqdFcQADtpe38fFKQ/fNBOVbkcU1NdslBiUUzw5pN9hkT4ZKcJhnJWVkErZOjo8bLQvQ0TDOJZL3Uq", + "MMyCzAx21q9vecBSqboSZQ4vKVdifaPztO3RNS/2JKUwYdh+pLR59DYAmDdtrfac4FxOfCjAjZgSwPfQ", + "CgO2iDTMqodLkDsgy8Zm4T5byPkik/NFYfoEZOukmhWfi5FyQFnQwqfzHzGj39w9ps4yfWOGKH9ciqJc", + "nT16NFKPh+xSzhUrV4wXHp9d3PIhlyP1ZMh+FIU/6rBhKTeLqeZ5ik290ok5e/TIV7US1tDV78X5VVJc", + "6rzwLyZ75NicWsGoGGN+AXvvtbFyihrB8syXZ1i+cuCpvtxwM1JYfpDm8lqoIfuz0jeKkr67QztSEwjX", + "ASTIccYLgfApkOkDnwDNpO9KQWzR2Oi8GCdcpeCMPkaHRygEeS/GPMuCz4nOMmHblktZNEphr9GmDGIO", + "GDEHDEAArHCfLS2M1CSVBn1SRDpeitxqUguep64YGuqqd6BdM9jjAYr2g961Mo7p492xLxXldxUpzdD2", + "Z099NZiWMvd+miJLRf0FDDNe2g7s4Qy6ruA60KYVtVBtJcmNt74mcTqyQ6hRO4aRonSC/uKYlGrJ4W0Y", + "tmfSZxOlFawIPNDTryM1CZcNNtFXgMAJ2l47CSiO/1QaVxexQmxh5IzwASZue4EKUkGMxTgpc6PzMc9l", + "ARUav/sO3O/QfFGubDNsAjL/LaC1BFSG795QrfqRWIDzl8GZIBSeKrOM0gRBJVgSK3iNpwISh/sWq/Hh", + "dzji2BFAx1iVaazzVOQiHVv2aqtAgFcKQWU4/FQacDPZWOiKPvHoscjJC8pvHLzYLkSFTvzqmGEqCi6z", + "VsorjcgHMw6JPCviCugP6w/ZhaJUewSdjCnFrTCsALHYljKkQVOeX3ssMj03fVbkPBF0RTnsm5FKxbSc", + "M1PmM/v1gebbGtaZFw5xKYI+F7B8eInmM4H5wlYac4RW46J8+GNedBjNjnMfXXGs85yBWuq9AckrgO6q", + "gt9qpZdrSBaOv02Ghy2f9wQkxvvsyZOQxz578iTyNr4jQSoy/IrregbYtkBtZNuyvX7Y0afvRiTyfgFN", + "77UaeFFrI/TSkrRWVVAc80lNrYReCGVV+uqtv5FJvyiVYAt947HD0P8mDWG+wBMgGtoa+nk2JKC653tD", + "ya7YKCNp3sUsQw3ElgZHLyYVzA7aq261ajpR3+QHyXXZcGRvxsDhsDHdxX7jfZDRLfnt2D0aAevfMsiP", + "r96A8G0YT4P8o3acpOQMeKGXMgmiyFsXeJcjysE5fuCcFBgd2eKm5KW2Ta2now+K6+DCB3RGVFFeLAbA", + "b4UrDsgWPLkCOGlftYlpej74Dz747XTwx/Fw8OV/RL3A/QDeQTK2yBRdAR/qvB3iYHM+kNPcTnjcKb7d", + "tfAB6rggd2rhPmO4USIfOytcEOPudPCvdy0GqlX3Li+htK224CvRacKveMEvbWk3VazqZro5IPh8LXKH", + "k7Rp5wBZtqWBJmnWt9fNt75l9eXfSse4a69EksWc+sApt+vK34Na7kEmsLz3XlZatq7r5eKanWIHFIoj", + "cf3isCkJGO6Yyz7Z6/f8XeiRCXoE390LEe57BIN3DXxxmazGhdYZusyCSR7sHj2HbzorVUId8JUcO4cW", + "+88nfOzK8rJYjH2+d2czAcOYv+/HBTdXeCmDVY0ctCFR2W0x5opn699gUPBvimUVeViFp3xVwE+JVgog", + "AdwkvYPyl20s7tId5YZz1j50Bm04MqtTRPN1IEzRbPuAvBMuI1gH8tlKN9U4ArJJstLgCkE/PjhDpON0", + "2ro2VmW50C9xTYmnNM1HyZWIGUlf4AeUKRLfQiNF+Eznw70Ms57QNj07SlPopXetYj9/eosLXHUXdYVd", + "aBMb/0/2Z3uXTrkRtrHNqWBUghWp9pvDKhczebvZ5Z/FmuG3TgNf5brQic66EChu5EdXo0lUvqk2wmrU", + "j9j8Sb+9+GDVVWWsGs5cs6Fhyjy1PCnB5ISZMGtTiGWv31sUxaqNDunh+WXGjZGzAC9722NZnU4zPm2J", + "pW+/Td370y73hjhmBfbYsqDRGV0iH984Y4v1ShcLYaQZ1+FW3CX5GaBjUOe0B+zr3XDjxeD92wtWNVQH", + "ZQHCtsNOvWo8ZD8bwSZf7yZWSS4WYqS8wWgAU2OQ3myhs1TkQ2YVP9sF5QjXuWEScvxj45jVOTRfV+sM", + "rcFEux8hyDIz9pvaAd8Jv7ikoMVCgPXf6hM5BwhOHAYrV1ZFmkD20Qk7AgVpPVIunyyhwWBqWmoMYxRx", + "UZZCgA/nhJwHh/7ddFKbfvB22kqAkQyo/Ra1iQaPA3JPHvDsTJuV1ChtyC5gc9KRAk4zCdZz4pIg9S0X", + "ykVsKhiKqTOiDN+tEQXOcodrbdtxMTvOy2tQ4SOaSA3wpBbQ3HbsO59uehaLt+88brff2TgKKr19hheK", + "nEcjzr14621j9S8XvHiHVqOXVOOu35NxNJiuXgebjsFWBKsf2B33D03us6240wnUzXX7QqF0vN8lUD8E", + "h8ygcQVFBZRCuv72bJyoO/YGnrYEG2cHz+ST0xAi3XmvIrMnbdxt37LKn7r7ns0yXoyVyONRsc45AESl", + "1AXqtpczK65MvEgueCryfRbQlg8cv736tEcTWCNopBaE0UHw2LLc9QHutehWpSvzOONsRyVy6lj0Y7y5", + "XTPIDhD19uHsHht8v0Pz2mkhPlD8nm2sVx0sB+1Qu+2N77VyFO3sYTMjeDPpwZtYSbdtkZsEBapSh9dL", + "Uk2ilSlyLhVzDM9reWbIzl1pj8DHHd7dhqm/2u9tWHW1Td1VkHZua7FD9tGHGdVnABEdh/B7FCzQGn+B", + "VR8/+QNIau6fETHbnfUdq9A1UmbjKgBBzNFFt6pER81FdWHptEC+3V3rXOfA7dQJru8DDKhx2cTAZYgo", + "MhVpuXKZbjYpr/osxrVr28v2tQs1uJeWUo0rGILglqtC9O4TZGebdwP6Jh0YuZQZODG0Rhj+4ft7hQnm", + "PLka28sHDYqdVtX+Ml6WphiDJ1CHOru4nFlpZSI2RCdi73leP7TG4bVfwNrLxD6rom8valvxWWr3k/HC", + "c+ejs9wBhPluP3XVVdDd2/lSZCIpDLwEVrMKAsNnfCmzNaSbdSHgDCVqd48s+XqkeAKAqmpdHV1afGZ0", + "cMFUXSRcOU1fFmakVjwvZCJXHIM1Sd53mPvgbpATNUTjNB9YCXH8sKMq0uk+q9p4CBWjfYRdFI3O94Jr", + "a6O7btqJr//G+cBoJcjpu7lM+wlV9ih234HqQn6yuWBO3PDW1CIHdw7ysM2kgWcXKP6li8dtfAm6HdCG", + "ogZeGg+4j7gPuCJhQ493sSQayXYehLaIzfcMfRuzRAYicCU/PQs361lkszoZh1qNQm8gAu7DKgiy7OZi", + "0Gov0GFb3lN0ZQR4faYiE4XoRDb93htRhPERu9y2f/701qdamNmqUs3Bc/pnIxzc+3emXoQguVN9owhS", + "DCQu8l6BdIGX6O9nIKDrp8+fP57Y/7nE5xpwTzMiKa3s4XCrIWDMln33lvxMjojVG+b9RwGa/FpyNtfk", + "VAquyBDY9PHVGwYPF0FFWx4+XiyxSbwuDOMG4nDYz58uMCzqYwZ5DG370Ij97fIpjvfIKzDm6TiAsT+2", + "63Tp5jETHBimc1yFqZ9nmb4RKVtoU7CbhSyE5QTQYS6veSHYxUc2zXRyZVfx6PLy0xvIfoIw4zCyV7TM", + "mDEEHKVhdA5WwVl8Izcax97HLS9pf3XjYXrGeDBSxZcEZRuQxMWMieWqWPchzBSaJPhxrHgkbhOxKtjK", + "T8wcD8Hpf7nKxBn7ZdRzDvP2h2Gil6Nen416fCXrP34Zqb2e7WD9xtTvWK5i4nvjDRTATh36F7pe+e3I", + "uQJaaW7GcKSgnkjP2OMnvx+e2v938oc+e3wa/P37J8PHP8C/Hj/ps8d/tP/8A/77h9g7QwdXJ3ghIXct", + "l0Mlo4sqmOj3p6enp23vEO4AYz2IfFxwe0hEbthRkZcqQez4mYfnO46G5aLPGJIkJLgJHfQ8WMqzP3z/", + "+x/aR5PWaFoqctk7ohbO2OPT03cv4gOon8Gdhveg6N7IJf7oOTz1GmZJl0BjuCpf8hUxqoifGGRhJRfe", + "xBd0yb8xDsVl3YA5sSVfrQB73jJdPSWv5HSt+NK35J4QImj0zm1j06OSvsCDI+Ge3BYnVZI98vfs4yM4", + "PLInLb4R/Z73KzQt5iWXUdZNC+PBXRIgfNL369GtT4GOiS3eHT4ToHD+i7BUfrY4joFbeboLghmncgaQ", + "dkWVOCfTc2hyxYtFdEgtftsvtUpyUQhCo8KR2EbI9UTpGzVkH9EvEKOa3Qa7B2LMHKKXsoDFau997N0L", + "N7YAmxz4N2dQuRmV77bm6MsD7swQRNDio57K3PmZqConD6tqMwiQcNFK7mWaVTqdqIXRcJPQS3pUfw5G", + "tdJGxrO4/4fI9WDKjUiZK3TP0YUO5S2IokvC6YMz3JIKK74vtUrdNicSLRLzz0FgZyp8Ygt75L3QQzxw", + "GIh1Fj9173DQzAqMzWnZGt0mYo/GHnQMxQ8g5qatqrZMFXN2GWZMmSysKOlCQ/qswfX6bs7ePwQx5Ihp", + "j+nrsBVICEA9I6IbwNJRfpgqYAsABtEVCrkLUuk1zyWHVDbgtY+1rOgWXjIjRbcMpQhbr6BpnCAE+E36", + "bEJBqvbPlBfC3ov277nQIK346DsKmuVmvNYlRkQ9twcLszwpXTAjVjwHZBRi/3o+z1y4jJf93HH3qcMw", + "GATsWiCngR+lRo9G7SAGoixhw8RhmUMmZyJZJ5kAoNvorkMkg1fQeMZ8nSA7TBBEVxoxZK9EkvHcVszr", + "idFx4atcPVYNk7kpKB17GDpTGvGc+Qg0V33Br8P6CrE6XLrY54wnicgoxiLosWoHVLWKbzmOF/j+Ndle", + "S/gi8F+cpvPphL9gLPCX7xTAT/244iHOFM+3DQJhk7SlYW4MzLXgZuRzPY1UIFiBNXTIzlmis7Sq7Jvb", + "3JqRwt2F6208XePGJJmECGSKb1FCAIzkSmdNCilNm4OR4/7bBFg8lZ9tyehzVZ1RBOtY42bxq6DlEHxp", + "E2SJl3+OXlpvAr5hN3tFHAlDnkTKJrcDDpMZYFwmhgc1WPdIOfF2yP718sP7AUllOrfLLVLGM8kN8S9V", + "QdgU2hsrRgqAQ8xKIxA55ZuGoZ0RctAJ3ci2nkLcij6zOwPZwHCH+iPlo+lNwZfgf+a4Xp/NhR6jQ2uh", + "mWOAGAtoP4EnOCv0SM2Fhn8Ma4gqxMwWxTIM/geAAzsa/Gsq0F3aRcVUSuMUfcDdcOjPHmo4MFrkjM75", + "248W/3Z+6n6ktbxaoFJPwWwJmbc2uXn0CAMFXHqgq2h4oAfBmlp9ijOP1ODliiYM1rwljapHwKqpkJYZ", + "w303zcSAlF1kg8N47h+ep1LxuHIWxpipZqZjCpU/upac/bReifytnr/V8+POcuAthuK2h1xV+KJEFCeE", + "3IDToTfCtkYIHnh3I8ARWjIeVchdkJsXLHcgH9LsN8XCnVOPq8lZIfLLlUgim0BQFZ5U0HhZaBDm1vYP", + "pE+XvL0XNYC3MSxQ/wrtOtFRT1i8/mPB/m+oGhUAo5/42xl7/beSZ2CnU+KMvdcFE/6XeXEyL8QZ+xFC", + "bu0CcgXYMlAEjH5ZcZLZIm+FMbHviVYFl8qcsZf0FzPlFAdMWGEzeXvGLgueF7hvAK1lRaYz9kIUN8Jq", + "Mje6ImOgE7PQZZayqWAgJ7FfllL12ZLffoFupTpjf4GCUjGwHMbrHddYnfgb4JxYdgPsB/gUQAtl8Keb", + "TM974FfCnYy/j7YQPe0GjukIi/fp6PY9X4cUsjA9gMm0PZ1I5ffQHMcfLnoBHbgRRK9KnWX6plxdFmLV", + "3RDv0HmBP7IZtDEoV8wg4Ddgc9WMQuDgq2uwouiiswCxQ6U8T53lqGrOWw/iSLWeCTibWL+VIVRtghUV", + "Bgg3IUY1hY4Cz3Zh7ARJv1KxygXYIuOm25eUdZM0sRtZJAvQbJgpxIpBqlZoLFDWcW/st5VV1VTx3NlN", + "qJa92BHggMk2gPLIPudCztVLnZXLyCNQXB3G0sh1SE6dYTtVnFGAzbHEaPFNXbvMskpa3uFj7wHEEuwc", + "LPaGvf/57Vs6/sOt4ml0BvB8YktAQAI6ts/88aMXHSTI9cp2LfJEajUcqZd6ubQMHAAVwczWZ0QLfTa1", + "9Fz0GTibBEfWizV9VpYy7bNfjVbTGIrLzkAt8mtvdfaifb30LmnNQ2Lnv9UjiVbI58qv0oLV95pdzFz6", + "2j7VAsF2pKw2O0ilIV0KT/xEKrxdJYB9WLl4SLUmlSIZ4FB3cguok3FEUU5NzORit58cA4GYj4L4K1z4", + "441ZW5qZcvC+cA+EbPK7r0ZYle7sSqzBcns3Ybmg3CGEYEkevJXd1VnEoSamo7bMkAl1LXOtwK7sxMAN", + "CA7f42o+To2KBrKutCnmuTDjFqX0vZ0zIeWh1mkFTylumqf6IzZ0+W9v/fQ9hmVzYAlEyonc9LY4NDbg", + "UO3honGEy+wzC7/0KYXAijzyMxv1LEP02nxdO3GF6rhE/tdOjpRIORtLGT1yZZZ9FrfFOQXCXqhU3FYH", + "cA+cC9tKybONJN6UYRs0Ut5MAN9HwZYrpus+r4MMwE6rJEQRBcXF7raBoSiilCB/feEGWcsQTkMdsvMw", + "67thy9JA9mfSO+xQicZ89vodnistArBnWGj8MODIVYkX0vihfWdYLhKdp42k2/1qGINMXIussqNItRC5", + "LET6HC/iqbYrnIuqKp1hNBwhJIxL0A9bU1/1ffx4/Z5sozWgsUo+i+9qy1PSJUZwEf9LtLoWSlp+Zc/g", + "csjelYTGCpiMRl4LcnqACuZ5aLlwLmmYo5lKDNkn53OA1jhvwZLCDP3oqPSfrKCEqcMNIe4gpYOZOc8w", + "qdoq02uAh3+olODbXrfQWwSJgeyfHP0xhuyvdNgqwid7OuYNJH9C4GjQAFG84MmiqoP0+NwnYtBK9Jks", + "fHVe4ZCgbsouYXZO4QHEQHvFAYg+zhytSjTe5+5BpA8aL1p2lFbIN6gVeq9kSjPL5sD35QOIlpoAdO5B", + "1nsAuYgl4HhtMZ5qIK+lWOqccogT4tVBLgnUL9FfhALirA45nPEkUeg5DO85hWo2dteSM1dhPvuAcy8B", + "/GTIzuu8GknGQX0S0tSCe7t2jU+hgMI7st5vfii7yWtb7sqaF9wPz7YFEfR7OCn6jKrWgag7tiKZdpfE", + "+Qae8/XOfvnFE0q/wVm/fGly5MuCo69SgyGbtUowAIbycO6Bx2zrLnKtdOlIL/dW4VTyudJglBz2Dk9t", + "MeXJ1Uxm2RiTvG0HckbOYIkbCmdrlotpKbMUuA3mLoQ/dc4Sjh5YrFwN73NU/QAx2Z3PgbjN5lclFHEH", + "Ni2Bh7pcwDTsLrl9DxipSzYYsfNUXIAXlCofFc4ggTW3l4DHxgeh5nR4ajnh4+Gp5RsAHkeb4dLU2k/h", + "dFJd4mNKPbrhgNm0PDB+CF4W7YCksowBXxar92Wervsu11DfZ2Drs1TMc55aacqq3/DOsvmY3H2wQGzj", + "chVQ8cEEV7V1SBLlA/rBdJgbDoMHtHRIitk9uumUpnTbsUQz4CDTc+ZqGQY5A1HcoAzdQaZluuRkluFL", + "JXY8fNiDm0pzNS7jeLSX8jevpxL7oyebBx6DTjosoOdrAGecCUxaHShUDzgikc73yey8R8Nx+OHXYTpL", + "JmfASwxLwECvNMgzHk/7HrwCM0uIdDzPdbkyDzy5WS7M4ts0LVXCc8XjPmEfVtwq2Q6U0pFsKoz0IiQL", + "Whiyl+QIsOSY5x9UCMp255IjTcsCZUm7/jCaVS4KyH/toMzvw7ZRGm3xJZP2X5D2T+feIofzQMpw7kGB", + "+adyt/lyD6VEZsIUWu2OMQHB711VPPBc+yYUoHT6bY5kxyTZm5hWiHgT5ezM2YApfazj7lVfAaN/YN7V", + "OVF3LBWDH100rX8Ur3jz5mrCY54O/sgHsy9fH/9w97t7nJidScq3TagqtsckvvmuVIjHDQawjV7IQemM", + "JfjMEuohNdkyFysu87Gzsg3vt/ZL3QU9oqHFOdXc1ycp5qFUuCq9Ri6upUPqfMB98+J9J374yZW+xI2F", + "BtzudNQvE2/8D6reQ4lEKug4fFs0HDsQkCOqe6kW1JY0pvw2fHyzg7EwBTgCPcjIIWFFY/hNKEpbkEEZ", + "w2aQxWW69ijzdB4HtiHGE2jEKubeH6duZXz4paFUEgTAfs8lsTpQ90uSTr2/H6lmTYZ35p2Hn7wdq1cU", + "q0eH9mTpMFgQ/qAaK1dsKmYIwBccVUwxRwd2eM9DikrlhhK7QWK1ddxYPBx1Dnb3Qlu9IRjew64q5pL4", + "JsIevB2X97hvqgYe+MKhZR6DrnO/MxS25G6YQ+9n1xj5yjzIwCp8n0NHZQqeiW9DIXRSvtGtX4h8udsk", + "4V9ERL40FKvispA+8HEDtGZIzLLfgQiMrrbyQ1vQIZ3U2IURdLRKV0zrYdeoVBCCN87FUhffhOwaT9aB", + "Ch9/tP7tNy86crUm+IuNBCZVXrsoNk/lcve1nodu08dyI+Xb54VgMzcMZxdBNxb2kivw/VTOkcuqDCNo", + "etQbugms/83lRm9CO2izE6ftBRQK35w33RzCRdr+hOYKetDnIKB6y6J+//2OZYXT3gVaIl9G97lKrL2f", + "96jdi0AtNYVYDRmgD4N/o3PKdK/qimcj5RwP8Gkklka+NFamxERC3GgFSA/G5cyNepAuuFSx52AuIbHo", + "3CXzB1/RIl9DrpMckIrbnCa+85W+6/xICh2+leoqDr/zX9zbtB+kcB/Tarfiqs9LmXJyPQlpoACPCIy+", + "YfDojS4MplgjzLx363oBLiyJNITxJ5KFkgmH7FiYgy3RqXA41obQh11G5mjwot+yXRv1oyvoM7P2EHl8", + "bNX8Vfu0sRTDUjB3rsyNCJeg1wkwpTmCWlLZCmlo6yS0nmdicyrba/1F5IW43bfWhyzjS75vLfS12ruv", + "lVDnFxu1vmymzw0RRjttt0vVCqYBXgjMALmr+ideiLe24EuXiqIjeMYmrD72/mXznjtnpZIzq1AmG2yW", + "e+q6Fuz8grl2Go6cXyG6x6ExPzl99gcPDtebr4rBs+HjXojK2tMrobgEIOolvAeXueidnQ5/fxcS6Mdg", + "ketj9kX8iAyTdixLDJqbrison4BFBEqykwbmwt5vvX7vGqiz1+9pIDgMfsBBoqtHNC7jR6Ff6BLi7l7o", + "W3/fN0er5zlfLWSC6WnttTLVt+Q+juyVwmExrAFj6SBxnEl4xnOW8UIWljNZhpVpNcd/QXNwtWDwd5WF", + "nRrHjI7DudDj6VTfDtlbX5dgX+AZJ9cGr0oOC5TLVFoJZ00xSGu4CNmNMIXI1clSqmAM8NQqDZuH0T0Q", + "j8qpOL+tij934cN2t/psKdU404o9/v0fh9/D5Cwl2V8G+BPMSxiGIKtMCZ6z/3EyePyHU7DXCmGqtofs", + "rVsmWhjnz0TpHvF+hT55wUbl6enT5E/YI49e7wdKarG4K0iGVSxcJJNd0SqiciMqIYA0wNFFFASdFwuR", + "q4o2YNJWrsBJ1zbEvc4BreACxP0+vPD3xzBT9OCPUchO3KvImyxufZNSh+wdX9uhZT7MyxEAXmgV8Q1u", + "ckiCag9KA2tic6iP/1AbK/wzii8aXchLXUYX8v7Lg3OLGMtEy/JczOq7RguMbseWZcBJFZtn9f4L1BIB", + "5latmk1FkhUBxEV7zxo/6mw91+obq0Mr7AWPVHeYyB+F/liHv25JB9Bov2XOr6QprHj6IJNNqTFMcufk", + "18enV8ut2DubmU100vLM9wuQV8aLL6GqscsdrOZ3+WQrVGQzA4UbSTC5lpX86HDDmwl0io5DzBq2rbaS", + "cSlZX7osa0156Ueh//Xyw3u6prEOaZ8SQL+tKIIiVS5IJ2pBY060hnjsog6OetcOuNkpECVs9suW2f0o", + "9FIUMTrNA8R7JyuBhGlEAqjWQRAr+qvHUS7cGu44gbjWzdm4SH0/mG1z+cbcZR4sVZfJ+KVtTso3FJ1M", + "pqc8uyx4IWZlBlMKENgrPW3bCKK1N1WYzoFePnprz6iUtvCniAZiaMhsDvN3ZrbPMAbA1CekE+PA9QGu", + "BMwPuS4LATY3VEtfYzq5fD9jEta1Os7Rj6ASHDOPDxGqPSN1/vGCXYk1wHJO+EqOr8R64tzwczb58fW7", + "i/cX4/OPF+M/v/73STREDxp69IiwItk7qyuZs0ePGKojA9/14PT0sYcJ7LOnp79/wlK5RCjOR49e6QSq", + "LYpiZc5OTrgczmEiw1Rcn1BjfCVPUp2YE9+qqStvNIneWW+ty9yWH9h/Wg69xEx+vTPbc6DURYZZ1/BI", + "r7qLJfSAvr5GzK1uE3CF0czKMwNPcUYgEmp9eeOrG3HWuMllIT5AyIj33vSTq4Ei2nluDs2Xdmejoo5r", + "SPPIjn7/wx/67PH3T38ABxLYqlwkerkUKhXpcRyPo/1mDhbkZaZLq/YRFMSRGM6HffZdaQaJUEXOs8ff", + "HftgDYzyQGMLO7/oVzF+9nckbrvEcW3DQbpXOd1a9npzsCrgHNXqQIsUjlMP9m5pOAZSBoldZdphkagw", + "u3jFjlom3q94SG2ljlsh0pwRomE0iF14ZFbmu5Mhuml/cjUqiyClqNmc6s+f3roVDs6KzzTpZ9xH44GL", + "w5IzhCBDQBGixeplJ5c7A1z9KjgSiV5eUbPgfgw4bm/yTDmmpO/DUjpxhoc7k20KfZW4LI6EU4HvYMkQ", + "aCLOR9oP7pPh94NZxs1i56EN1n7nqQ1bfagT+xDnr2ZQjMVpQrLAnKtULxXlng8shEenw9PBk+Fp/YRs", + "ZAJ5sisTSJUwsTZ3vRpcMWOXEsQKnvOlKDAneuRpTa/Gq9jDcJKJ0rQ0c3j2kgOZzjdiJTlfLc7n81zM", + "eSH+4tBvtgbGN3PCR4PUz7Mbvsa0kkP2GnDuuOuGiVuRlJTDQ2aWNkwheApwUACkjnZQQBPkhAkVGJNt", + "o19ir1st6D2vRCKXPKNgXGY0A7cJK+MuOdhYjcmIRv+VX/NLqBvxEI7mzW8sO47BoUnuXnPzSRjgI3vB", + "EbwkREoCiPRLa/DRlbOEKw2PbQxM4uzd+eeXP1X6RYOz++qHZnOI0dFdfzPdMUZCA4nbGlW6dVPBBJA/", + "eh1sryIeyszSCHfmt+FgKUR1axKJILiUVmM9sFx60C0/VWP8QAcue3gUryGNBkzg7uRAA2joqKERVxsT", + "1fpdTOvOgcIJRErDQNiWJPVBh675L4e73mxQut25PfnLA9HmS3vi/VC20OYGXSq+bBKgXUbzX4XsmhgV", + "MZJpMp9sLqY5l8nHjEPoZ/BE3Z0HeZgP7ppjK2qvSloBzgNA40XOr0VueDaMPP+UVnUb+yKdVuEF1vrs", + "Kjm5/q592pKbBiHsR4ze3W6vLUITrFRJsRtWC4ZHXB34xgBivcDISSEuUhh3pbp2rdZuEI6AcZd/AhxM", + "AUhkWqZzUcCzn71w8XluAtsydk2MsdDYZWmY7LqVYS1aMM3q2X1Lte0itJsCwcrIO/dhDbbqIXy6dl6g", + "kfbhqWShc8T/cygNEIfw2kVJ7iMruTpewIYt4NDHmF6SXVIYysoZ4/sUhNmSIrggbOSqHxhMVGKqgnuo", + "6LMnT770N13VdqT0pmZc7Gg1wnA87WtMHtEv0SPpIGb0qgbKYtiSp6LydAW9DrgQoQAH2Q6KRa7L+YJN", + "xqlOhiC+TSLpHHUyrhJeHZZc7W7XArxO54KwkA9ahHd8ZRpwNnakgGmqmEjnCFbcZzdCzhcEKkweEe2Z", + "PbqmDa+PBV5uXFW/3CzRKwlG5ULjSCugBZHCEIfsUigjQT2+creydqk1rHSbwNNtBXC06R3cEQMbFv8z", + "Dc0Lrbg4h9TducHvdfqtNhjJW+nUBRmD9BLZzU0DhnPMDATP8KdbqwLxLG4G6JQL+aC1+ogQSPlBC3UO", + "sECEopSz0hBCWwUONZjy5EqktG4+QD+ahX/c8ihoF4c02d3p7qjgR4RID7SFKvWIEUV0lR1Y2A5u46Y7", + "/tXsG5Ty11wWAuGb/KLV3NSeM1lQKDq8R7IgV5IDI1LiGiKTrKxQC+X0ou+mDbBbQu4oYSA2USD1RcA4", + "Sa2hXr50pbla0wdeRJg5OEJ5foWHDF2WCMQI4Fgg3WkdYex5xT2rkioVOePsJ67STEx5bnyhTSpu5pB0", + "PieuxnYk5L0TXW/DMK6t9j1W+bwBw1i7RaTy/NBeKFZatizTpRv3nNNg1jsM8Xv8A3snXwApP+6fnp7a", + "/6uyw8KFz4xmmb2WACC8T7GSfWZWmaTLNBeAGUqiuN1L0m0YTwF6AOPWIY1ghQpW50XVJsPC7ocJxsGD", + "ugsoWDvc5A42EziOdz6zddGO0Ev2amFTNrqr5NTwNqsS+fo01yFmePxr9SvsQYvUDfn6xgC80lm8sOPG", + "LBuID2H2mvWmwEBJi7agbuZaF+gDiSY1F2uB1LXi60zztEqlySa/m/RZqouBS52TsvPLlxcXQR6zIGPP", + "74Y+QzBm5anBmyLWOM/Y5JdHXybRWvZDw7Z6NBr97n8fjX43Gg1/OR/8Bx/8djr44/jL/zhq/nD86Gg0", + "+mU0ejQafTn+5+N/3m2O3QrZiWYECRZGu9L7GkCUs78GYlexdqZJJ65yMr5OsaMhe6nVr6UCJCz3oz2d", + "mZzCw1O2dslfxO1KG1FZTuwRLhZ95jy6+s7b9YwVC1vSYVBSLiRE4rPtiZlU9A47za2+KVKXwcqQO7At", + "tRIKHD4qAz3G0MTUIZQP90vpbXljFQiEsS2U1nTsw4MqZgZBeG5Vxa2E1KAY8LKSyk7IKL4yC108Z7pY", + "iPxGGq8rRLWC6Psl2VLcqOj9cpdqF3fr+XADZhI0+aJrP/iSDvAX2u7nlUJDdmDMMZFWYO317qvD8sv/", + "ZKNRMRrlo5H6cghaRjRwwa7LrhNy2EvFeSMDgbsPMTyO6VlwXPxh6PCIwS6UWYkEkUbMsMo0WmjfIyTW", + "g3B5AUlK8C6/4YaJ2wUvjT2Bm7T94FZ8N624I4O+2cOH1vaD2/BJ39TdQx9jmtbNjOtd3wg6PA/AYB/q", + "YaAiqwOeBfya7rV0davsPSBNnQuj2ZJ/lCxRhWaLdQoeBSHOa4zpuYA+7b5DSuQqzVD31MUb7e+2PP+E", + "gwzCNGcY90OHU2k1gBQ9/pRSaKIg3tzCmofsnKVczeENnbRtj4K2lJDqDHskDHAcXcWPpWEcX5MdZ8dI", + "HtCiIMf0BOT4ib8zKreOCG+xNx1erokQKXsMkv6QfW4Y2315e5nSBoJE7X1+de7k9kCyfu6CnyipEajU", + "DjWfLgG3MiSoo5Tkk0YHZncjM4LYtxPHfOiBUdOvWzTU0we+BemTT0M/BeQX2xKoNJhAhI81b4r4k8ye", + "N4Z/Tsr4DSrQhWaJXk5loE81HpVYlZRuk6Nn/CbUf6uiYyOWEs7Ql12CpG2jdd6bj0oxCQNeQo4eHQP8", + "LvwNjw/HSM/+LQe5ppN3gIBAkLBlyXOhz5YcMrv7Sr5tqSBNCuh+mAETZPTCB+WFRYFJGMAhH4jbAhwo", + "K2HecvvntZE6xwkYVMgHcopPw0FK47mF3YyOwa5wB+mbjZfaLupS5GXv7ovbnVe0SC/ghes1PXAd8oDT", + "8GVt4CHiTrgHWMR1IaMDyRuVd4NP/BkKCv4trnrp61WPh4j0S2nRY6JEy/NS2wtfXMl17KHNmxCCLfB9", + "MREyq8Chwkm6eXdCGNnxoOVXKu5D7ziBL8ZuQBUS9QfScFeqcSIEAc4GVyfqG5eLpUhlyyBIMCyNyAcz", + "nsAtF4bO5yIt4dfGiGA3W7r7r/qEF+5G6Nld0U19sVoZphOSXmidgRH0hUa4rz0OY9wM8unNS/bDH08f", + "Q5ZP9hEFT4fhBca5dOBFC+9Z1qZt/c+T3jbb6I6nbxjhTnPoxlrga/O+ArHW2diH1ezkmPEN2Ljxq0Z3", + "Dv8VL8Qnruai01Y2rvyCZYIbyLaBaTN4XkAsrUtzQI7k9i4TaqbzBFkppnegjC+V1QKEBG61xBcYoGyF", + "OEsZT58+/SNIWJxwfnz4eqmMnFu59WclbweKK21EolXKUr3kUvVRJ338x9+fDk4fD04ffz49PYP//x/+", + "BfnJ998/G5z+fvDk8ecnT8+ePjt7+nT4+9M/fv/94x8ef/8feN0aq3KinOsz11DoWp+m7QF9SUK1a4Ag", + "3Ho2M6LwSV78/aF0I2yJfmic77zofbmz92nks1Cp/fhlI5BEoDJcBffxQgwoNeyBtgdST+xSjIXTteuq", + "yf7AYJtN44zDxtEmdXjb/3kcx8/lIfchxq26H/ODuJQd9RhzcO7Dpeq8pTnwoNGdw69G3TRTWYnVJBpd", + "hBu7tspFKhNwXcaoN5CoefDoPmQXBSFJEDYEQA4Ftz20iCZ13xzaXmumXwApgkT6imfr34RXMqsMNUnG", + "sSeyrgN2Q5+tFjm3ivOyzAo5ph9t+5jIG6ljyAAxKxyCe2fiqoIB9ZBXQwbrHpYvjXA5f3HV+2D2d3/r", + "nFUbwhBeIDc4EmTXuBaUTdVq92maQ84QeiiNHyip2MSS6GTIQNwfFHqA6kYwtqnItJrbsrDr7+wiDEE/", + "2VMR8fRiV4uIppMq4ip+FvnykHofwS56SM1PYi5uV4fU/KvM0oTn6SF13yMlhCxh73U6uHKTGe1VGYjj", + "PMsOrvteq4M6vkjNIdWa8uFelbHOC6827F3VP1rpg6q/kqaq/iXOld3o9pMdLxu8mu71iisM2cSKVGge", + "nKCX5YQeykmsJI76nE0w4fQEWTkYJ6uBW0FPmIXO0ufYlm12rHQxYYqiLyQkmKNSIq3V5gbEW8B8HLLP", + "i9IwXrUxXEo1sbLv++AKqIygYHFB+E4nK7/HaCFeYJPfGX8xAO8nIaPhjN7MYO5uwz23E5T20hQHVK0T", + "Us/N/4CWajTV7+HW3bedu24SRDiLfX3IseaeTxeR9e/4fLHpkO0G0FFaCtdnX3uZ+TvPFT5GAj7Ryh28", + "DPih4RECpQvPDSZanIGRVl2j0wf4u4MLy8B479TwcBux5KqQiXkOEfSFZpARTC7hBV8RcJhU8wEeUsd7", + "8N3jZ3DnqvhJnyKgKMirjaOAz90cHiZqzKVpdaujjf7wbCvY6KHqQ7XXu8kqEKr2I6gDoVj/0/S2/zQQ", + "2L4zPlUrtnNbKrlkv02RDaf4mFG5/ki931trE9I47TCXhnC334TgrI55lu2s1xhEY6RVO93GGwiUhwxY", + "aSUeZMTQ0M4hh5L/A9j8loDYy5b89n4Wvxbbmm2+aVlb8tuu5jN7q7Qaz5b8Nm48q4xRtsg3snP5C++B", + "rFw01AdIVelblOqBW/zP4ugHmcwiKvJ+x7tmbdlLgto4oxt+8rWmd06lZp34R3muwUt498VJ5fodt7Vm", + "b/lHWYscBr17Lahc17UIrF3/KCuxvzDVaRX+i12H/329tVxvB79d0XgOrv8PdXU1rbP7He7qWWCvS6t+", + "jGInsut11TBs/6Owphsa9m725EvuYlE+DKU1j2MVMAVBwkP2FwwmcNje6Fb28+c3gz8QF4LsDYVmvGBL", + "bQr2wzP2Z/niOZss+S3OdmJ5Fy4Mt/woJZQLSPyAR8N7EYFFtcr3z28HZTH7wwDcnSYs4XkuCZAZHWlu", + "ZC7gK0ZxERyHW+Ufvv/+6Q/bnfgDJI56b1Q7XLq/QkzwJ0fKe/D1C8fI2ErkA1heDDD2sPQ1zg+MO+T4", + "3oXB7cNMKllgfAQ42YFX07V4jtbh7ooPAuniICARv7yWaQkwSfA8B6Fz+D6phYHaiVamyDmlR6oQUSDY", + "CAmOPOp3GZ5bdYx26KyH0DceoPVW+/AGxM5BiE6YZYcXECxhXHKIMLzA5QHAU0Dp+dHbWJcuuBShOrDG", + "pj+sC9Ede5NNW06sVS6XPF+7vjEM0lVnR3bjjR1DhTpzHPf5a3U7Pt3pmdYYbSuLC5z7N2b0s/l/2fv3", + "5raRa10c/ipdrFSNnUPSku1xJnal3pJteeId32LJM3vvkQ/YBJpiRyCAQQOSmYm/+1u91uoLgAYJUPLs", + "nPM7/8xYBLrR917X5xHlzJCy43Ci7zgvIcEtMT2swRJL8pQfra7rZQUkb7PrY3bvUybh/Dv+fn40P7o/", + "Zz5mkN6AOD94XuKJo2yGpt5MJopHb/H3EFEOrEWYbE8ZeKrgMeVz8TTNb0zsO7rO3a4GxoHFHxdTthSX", + "MiOAlz8spmzNr4U+XgBEKy+Z3r5w3GLVU5vPrc8S/Kjp2c9rWYnoTL83xZBlfNM8fxFjKmue6qcvVgxn", + "HUl9ELx9Tv2Hu0J/tZlz+/3xQ+Tqbx3exw9/aB/dJJaayNY/Tj5/3XGeuymb4ZSRM4IG8UWeCLi54UBX", + "k6e//DLRy3IynRwdHb+afJ7qv//0Cv4+OaK/T17i3y/x7yf4/pOj7+nv4xfw9/EL+vslvK//B3//6Qjq", + "0//7PP3lhz8fTX/487H+19HR9IejI/3O8ZMfdJ3wP/j7h6NT/bf+n/77Ibbx4RHWMXl49PAH+Puh+fv7", + "V/D3k8f095Mn+Dc+f4Tl4X/675c/wN8vX73C569OX+ny8D/4+9WrP+u/X716ju05PnquO4X/p19e0C8v", + "6JdHjx9BJx49foS1HD9/cQK/PH9x8gh/eXn8p0f6l5fHfzqBX06PEJQW/0+/PDzCX/70avL5MyyQT9Xq", + "h+d4RX9//BA4OEV5LZJTBCGEdeF+RWMFzO/kD/pBjSv3J1FiDPcENy+y0uBPx1+bx0m1fZcn4gwSafPx", + "hnEbxt2TEuAnjgKu03vKC8Sg4o3gmXKEeC47cLgv7QOv1qeEIfktbPG2h/3ncpaIL8NBaj1ADkZ4Am3Y", + "J8oPiQzc2LCU6h7Qs6/TRvL7aBiKZk47JIYHpvuNVJCqZOVrZYndZQtqQw2dXyPTuxZ0fKH8C6Srq6gQ", + "ZeQn7e7Dw7WBa+bSRHGwgbNQiNIFKGCEKqU7FKWIRYIsylni8r0SkdRFKhHLd87+W5Q5RlqAjgBQBUzx", + "lai2eE2we45p3KIy6Ls2w1xl+LAHzAd0cykGn1NAvyKR+iYvrxhHBqb8WpQpAUFRizc8kyuhLKyfJ6zs", + "EVeGZz1uRFXK+HY4hm+hDjfjoYQISzhvMK/wu3N25sAglMn+X/L46rKEISoF8PxSzlqW4Byo1uoksAyq", + "FBYBfGaq9T8gW4XfEFYKiuATWPP6CaocGBB4/PAHEhFADoDnWDOIZcEcbuBN1gf26LRZKNa/Ww7Aomli", + "0JgqAvv/PTFjxes6u9LyEpgPAPKHMEyUSVxJLsUslVeNbP46k3lmoE2AjsHHLbJAIgTWYFRFPL4Z8oag", + "zqb7ypYihhgE/MUratV13BRm387ZKS9Tafe3Prsy1Dch2LRIHTSNd0JgVbpaOPVcbV4a7AhoE1CKhyCb", + "DF8TfdN4i8zgEczTQHUv/ykOIbO02D1dOvsuJFCDwlOfefbLhFJEtKufAMy+lb/MvXoM84OaX2RnemLJ", + "kqVfskD7ClvpAImaJIaT336rZJWKr18vst9+I4Str18PCWM3EkkvwuqGqF9nlvoVZH2zSx0w1OeGsGL1", + "906ml6+Xeztwh4yCeDIBfGyT5Ep2N2u6XwueVmuE6qBcV5OlugtDtYGd2mpMpaeiEskgoumv08mKp6m+", + "F4a+jwm1Q982m3j4+1ozjyx4zv4yX0NhDodiFXC1zWIE4yRRcxfgyaFf0cO9kmkacbChBbjsCMnCEpND", + "5Dy8nG5ZKZa1TPXum3onMPypNXeKPWMYG3Yw675tJJywUVHmMcTt77IgodxoaOiTGtYxSXWm2XfLsW5b", + "WZT5ZUmxYi0jPn6XmTf0+cUZCDiY53U0h/C9Y1AO7yBuod00SP4MYThRdiVPAQ0OOdHhZZvtoX/fTllZ", + "gxIzBYLtLfwrEZclT0QyNVbBBgjOyGN1OoFVE9WFtyQPXjmurqJIpUgiJX6tBTHu3eHc2+8Ua67EQX66", + "dk0IcfnNGrwW8RX8GJWiSPk2qrhM7dciizjdt78SUcprkczS/JKZUoq0MX1WJOTecR9CC7KqZJqyTCB5", + "Xsq387vdhUkeD2i8xRqJtU7m0uNbqJx32C5Q0vc2zEMPZE4FpVPsW7TImg36NMMhN2UIStwewVq1staH", + "kF5l4QNahir9M6NscCZXCLjE4rxOE7A+L4Uj+b/FcSO+FCAWRBDDrO54n4Fm+22qRpQF1FojQtkYoct/", + "xBIocw7/qsxiXmY9mATvC64VP4ezg7QyiVAQRELyg6thzl6kEvYhQrxuCq10UhQ8eBd0LcvaS06G1hSl", + "qBiZCCqjfh26AFAZrMJO6iDsFvYD1yOoJoiF04C/6KJcjDDYyFSoKs/222ZAbXjrXofCSsns8tusOS0M", + "Dz3FQHD+5qeY42OJvDstdN231FV5uRaqCt5j7VvB3GXuW9619rv0B01h0Upml3r1y9D4f/Ba17Bu2+3I", + "lcpjCXkbxggcuKcDBER8tvr82/GTr3+4xTYLd8xxc+3skEfhNbwT33xWHARKOLQluF4YlnrKYi3LThsq", + "VEOSRpjfyAZk3G7sN/nuPKoQbuaJhxyN5UlmY4nkl1kONokQ3vbgdhmLcVSKa2kAju5w3qwyM+gg/Wje", + "PsOJhQrM7AxUjd155xW9he6Lq2Bg8/WrftthAZlFdStFiuqSStXeBXCnM9X+QCRUJTe8upuWq5hn7ea3", + "lXP9IoN3FFuBa2K5ddgEWNFMV8R4DJU4FCqpWsxgdz80aD3dRqCI33ZItMb3jXVi+opVaJ3/9vZN/7bK", + "cSmKvPxmKgH4kepbnMaugjs+jo2Hi0wC45pn/FyBNnlma2cktChCB7cWFZgIVKzbLSq/JnNbHHrXmsoo", + "EPJOGuY8hIe2CmuI/IiD4VP7o8cP4Bx3rakkueY2M6oqnopvs+nowPhGYgbEhn+LWxH82hhKEaKbrXjq", + "xU4Yk7ezZN3tPVRnV1l+k0Wl2OTVN5mndpSP08x7o3z+drbWZ7WqPvBqPTb49ZXMElYXrMrZ4mrB0jwv", + "wLGL8aVgkV/o/y7gjSpfmNEl8EaRmNB7sHAlsiQBgGFArnWi/1oLLUvk5I1vM+y1wrFYkdYqXDNiDQOZ", + "lgnqNJUuhapmK1mqyuitltq0ESPdhkqgmodE/by0L+8JOzpxofnUey8ACRY1RAo3AvjHudB9foum23wA", + "hjY0fQTfVjsUf0RMeB9c9x5IbkKDaEFyt3zk47C5bwW6onfBQXF/ByCCn1QVj9fIN2aJPVSLw4HTLv3l", + "87zF61Dtpmxo4ILb6m8LDR6Ewr6ilAMb1LUb9hrj5oBro4Wf3YeiEazEbvbB8/XeloAr54B5bh3csFqg", + "Kj0IA8/uv9eiHIvFilEQHe4gZ76a//Hi4mz+x1C201Wk6PNRYe6Ovd1uXTfBG2sSqLt/DMRtIn2vxLY3", + "xhdIDOm+IhYQodc8UX40Do89pCN3G7wLje4dEECkGHmPL5DKdEFxaBBfBmkWhn/DcOK0QpohyMjFnRmu", + "oDk7hfRW4HZx2OKlPpEAvRlxV31nneGyIRUK0tM61zwKoV5A2knGFjJRCxubCGk3PtQO8IlCYGS+aryr", + "pi6CFb7znbJZM87gG8dGnMHELSESxbKcIcYsZn5AJN/ZDpzIMr+ZpeJapIzX1TovjcBvcrhyivGDOUBP", + "DaR/ZJAgRqYxMKnzSz3WJs0l5pmboCbykCNAy9gikskCpRyfbsfH0HapSIZeeBe3bYBmFV8/gMvDiubD", + "JRdY4Fqi2CO6+MRdh8ftIr5LchD1uF74TqOwueAO5u3fgv3eyF47gl/zEhaUSFgqVtUsrytRWh4qwvWE", + "jlYYXG8QQQWFnwLZqR4qE5Ve5UzXh0/UlIQ0GDSkNRgnzBoONTz79i4LABYdVPHP+s3TL0UpgAOwS1OG", + "696sNLOedx/Op8nobNQzC06LioD4UnCAcHfHMilaeNpSXjOoXPDLnFndg5FgBG8t8rpaPANs2IXMQE8r", + "xbUo6YDwckr12bpY5tUaXoI1rKe4zlADgngUoopby8LmNdrEAQvPa8gdT9hKftGCpcwuUzFb50WzhoLy", + "gRQr1lsFdFJKpKuZ1jLdhbHUKmh1k3uUAbQaTW4334gm0duc/UTxsDME+PKGE+4E/fZMSX2WopD8lHF2", + "D26JqRYL7rsbQipY0xUiC4M2lmexjd3OM5qrxIyirm8K+bwFCdxxmivI8suQ75mnYKGm/Ae683BaIa3U", + "g/uNt3EqfFL5Ui7riuAmvEBrF6WKQZwKGf9FObOHkpl1EyrsjYnhCzLXPAyE0hdqnWESJuhsSB8gjN+a", + "ZxRkglGmupWwjRq8ahCGDlPvWO2U65+3NFBMgCuqKxjg6PCKdB4U1jwojTtV2X8nvXeowtjhxorWedHU", + "Eo9H6T8bmd22ioE6ULPpPaaQ002h1crSUu/B0SMUQahYu4gv2/CO+eTfw3LSq+btvjAO4Lh8KeKU632N", + "idFwrEDe9Mr2HXJ4KCe4kfuLlhPLhNilOWRUu4k0anIl0g0E6JyQtIJcUE1VQbejcdz7MoA5nKz9zRxO", + "7kyC8m1KzO5Wv5XNZihVJHgv6Za1GKR3SA7Zj10Aq8MGH7+RG3krmiQeV3VIDsT0NGRxtEQymIuWShfm", + "Q9rEUPKgHtojhDK0n4kgme1A8iObHxnugFbvigruL7ipR3ZgF/vRvzMXkOP8oQnfffoMsyt10oMtVa51", + "XMZryOBsaqpGJRaGT12f2LDb5+w8L0hxpkBVntKeQvnPKNBEu6vivCA+M1PHM1bUlanYwqBUVucGuRBX", + "x3JLihnK9E0XhAXt6GjNzzzhCon0U74UqWKcFbysJE+9TFeoZs7OyBMIH8IPYC4hLkXVpBuckQG24MZQ", + "Eq+1UJgYCZfaM7uRiWALpSUvkZAGrxaYQWs5n56xqtRnqMgq7zjFVIVSbLjMlMlfIu4Hxym8JHncDEAi", + "eJLKDOAoeBaLNKW8USd+Oio1S6Tuy6sgsFthWonKgGao3DGw65nmMgVBWSRWHJWl7TnMfMI2YpOXW0og", + "RqwVYgzr3gy3sXxujIltmPmAzgMiSx2Q8Aqv9plG8eO2xv69C1HKJyC7fhSqyDNMrGgOgzt5BoY9m/Co", + "8NGzpzXPa5kmH/ilOLOfDaa8BfAGKPOIXjBeWxc8xC+brHX9x7Y5epKozmQoX/CFeQG3Drx12JfqUoVC", + "8ynMuxSq3mDcJbzY/UZn5cnKi/Yc0IKUqyrqSRB4w1UFX6IzFl4LfjUVXIlIfCnABsirKAQ+9CmTX5go", + "8njNNjJNJZpIHdssfAoQs0CO01WBvfYI39AHODxMBg6uri6SycCB6Mkr0qtYa2aRIY82ssiqTlPYZHCN", + "UASBM/OoOC/hH4Cko5VuAoD4R76MCF1nYlKFt0GhxaaSWXpTkVGwI47DxC1VPSKQFxasCYMcehbzqQkC", + "pBz/Wy5pXVyUNO6tcE54xGSCV3B+kwFMk16ClNW2c3G3cQ5hwvwV76bcDF5j8vYcPQdR2n6wiAwEbtAI", + "DJ+z1yumv62vGtJOkQHAg0KwSf3cYDeIBAoF7BJ8U9hAWdK7j+Y/fD+d2DTot0bSxTAj97tRx/H3AIKa", + "9aLuQlMzJo4RahOOrtaJneIkMn2qJfvhP79OJ+2Np+e45NnVRM/FZSl044W8FNk1+vemk7WsVERiY7U1", + "P6zrZZ9uENk11LRpfH803YHrf3y0399MyBuNWicOmWPSMeM6kxzqHg0cjzxLtw3+eb+iDc+aorq//1M9", + "ZCZLw64cwEA5HrVK+lfG193by1sA47bYe0ORDWcD1k2BNwpiNlzgy5ydkB0nlcq9BN69CjA44OkzttFS", + "vQn0cabIvFrrjWjU1u4G3JCdxUxAkwGgGRky0mnVMBqNdf4OG/g+mSrUK9OHYb3bExrT5iLQ39tzGp9e", + "U2hnW/wj0WLATdQ5OPCshj1Hd2bBa4XKr6o3ey5Rl0rRTKkZ0BIQBgLhiztKhOKu+1+n4Es4mke2ri2m", + "mw/T+E1pyHu/0TMwzU7vmWx3LrQWJjwdED6RFwFQDyLuoaxHqbSCWiDcypyhxYqAeK6EKEwGlEjY2cu/", + "gdtcMX0bo8kSUJR4Yux01iJVTaaTS1hLaQX/gevoVy0Jil+D68hyDO9DH+hsGxwN6O1e7mEcWPDPdse1", + "4d+wu17FdB2Fr8mhc5HVaaoakrIsFYwNh//B88j8iH/Bo71M/dSEPX3+UOYrmQY02UbweUsGXq0wPovZ", + "l/COcGGoVhYm0a1WPQoYhNRqwa77mR99BDwrAjt5MFgh5R/vqtGXJj1MKJtl36kTiUPDVeqdYfKK0TKH", + "8TwuwwC/RlE+YExTRLm+oxsuwr6Tg6ini0xwU4uGQWEm8JXF1PyJH1iAXrho5meDULiY95Ds3okNwxs2", + "26HGjDdna+otuelAA4i1pbaDWXkCFsMgAtycnenDdrYUHFBYOvMkBRg00Zmq1rxMjI/WcsSyyzRfArVt", + "nOeAWls5rcZYGuEcdF40tDCxxjxgNVErd8HmJgFHt+AJMCGI8lI3d8MrPQOVjOH7MoMTu4I4EGjsLM3B", + "uQ869Q5L3dDdVrS0tq7nZcfxt2MDQmwETwZXF7WOJKMg2PZ19AM/IzLllVCVvyRcDi/51SDfg5mYMbSw", + "u30NgrMr5N9rfgtQgwndCeEDxMTaGOTlK7Gds5deSElLAR4wXFVeRFedkN2wK0cfA97KhTWDM6MX65y9", + "ve0maBtCulrhHrWwz2476Hr7iMpvZ9SfpyLbcUQgwOBaVmYX3X4cbn8YdLfykis/fsKPLmjNNsITSlEy", + "yrI0Kwti0/WxYjvLlmKl/8cBxc7bANShleBVXTYNXH0y2XQS8yyRQITdk+b6nNwSziV1I7Mkv8HGtJBw", + "qAnUwny1UqJyEJ3o6yBtFCPlrRmJYjwhHolnjCe8QAkmr0vgj0dDGn18ymJeFBiEdAxoq8x2REGcrI2p", + "8TDIIabUtApyZpCSgb3RykBJdSsmNxDD00hj14e5/hrGuqIYncpM8BIHYqaPShbn6vYbano3V8Bhhz5I", + "XVXOlnoDGhxPwZTgZbx2i3CXbHfwNYDne+gGIDcZ2JBg6Mk1SY22nBP6y+MPfgOBAj1rGpU6G7Wxx2hx", + "VYAODDL1WhqmVnLSQmQzp51Lkmx7tlq31oBde7szpbFXG2eKrIjoftcBNKiNd3ArgBT4UlRcpgEzD5yt", + "3ow1wVpaZ2WjxyhbgyKdKzli1DvHeWMRI1tKa6h1kRGjNp2sZMbTvn69gtwjdxMgfNrddG6UpqfPR5k0", + "1KT958IAnc9ETpkN491lfbX2DFXHlUEVGr13LStQu7TG7nx2emhvuGJ0Irho44EbVdeGUTAN+78bZb/R", + "UR3EpAweMTAwdHzoz9Ypp9nXC24Leod/ijGDjcrkCtI4Wn7G/nVALRy3zJtjPEoAaTS7Z0jOy1oQjrM3", + "fo2ZAvO3Cw75TvWNhy5Uq74UvD6zaN/iChylnt4Cs+YvuUFex2BWc1Ah987A5ukUWmjtqQ0OffMA6hmP", + "vae3ghvpt5Bia0+BHeJJ1+KB7udxaTNo+oB+hEDU78iIEpgd318+zEzSwNbrmn4NDHZruW24XjIZz2LB", + "CPkBA5sMyU+1teDYEHIFktOaq7VQUxBOSn7DCB0XIo5vMnBo28yeQOJT/Gst8V4ZAWDwdTohyC0LdhtV", + "Mr4ajBjpeVi7exYCOcbVt+YqgqiDcI0yScW4CiHwBAanFEnbqbO/nFZ1IoB/GlcSQ78iG/cV7g69qmuP", + "VMXLEaDWUNRqzXkZqUoU49qI09O77eEddFQdMgLE7xBRGOCoshBhGMUpl5vDSprAqXFl11wJFfHkWu/c", + "kWWtPjOmVCkqWJYAoh+VIs7LJBgHlqm6k95CrzOqxAghZK7zzyC9YYbGEkO7KLhm3JKC2CzaabBu6lKM", + "LL0nsusFpu3CWxaT1OG0rUWaNPoNlYIw+U9R5ia8i0K/9L0v0mTwuEADr8Q20uf0uG7p47vvBIAXSpGJ", + "mwjExmDHkT1DVQyuEpCm6Fq5kWnKoDjAqH6z3ueqwnN58IxmOdxVPf2GGy2SyajRhH7ydAAsNOE2doaD", + "URXKxn2TnUwqM6RDh6TMGzH1kzjfLGVG0XL2WJ6YSDX7j6jI83A0hwGJGjkqal1XSX6ThQfau1a6Dyt+", + "JfJrUY6TGOD+HX/GYrGR1xyGFB50D2PRw25XLHvw9YrFD7o7seiBlycWPuj2bBQdPbVY+LD7kwqPvUCx", + "2J3eoMA9aY6FUiQ1pikvgToYhe7hLTvkDqUyo3ajDYQdcWjsCeU6M5abpoqRUfxWWAs8JNQEajTl96hi", + "fVFlsL2i28TSC6JBablKdL1+YLjWPfSfhJ42ME6ZmndQAL5tGJGN6P01Z5081EbrXJS7flsrlfrSx2qD", + "1jpsYCOev0VAbR6ZXAeDqmk8TI1GOqmjNWyA+Y5VXYtRg/ePfBmM8zbozB6/7t4JZf/Il+3WjWrM3aQg", + "dIcslIowunFwfAdA3PBrGM3vuF3MsoJ0LjNyhlnHW3LOp5clDqRu69DU+aUA7/uo7OVwIk7AIOR1LarK", + "OosNknDYV+W9DiZFajv5GzBXm4YfGo48FEth8Nl67I9Y7a+1qHd9nDMMIcnLBgoCDqZUDMt77CpOe3G+", + "p5w6ASE2Dbt2X7NIurntcgxsF6r5Vqvydmkhv8PpPDKXJLyB5+wNTD2KewhkAoth0X/sAr3DDnP2aTNy", + "pq3rIrbN/uN3RD5a3VhGgwrtJzR4KRCeD7jvWpwGXkFk+UY/EyCyzG5kUq3ZWnzhiYjlhqdM5bSFz17+", + "zQNm4WnKnjxmS4kMIoNZD7AD16K87FOREpFWfJBsc/uUFXcGHhJh7aUo9Ynwt8jPw+UFb3TuCpNUhQgx", + "wSOrZVYUJux+6BjB+5BFYzX6CKLqe6ZtIyqe8IpHlpi73Ssi8TZmpJ5wJlMPw0YNTQk0SX7GOKB34cTs", + "MMytow/iH15+XVGKgpeiHeTOMyJi5lkS5XUV2aQkmckK+VKjkmdXyuaqiaidFAg6jXvN5Ct13rMPWgWA", + "/CIyOwbD96kfzfYCJUZdRHnq+6fCaR79LHvPmxx7ZtnROmuFFU4PIN67XboFXua7/ZGYStluK7vhEnON", + "8u5NluVteWHgqitFLDLaW0GgWv24ubzx3SnLxA0ACcgSw6TGinF2f7aFN4B72G+wi/NMQXb/tWgKp5Rm", + "3ZVPO4dNawjJAE3+cDVwCG0ubDc09O5SXyhl1CSW9uS1dBNgejNjGrKpt6H8y63ndjAXXOf279XG2yCn", + "7WwY/aRBHFVDGAAgTqRbA0QGIYBTgrNQVB0EUOiZyjPxfjV5+ssQHNcW7OrX6WC8p+qAkuhC/yhWzaKf", + "/dFRzs8+CqlkU+QKFKqSAD/gQOMs5lmeAeyc5ejFrIiuF9jkonV4h3W1SZADDaeKYqjVmheNxCPMM/+8", + "I8rHmmVuSUnvhJ8dYUj4NXcrX4ktIab4qSn6CLAZNUEKeIvH2Q5nMeNL4wHvPfPScxBQHr32EGWbZ4j1", + "pt8cd3LiKjGQnh1k3lBcxJiKMWahA9mLyXYWPgBe+nw40wMt+ur011rPO8HNjoR7SsVqmDAI+J4AM5Ul", + "cLsMBsBrlmwNCnzf1Lbj4KtOAY/8g9EuRu5vi12aiUuwgiElgAXpRVAigxpp0BcNUimAoEuF4GjG2QtI", + "UnP2umJJDniRlcM+1Ze6qa17Utwt3m1/JnF7rPfgkjYBVEdmcGeiByEWzzgDzsZs5ggyXyCw0jVPa04Y", + "XNB0NHUhvicGqWeZvtsMOKbkyk6ODUe0+UeE8VX66LN547tzdqoPDiTQX0oUFAErqBIQ2L/ckq67MfTb", + "tgJ6/ZkRdzKBL00b96xXaSkqLjNDtUdWMMTDbmDlwhKD5jEloK8GBPd/aPn8+2And4bJzsYuHOVvjpp8", + "dxjCe7bmB16N3ZEnmbUXOyHGbsY18T7u4VABehRE3MXal3petEoKUL4muLlnfYaBpOHxM/zfL/KzEawE", + "yVyw5OD33Pyb/S92/HncBQ/MCoPgwVPKSfutB9v0KBxgkezqYmsY5+wcnSUWIQYkQBTR6TbxAPUP6KnH", + "FeL19vs9u/uWJBe2eGTz3Vsi3RpitMvZisd6vWDUtt66GwjcXG4dqQ4uCAbaKoyOQaCdMgzNjVS9sQ/c", + "T1MX5MxvoI4ZPmQFnBlUGf9iytDP4UDsnag47qPBcFStS5s2UJu1JJFTOhpiPsssEYW+YpDdu0EwZLZV", + "g2hoLD5L28Xr8MjtUvcnvtGr7pTuPJAOgDDXYgKMj77ovXNJD87copdnCYGWl4KybRB4kXifrKIgaNlT", + "4G6duYPMooXD167EVk2Zyh3KoEE5rDO+WcrLOq8V4wBgaxIlAXHQMGhbHpBErlaCjCD6tVsASzfGsYEw", + "fTBPkNHN9jHTdeb0IL4aU89I5Ju+JIpXMpOVoyH2ttE8sH8NvVhdgvPPpLUS2ofM2C9HU7w0brODfA4e", + "N7FUynZl7y556a+JlpXSwtp7TGV5KUVWccqyTJECBLJKJBGR6Ne+U2bHmI0wRyh/thGcfGWkKTcx9WHP", + "hJjZniH+vysO3zIXdNJCazaVY0V6w+pqoMI5O9dNvRKiMERwaa4UgF6B2MwNjQDeeyKDg1/LfqWIZVGC", + "O8//mPItI3ldgfE9aBzprtTRdEADIFD6cKGNXIpmH3MglQ61Xx9lPlJ2i5Wggx29vS1y9GC8+S7D0O6V", + "/d4XH1rDYB4RbkgOl5snfqqnF9nM3vFPGaCSyX/ied9iY5ybV91V1Sqg8PZd4QmS5dnMqPmN2xgr6hwl", + "TxngCpjKzAFDnzeFp3TGgEEVdDi3L0jiqHKMZaCz5yLzlqwF1Eds/cat223S7oV9kI1TX70GMI9TCEa1", + "XpGaijY3vW1FgM9NKgpvVxUTqdiQ9AK3NggZeLlCniziHxvGD9LpWcFLnqYixeUOEmjg6iT09XHEqsi6", + "5FrdYrILMgNaobHb1528dz6/QJe1bjx1O6l1gy7eLjqm/vG29kNdM6Yvvsy1SHQIZHwPiLtun1nXCVRO", + "Xu6wSbsfR/22ekrR6yNEiDBDMzeD686+jmEn9mqixN4GdI0vpIQc7qXAeNfgx8k14GtIl7VMIGJe3wBg", + "T4urJp5FXjr/w/8Q4Px0ci3zdHenNjxeywwBIMivwhXhrdNpecMVMdHgJrJuDzq6I59opUG7EvFlfi2i", + "PEN8v2qtz9EovxblKs1v9oOMDYTId+umqTG53jcneOdlebeOKac5NSgS9eposyYacKjfwWHVZmn04gwM", + "N+Q4uwaNWdtFcxsHDcDOD/DSYIvvykvzd5yDW1lSETasY0XFI6xt4nMEF4Qp46DWpAAiRqJW17e2MbFX", + "XN/VCMFdgRXbVjNnJxbxf8O3/tuGQ8d9cqqFg5u1jNf2OW55gqxqc3DMGZiCa+Ibo7AhkbQNw6zIUxlv", + "/31Y87x5efrbGM+5x6kxyPVNzlExoojPsTqiWIhO1rnY4e8PvOQm1LUoBcUEh0Szl/Y5rT9EzBHKM+Jo", + "SQzOLi0ibkQVzrBOL7Xcut7chr7fVhIVtgfDrSTDv5MIK/1G1lDcun5H1Pb/OW7x4WNjhHt7vRw8yuNY", + "pofX2yCjHkqeNq56EgzaHbjTLjib3b6wvhH1WjX4buvVW25gJPC7PPEQ60m4NFDZuwo6rektuDa/3oF8", + "sP3oJvJgIcFJTC0pIc9mApKX2pKCkSsxnMe4sJfblsTgtCLAc/w//8Y+J9t1743VlIDBjrXcsjfiksdb", + "V40v/xqDECBDy8v1Mi9BhPTl8RCvubPlfZ4efLu5Jn3KCL5EC/JWn290Bw+LwjBN7Kd73h3gAwVxI32k", + "Yq91qVPSr7B+w/XkADqIzm0nIx8Veg5lmsR202FscXsFsV3kedN9xox9OkzXuGKqRKjhOrOUIPvF/0/u", + "5WZdN3l5NWpcf87Lq9CYfrWi1/YdoJOQkvx1VGTorlkbJIb2N2+g6tgd9kFF9yzlQXWEZ2pQ0b3LsSWK", + "d75yJ4a77sIM6fSEaNYHXucuDHrRw/2GDbsAgG9zZiLeN4Zx/cMQfhLFXqc8uIKj5XYxRSPvwn5soVXb", + "xR+o6KIHI2+HqXGfrbClXqONkMhFkCVPmEsUEcxtW4gltUHU59gIe+x5XI0xsrnqDCIhJhmQ2wmi3WNH", + "5uffXUhQHenilbjcRlleRW7+PTUk2tSqipYiQidcR9mxz/W9NGlIVfaR5wyBJMfIRKF3Pktx9/ueWwOO", + "vl3hK+C2dyZCfbv6b4kv+heeRhBjZr0D5vJrf6b1OuLCd94yRGGGPhsvjfZbKUgQUVMhDtQFy8TwiHWe", + "e7szapGt/J9BQOpbV81RYld8ryn1o1B5ei3Kg5i8frJiqo0CwdoYryrEN6IA07KSKx5XsyWPr+ymt1wL", + "zUPUgUXzLIssSnE32dG+B/JJBy1isi/ozFWAq8Y/uGGN8riKrsQWE9lW8stkOuFZ2ElOqaM92TQWvGJ3", + "g1a1Qsg7QN9pNSfLpdpGuAA4otzycFuomqKUeBndgrjMr2qncjmklqqsVXXLBl2JbVSJTZH25Tb1QsDp", + "B5HYLAUgmg94JUpkAGJv7yTCHoDrIDLLvh9opIz+QRdSD/nHnjqa70TOA2HwsLlSovKuJfN3vK6BDo5n", + "YSJFG44RpKnCK8Eytu7ho2tH0SGMZUXnVruf4RFszfyu84y8Qi37nKenk7ul8GKgG7r5nD1HL74ySV2W", + "bFhheKS+mB+AkdVo+7Xy4uVVwOUDRthdOqP7xm6B2r5GXZ1OKDR+dznTJVuKLKk7ivj5X9bltEdNUM5g", + "0lJ7YG2O03pabR6mcLTHZ1Apv6uDVSJb4HNz9VGzR7owqBSF9tujP6vT9PPXz57dYvJ08sdAiUk7EWtk", + "eLvnQvASxubM5ZHBCpi2UtKmEFUaA6wUbCXYFsYJgXnec/ahExqDW6blmTRMN4FQFmPzbWGw54UJcTZZ", + "KyAAQ9spowQa5DnoIOgPxFmK/cMwXraRWa0YSrhDQ9dHuB1uE4szKgTnfyTyRlzrz8ZiXPc+cAj3MoWZ", + "yK5FmhcYrgH0GIR3SOzvGWY72JzJWzSYohJb1kozzij17QpZHGGXR/y7kSNT5qS1d8G/VTBVdM5+wlhd", + "vR9RYFJN7Hnu5X5SQYLU8mHrhU3/vG0UVgv/pSfo3MuOCezhqYkoq8h/BrtXP/lOsVWdplvmgtRd1OcH", + "2utAYwr9amQb649Bvk7D4QXb6PdJFhk3mNGeDCBdTSOOP7kMj2ZrYGAI4AgMDtE3yRAa0XO7/wPiXHop", + "liWXceCUWOVpIiyrDOQL6rulfXEZxCFbExF8gTEH1KyBVK3De9QX43yTQUwaCI1o1GoHND/bnegEJAnc", + "amK/c1DzlG7n2wb1hPEIxokxhNmAI0SYA6C9Nrxzc7YgoRnsm9c8lQltCTB/4ltRKVYLOFuVrQVDbfD5", + "s1DUhZFu0MKmZ0iWxPOFcGgmt4DaCAXLfC2XsnKtCrB8OaFylI9uOrHWkR7uNwEJODEYiJXhm6uVoNh5", + "NyxoTG69rcfPQHDMR3NQuYHuNnDxB7SbU6sWlpsa6EkgyJh46zCsn96bs8UfUJ0zP1zUR0ePYngJ3J3w", + "t3D1NVeIX5k+L9uz7ijf7GztXAh2cmU1Zyf+s0aHGssCYuV5vHYLB0LDg7uWHLrQ+BlWfDunbmuHe1O0", + "R+v+mN/cga/daE9f/61c3H6E4/iQ01SAb0NVvALcOEXH/JJo7xpnU2frm0UQ2QupD3qoKOWG29XL4H23", + "hu7hiZOXrMxv1P0wBHDv/g0au3ZgWDo6HdNLrdEaN4+q8gaUJXJLyGthDpMelMjO4mwMjd+gHYtVlwlh", + "SgEZo7GmwFndsv44yxBmctsHIHRv6qoGAkjxJU5rRQmnB9k7oIWj7R1Yylkj8psXWje2r4z0Zlqkq/07", + "l750Dt6jziztITRvlQ7AHWsFH3OEzFVDW4Yma0qCPOgIFirC4kSgpFdnYLDUt3ADu4JV6zKvL9eoKQFi", + "hWo58/7Ynz/kh3eOPBleSdLYl8BJC3J4N4fQyHlOcPUzG60b8RaJs//XBDp+bWQ1DP4YhjZ85Bn41AbK", + "wdMJsh6FzhEMc+ix05BYVeXmOQB+WgsP1jofrn5ASwbDdBoLh4vSOzg5utO9hisgbGQEt2AzUUy1TFyc", + "Elx/+Txvmb2q3RYtLV0lPLuEBdjNQ9MKlLKfoSh+bKX7uFSMpzd8qwI2MR+R049CbfArDw1JvX0W1kEp", + "5r3p2L2HczfsfdwlYv23fXrp/I8XF2fzPwY56BuBfk9HBun30oM2q+3t+Tk5m34yYBw829JN3m7pAETd", + "z12DewoohnoHQBJsXtqEWy0nyRiT7yHqhbO/8ixJxZKXihkvmIfzBMAKems52mjnZq/EZu56ZTSVkZfV", + "81LwpFrPAFbT03fAhtSIwXERLEA4sSUpahc0znbO7AXEEo/2HNLvnyEANea3V7nNE26nz3PF6gxvPQKH", + "yusKzheUC7xMeH1N/L8b8//dmAfdmAdcfK+p4aTUW/fad6pz/42/62ibf4uLznS1A5A5rLuXu/AuSa51", + "R0lR5iuZit0tsakp7RYE+Mic4cve0EdjLVQ9V/20x5rmegOF5uyld5rlmWDrvND/vBKiYCuezfQRZTBk", + "55NRAsShTOivTAEXsUD+NnQ5TY2XBoMdcK+0PFHqAAJ05wfbQw0Pg0XNYXlW5c6G4cYXjBmNXb8HYcRL", + "eX0y6GQ0UbGBeNG89O9AG0imbCxJkPjcsijDyjwEmxo8Ps3O/DCoL+ADGvShFo4wYvJFfbT+eCr+j4yF", + "O5DdYDx6uAeyE8ehX+Zr5IX+fpKuzXR5Ojx5lae9Eq6trrejgJ94gtFiI/rIscTwqergNB6Otqq/vbs/", + "3nfGxd7Y0RiWS6HfNjjBI4vApe7Z6Jp1jZuMLKew7IH7ug1s3IW3M/XtHmbXjwPaawsOanADozjYXqyv", + "v8E9eS/jWp7IDRoFA7wOZNwuBQYyMkuTbCjFXM5/Mz0g1XJfZDEFzQ8GXLDxQ7TcVghyjSSKlFsG/Bge", + "CyAIaALZMdD2ib9Q8dCt3JOyEsx/+hy+VVFECdAUAjWOl3gSCwkCqwk0cUNlRm8oLeGulBOTb7o7iWYD", + "wSUmAwYsw1rRNrjKoUN6dC7LzTpXBm3RJol4vfbMugzH+u5Bb4D1xGkJel7/B6Fubp/asEGaRbcj3RIc", + "gh6jd+t5WYcClPAhq/RTBhoKWFXQxoJfamV7fSlEXBEbTGj9V+JL5VeSl4rxpZb4b9YcFFqswpn4VZFn", + "Sswvsg9lfi0ToZqT+ebNW/aPGqJ77on55XzKLiZnayB901qdXl1FKWG6KylKdTG5j6hpgelOxTXPqkiG", + "FHIbB/f6pWLAQUZYbKYYu4drqyql7ptRCu6PUMJDvJ1/laLkZbzenmSxUDujPzBoriWdmDj43iSAoVGY", + "nZbJJFgfRRGGeIBc7JIL1AdTRaDAoLHYf912nePN8cFW7Lt87YcpBhGX/mAjT53J6pt+YvdgeekSY1yd", + "6zq7ItK8xv0TOsVGDIZJyLDE2ibtwqRl2HyU0N3au4xHTqQZlDMsRpMUyZ6lGEjWgH58HjLwZ7Zp//cO", + "/92P3os1UIeWYjS6KjZjj+kjFdci9QeCFtBeZDcsCMff/vaXIhtNlkLtal2da6R/vBYpXJIUNyLm7FMm", + "q0aiAk9UCzFDfKlKDkfKRbY27WMlB1jjas0xDhqusVhgDiKgbSKaBxi8kLQLiTt56bGpNhBI4aALg7OV", + "AxgBg3PfhaSEn80MDpoDsBE934bh463Fj3KXkRdEX+ZKC6JZJXnK3LDZoZ6zs0LEcrW9yBaRfT4vciUx", + "TZyrWBCPxwnEXzFeFCJLFFtEMvGea3G7WouLLGkwaVZSzJal4FdEU4IWP8WWebX2UKjBgL6W1Xe6XpWX", + "1QInxUodY7Q6oeKQJXmv9GuvcbMUAiOyf29hLaEp9SyVexgHOpN/BpmsJyvybgQZuk0TYZz1Kp7JhFV1", + "AaCIZGclibQoRSxg1ppjvlPOc81/6Df/4a7mA6f3WcU9gJVRyUuBJWv4x5co1kMicp7q5Y6n38yIi6wU", + "11IFY2h4vFPrilMJIQvouESROM6vBTgcbfx9nGerVCJFglkxpcBEfdvuyLZ7t5o+DLgAuU1jSuQwH3Uf", + "w6GJlB7wffCxrYyresMz70NfipRnqO4CSOG2aQvOckRClFktevROHAhy14adg8jnKFKYxnwjKxPLD2Vt", + "ZohA0kdY036/Mb07Aqbq/Xn9najAcmsgYxXfWFp76pyq41iI5JlpjJ/Z4pLZDlCoj/58two1jy3EanPE", + "/e7vvGBO9+Z4hYU7k3g7WumCpHr7STsWqzTn1S5AM/MOGI8GC4uJiKWx8XUlqDzmadSjBZLuPbp/Q1PF", + "A+9FfZqngWDY+TDqT+8+KO082JadetqPZV4Xz7d3Iy/aWklmNBj5JrxTwLFcJmTnc6ZZRDuB6y4RX+YX", + "GciXHpOaFRRnlGqgr8wkB3oUHseiqBrgOQv/pFlML7K8tL+h38sEsziBIaabe9GSwqZ00MwKfikzXonk", + "Iqsb4m9THCWRftovl2Ln1WDB9C29f2hu0ACZFT5BlrLRrijPLLKrP+DL+6usAgaVr9OJv9BHKdMNvWrU", + "999AKasoREke925nemWnxrlneP8qxw4tIMuFE+8+8ht2jQS35iWTHCSVFhJZrO9IjKRKtyYFKxGZEjMq", + "iBviGUvzG1Ea2Z6Da7iqRNnwAew861FBn9jW3voKCA5x82OvX+oRjsCXHZIWjGIJL0xZpjuC0XgqZ2t5", + "ufa6jAE4wORh2apH9RxbEe42tHKPMWYHLq/5gjUZ2RNr3PlhNnf7cNBDbcdx/xkhxip3fbFmr+B3P5jM", + "aJUZZUYDASLy7RTWPIm8k6Q7L0XMayUuMrgp6FawxJbmronzzQaS59b8WnhEVzROc/ZJCaCw2RTV9iJD", + "nk4b60IuKRvVpT/hlBwbROVHOYrExMI1VeK90XDdQKlHffFNtpf6scgSnlV616sG4hMMJ4xNm6DtIoNr", + "Ny9meFNjsh2GgDCaaLbK0zS/QQ/hakW8N/gm3LUQloQ1EbsnBJYQqQr+ka+YbtUS8dU7bUZiFnIcXWSg", + "IiqzAowbRtd+U63n7EfiEE1pLdTAaZsxnvACc5ZM0MtFdiOzJL+Z6qZkrRw+bCywohYcnMYQPMuZEgUv", + "eSXSrc2Vgu5OLzKVM04F4eWV0Gcm3eTEDbPKyxhSVC/TfMlTP5lKxTybs/Mc673InBeSK3Yj0pQBaUSj", + "v1NmlrVJlSdJx4SoHh8dXWRuDo24lCXseHp0dOSjDFA7DRPcWrgEHoPCwy8Frteu4XcwfnTQtKJ2Hyue", + "6+FbnSzWArDeJiXEabsd7OIzuyfNRWaPGn23wiTraozIYw4jSgfKvWrxS7oa1E8vMh7HcIhAZhqdLZAN", + "ZZaaa4l/ILHWeeQbU+2h1Dp92iG5o46h4VP4+gNEHtsYrVaidJ6rvdLcc3hJq5gyCatKO1x1jdQ2XT7Y", + "yA2/FJ8+vumuE3jCPn18w7QQpQ/xTx9fd01PdRnQdFql2D39r6dS1/igyC6fLbkSTx5P5/M5pHeKL1zv", + "Ni0jBd+TPz1///Hm6G8/XubzecA80+qsbtKuvsJdn1UfKNQw1O0YX9GX7vtCZCevZx77D54A3aGAdkc0", + "ILum1Q66x6Vo9CNXyV7LLLERhkp4nc7MUfZSVFymKgR5UqTAiglmMb2BL0EAaOa+egmvFp4DRGoKmJaK", + "XUyk/drFJJQhHMY1/Tn03aeGtQ2E34ynEeShQBjDU/ZXWVmjJYXv9LyL17aprMqvhC7/5s1bltdVUSMh", + "kc3H1e9leVTleaqesnc5g3+xG1EKxq+5TB36hW6sjCFCR5eKU17KFeWTWMzRp3CDY8cyIRJ9/9VKlMyY", + "cczwgkqSWSNkk0Qu3DOikMMuAVQatnsynYQbs39J7cDhBDhkh78ZjqLi1sLMM2bCPIdFOL6q0/RcfKn8", + "7+wLWjw1jlE1phRmz48oYGFcGoU+hxw1dkh6GSK9nw0VMAwUSCZaGCzqssiVCFmhRVbKeO0lczSzGVIJ", + "pPQZv4Q0dvMuSwRkbMFucNoBUaqjTYmduqqRgBfyoEh0QrhQrUGQVUHZkDTwVaspA1/1LBGlvPavWPBO", + "Qm6YdWIrlgP2qsNbZaoqBd80roJfjLtdyX+KydPvjx/ay24Cmppxm1tvuVGzDaYvtiy6Pu6iN0btd6GZ", + "+lWAtINoJ5EQsuafnvzQ92nnmO/5fAQWhWFtcO3VS2tQ7LKbNLuSDwX9MVEE3YDBxioNRoUPoAOGnWOS", + "v8gL3lDljnpQe5t7ROrFsTEkuc20laP55GDgux7sT11Z71mIh4+PSdy6Vs3iAPAnmYjS3lZ4TKK+QtKz", + "cUfHpUCkBq0HbbgEvi2F8q7dQLRnUSTH42KmwPEtY6bqpRIVKwXof+DD9hi3g01Rz/T9k+RaV8vpvBCg", + "8cU5Av9AoYsM0l4KjhGwO2FCoTe9Byo8bQ3gdLIUSZnHV73SMD7uFIvztShFX6kX8LRT6FJoRa33isjz", + "y7RbKE9TvuF9hd7D026hQmS890soXnYKXYuy6qff+Ametgp1sUzNvA3HMw12Yd/1GOzCvkLhydxXKjiZ", + "e+/70GTuK9QzxHsEhdC6hiwKPDGMWeMgGPN3Zu8lXpQ2Eu2AxGUufjwpIXt8zbNkzt5n6ZbZnUuQgmWd", + "VXJjTFd455sDKQB2ru+mAbhetocfoF0gs/hY1qPKfu07ed/KVKgqz8SZdUePMIws0zy+EmVAgrL12sN0", + "2o1ZMMXn7BQMENb0UQq0LEpQJ8S4TF8qHNKNBIWEScU2poH6I4pXUq2kl8GxVKK8titAZjEvMz4Yoggb", + "MHXj83nv+KvRUD2oHA6SFtqz/HU6AXOj8QmPr6DVZVfb1LWsv9M1Ch0vKEzmoAAgUorYhipzQTeY/7yA", + "gAhkMzHG88i8HOV1FecbEdUZBLsvwHzr6trwLeOpXqhkzI/zDcExat0i5mkqSoW0n7RWzGFR8cqmPGrN", + "FMSWG7P0ciaVAuheAOO1X+yeE500HVAjXA9MdyfOoxkZw3u89R/v7f1oOulG9Mye3bAr0WPvCjkT5bWM", + "xafMGgwOCxazn2wvmhWXKTDj+HBSidRdXda+mjajZK+izKs8zlO2AvxiQOqoxKbIS17KdMtq11SwZ4Cc", + "uBQD5hdaFsW84EuZymob1cVlyRMRFRhGCZlf+hWoMPI+dLv5o+/rg/bz9JvM5geIPOXehdTMWy/zS8ji", + "BG7oFiqyK8qWYs2vJXEpc6edd4T5hVfjgm34FfhKFF8JrRWsRXyFKksCcXzXgjkSD2ZPMmtN0uo+TymE", + "kEGwH78UDRRMtuBVvpHxArL89ccycePX6q8KLS2kzisCUKTcRmdCizrJg80hwo8FZx3G+6PgicyEUh9H", + "sh8RxfjNemuNToCjLghzrdC7gadghMCYab+Rac6TiDbUxLeweD+SrNRYvZaDCE9Sk3ql1rxM/N8iZxKF", + "vWDvZG+HkGWgcMsGlmjBpV61Sx5frWSawi2F84iPU1Dw/VL7B1dfhQE9/41ciXgbp4KuApP6mAgFxqKu", + "RMFOTAC6eQcbTDTRkPCKkqZdnIj/ThjfnjMRs5CYHjJ9lHlfYaXYoPprxv0ZBbPSNWaYKA00hSch0WHQ", + "RICwQ24bFdHyQBKiRI8o9mvgYNa3kYH6RD1/4N1grLnbvW5LA5Krd+DwOM5LuBP0vV0p30QO2gJBZgex", + "S8y3eoOyrU9twAp5YWaKIyAiRzM3ZBoCwFK1ZcuaeMizvGJgkSlKUZl2V7p8MB6XZjLCza/6Tosd8rs5", + "DurSHCENkZ42ArgwebIdjPwQPMsCAr8FI4lMWHcoYPByLZQL/Ga2EOBLEar2dqafOjGhFEQh4Rl5vQNp", + "aO5yQ9buX6iuSd7NEfPM3Fw8Uzei9LHaA1HFcP6FkONNRi0l0cCYOhQxb2W3TcldaP02uRcMllBaiGrZ", + "5SB6Y9yEYzpZ+2AITLsy5+/wNYRH9leP9G7feol5UcGo4MA+wLvCLaOOD8CN7eiFEojsFv756utW3W3b", + "K319hMvkzANyaoMV68n208ndxcUzxusq3/BKxm5PcFnOGfpAgeWXX2Y5ghzzpm/PREPwZCMzdvLhNcZN", + "OTyhUO6F87H17haeUc59XuKRVwpVb8QU5fwpBGTRegZZJSlzjArmYKKx4b/BLQSnmoqGXC4w33BVW57I", + "vhsHxgLtDDa5wN5A9qrd2aJBx4hrUlyXpcgqkAq8xu0QETB0E04Z/WjnQdPngH5pV4ORJs0CchjQbVIK", + "EGOac++bfrrR6GHJi6Ra6j68NGevydmLf8/oDmPFmithcv0qjOaHeBlkIS1yJRK2FqVopvAsa5mS2HPD", + "ZYX/KnitEAmrT9rp2dnt9R6Y6e5y7N/nKFq/MAqkFGOBy1+KIs23WmInbyXu+Nir0cqMMSgICsAMSFNS", + "9XIjqwpXla9lh7a6MVn0XlhmSa/qNJ1V4ks1dZoewpR5nJKCRCO75OM80ydCwN9K15qyINBCwneUzC5T", + "YdR8OFjSSpq/LZNuXm60cPNrLa95CuRSOWt3BiFY/mIPwvAOCpfad6E1s66Vne0t3W1tCKX2OPd+t3dd", + "tS7k2+mUbhZwXJ12GdYp0eJBG6tPpzTqXVi53KNHksbY+3yQUhgWWkamN3h5Cb3AOq1V7OBUJ0GK0kFX", + "mJsKpFeyxm+rD4FQmmeEyG6c+gxGNry4v4lSQZqSbfKdKRnhJb5L5uyqwXtV3p6dOZna++AQ6c4SVHSG", + "t0E84UKIdrqaTXTOsPAd/Lbnl1IDo4tMQb3eIswEGhDJZErBub8/Iglf77py8UipKIDigIgqU/GogKph", + "hbqtHxNORWV2rhU8ltp2FOND3ekwhnCKBGpCrG59rqo9R92Q3WcmqtNqV/nQSjo8FBQEYNNkG63uHalz", + "isPpwino9xuBNP5t5dZzY1OYJTv1Nlj4KiGqpZPHy49CQSLs9m0Q3OxNzpMZeJ1L86LvwEaDhgHe/lFs", + "Npw9Zg+fPJ+dPH7O/v44OmJvRcVTfci++PTyxPiw5+x95hU8e/vDn+H5FKR4w/atMEgcvltZAA0tG4LY", + "XGep1n9lpTx6R1k1zIe6OpgWfYsRXTvWt2dkko0EvMeD/P4fyjwWSs0gt1cPlLhECDAj3nFTPVMCJNmA", + "6CpNW/avS3qREtrbbe/xypue1onMX6zr7KovKgreYCaeoGV50avAttR3b/108pLoUAJ9y69FmfIiSqia", + "KMQ+9B5fYuYlJjOtwaYSY5wUW4rqRmidzjVQsXvk8HnKju43YI9/ePL4aAjz0DVP9o34TycvMTgUjhNM", + "zNndl5/hnf6urACj/4tIZlib3yc9mq5Xj3QfDunZzlXwnMdXIkuAlExW29OsKgM4L/QWspoBEax+D0NZ", + "wAeZ5lq0ucSEj8USX1/o3W/+eJqIaxmLxfQiA/xfhn83sfn1rl3ML7KLjL6oILR8kfEKXG0z9g7+xV58", + "+MSoYnghz7Iv+vH7d+/+k5Gi2nhho08j/cZJUaSCDif/hbhOOHzgp9cvX5/goeU//5LC4w//8fH8wX++", + "OQHbCpin/JduxPKyqLHfN1xtdImfudo8+Fksf3RN1qtA/8xA3Ve6vy9hNKi7MAy6rXWVzxJRoQ6sjw9r", + "+jLr4n5v43/88AmeVbpBM4aRVez8wyd2D7LZl1v2n29OqDy+8yrXcroe2zxLtxDX7rCG//c9nIV/6bH+", + "F4znv/R3//Ul5f/Cjv9L9/n+vaf3dAfwYVXU/4qL+v79/98fJrvOXZpvmroQon9zUpWbAEfLac8jsD6h", + "YbglidQJ320k9ydeS/t6iiqf+hPb4cc/98LFwyDt/lxjIQ78XpgTfzrB+dn9vayzf275Ub0adn8ytCdv", + "O7J6oe3+6s8nZ2/v6mtf0j2Lxp4Kd/LBPQd2Fa9t/N1HUeSh1Kj3RsU25LM2uGGKSBPO6Abm20T2UH6t", + "eJrqTkU+Jv6OS9S+7+y3WZ2iqaS5hnxUPL0mo6XLyt79DXp/YItKgRwEfVyHr4nSkMixydaI15oL3lxu", + "vUCKKXP4KUimYySv+WS6tzmUZTm0/QrYd4a93QkUbH6rM9StsWx9rDN20/Zy+LxzpcqMl1vK4AvIFPCY", + "AbyuTeLDjGTwZLkoeRNfFqDX4RUPSStKPHk8E1mcJ3pG8UOQ4XgPiXl/Pvlpyj68+3HKRBXP7/ves+W2", + "6snnSSIw/oXWEEjKTxmJcPoAgMOuJeztx2IA8jS+EVEiUr4NfyqT6Ox4yhKpwFkILwc+FtidULmlFeit", + "Gd5jFiAisCgB72ngeBCK1WEjsvMwfMELHmuxNYxldgKHGlmJK+B6zADKEXVYiqrbOrlhFlOFxjKodsTV", + "HYKbdjgqGlTjvdIDfRa2nb9tt8r204GkO4s4DUZkXoJgKPOjPer6Yc8QrCi4MN7i4UXTEVy4jSO0h+K5", + "F1vtBAFQIHoJo/iqkmdKiqxyc2tBsV2vDw9OhAFvgou3BmHnIflizau3bg107GbVANxPrw5z2uqG5+kI", + "C4J+GSgA8zSKeWqAyVpX5UtjmdLvQZQyBbvj19E6rgfXBDXdszEdujl/0cXuB/PSzHcDa+bcfEuxDU+E", + "CaHnSgFKT9X+iH1wf6iNXn/hBU/TvRACMKZ75rPOrtDUnZLdd5gF2NZwLr5UbhoHFmretV8/64IhVN82", + "tf2vNahLmeV7hrAB3Qd272jKjqfs4ZTN5/P7O/P1HMKGPtH0pVAF7ZtvX789BQPnU1aJL9UDyGWfosnj", + "wQ2/njKLJ4A38178AID8cV/sTk2AJBG7l6+M0DFnesiNGQmi8W2NXjvn3XkenmJt7TkkkoG5g9II7dk6", + "v8he5JulzIQC15dI4PPGmobCkburrHyE4+fb3S661ALw0uDzoGMcJN4yHKR+6n+3kGg4q9y6yFkhSgcN", + "4tuxHh/9+ck+uRkE8mZcN9jNOhHd52YF6+FGMb7KWa3EnJ2iJ/47KPgdouNBmiqDxPzZkiuR2MJEk4lV", + "ZFoggvgV+Fs9gLdEqR78ph99fdAExzBN6x50BEu38zjSJ4Cu3rM1VutSqHWeJj2rDTAtmX3LmeZanWL3", + "juZHs+P50X2w1YEZCBR0WGnYN7C46+WEtkj8MYgk5uB0gtM3TIajVdaP80EPITqcqbrQKi+2vLOthtOj", + "e8csfHXfUftW6ymNAp87vUA23ebpP2i74a0ROLTDkvoHA7cMB7ZU5kiDz7epP1DxgftsPulnDe6C2u/1", + "MFt4N2zmiMOXkJrwvcPPX/J7HOpwDB7mEFxdhGCDX+ufraZakZF+zs5wRSpW3eSku6IdF7qFY/eULS4m", + "kAMuFeMszbNLdxTO5xeThS7gLa+nbPHbBYznxeQpu5iAmnwxmbILUHrxx4v66OhRjMg78G+Bb9jhw9fs", + "/Xox+brvO3q8sRL8l/4NGqhLNjZX69TVfTVjEjj4Rs6Ev8s6jHR6dvYLYR9p/SOvDh3Mv01iHq9FtJYe", + "0SgYEX75DVT8eM1LQgfTEtORWdxPjxqSzcSty4nbQRbfAtVjrOvIHPp2/h2sNK6g+RyxLNzn/0yfP7af", + "Pz70838Of5/gAlwDPtsr1l5ett5UKr3Na6OlOHwhg2ije1mU+aao7C/HD7VYX/HU++Xr1y6Us52OPgMr", + "HRY3XDEyasI9DCWDsUFhq9AJoI8ZyYTOHjU8jCdw0AcCeDwxpck/0RBIeFzVEA0Knph7G74lNEfsmkNi", + "kytmTG4sjyH8MLk/TNJwx3rbNAy3hH5/agBDL2CKLyZ+spWe85CKb1fBoOH6kYS/T1Cq9waBKTOjZ74x", + "9VbH7v3ufPXj7l0T0uogA1pCszV2DBac9/vgP7eanjnMwNAtVpciiWBg+ujA36KQi+zf6AcjHF3ZSCla", + "5cNTJahdIoHKDRpMB+Yzj3tykMjd+CLN64SZ19g9hGq4HzaFgZy5K9LngJaHWh0YPQhqt2KlCWOXAA1q", + "sCuXW6NObAsxZ38TW4jvvsiKtC4Jkbc7bAaa994FRewAiZq+YEkvyuHvi+xiUgqABDfPjYZBf6VcKbmS", + "9odS3JSyEl5p7uoGq1dcyqX9hUhs4Gv3WZHWetdDjvjFxGkLCkFQTee/U0yfA/rgOvnw2qHOUGO2MFhc", + "Xc3Zh7yoU0RlczH3PhshIVji7XNBk30xQeRScJ34tHJukwMYRh89YikupapKjEeQQuHxmeWJQZtj1ZpX", + "JqyIGVecRdYxG2UcAAQhawZNZI11bwindy57i/wy9IghIr/SbEo9CKEtiG3Ap+wegbiE2xBEhvyIo5Yw", + "kSWQzwwIk5hMURSpjFuh5z1agu3fviO8XwE8sVI3qIArEzy/yRMgxdBy+T0M4ZcGChO0d0k4a1p4vv/t", + "dMMQWuVt9Ul8cibiupTVts/SZN9nG1FeiuZxj+ixiOAGDo0ZBdBpjUEPzJz99fz8w72z+1O2kqnArIez", + "R3a0aRpdyi5P0/xGnwjqmUUMVZT5iw8Jgph0UK81Sv5TeJSkrXsWi0brXFU9qJfwfWou0++5xjRj0ViS", + "32RpzhM1Z+9NtB1kvmeuIwhIC4WBDioR2VZXCDWrOftZVmuMFI+KUl7zSkSyUEyAMymZstcfWArojmQt", + "wcB1niQlkuDTzqGEJ9dUkbCX787gK2BRguSypSCc5XTLyryuMHmaxi9zaOGFzDLMJ+XsWiBYCH6xCcc3", + "+fOj+fEPj+cPj5/MHz2efB5ztpmZKHi13jUT+vmsKMVKfumdCL2mnj54AAOkHul/ffr45qBZ0TWZxQkt", + "m7NXEDiHH6iVYHwJHnmBzw2Z6YNPSpTqgV6rD+5jIWyJLrKs4ytRPaBeUInNdka/1wWsogf3W6Pr16ll", + "1k6BcSPeWWUNIyeGTLQPZrhV0jwvtGIwZVR4ylKZXWHw6ZTFvCylKGeA8cHenZxD1AIoULjR4QiNuapY", + "IvQdT6lU7KRnsRr+ElzbU1rzpJ1JpWejEn6qKsYL48dGrWZ2JioG2jkm8HHVWCZ5Uc3yusK7vRDlBqA4", + "cQxwsRUilojf6foVVhXprABXdl5XkXFMN6Drj/qw6/V5ZM+bRpCncWY2YA8fHT1jRyyRSu9wk8rJEziK", + "9waObPiXyDZXn6ZEwe239Pjo8Q/f/+nJUW+D4RTOV7bNIvHxkw0luF3sXn07Gtf0BGH2MTUYAY93kJ3b", + "hiGBGl7fNzKpgDt7LeTlGo8Vi8Dozhd8Ga5/5UcgG4fDg3jNqymDYBcv5X2K3CGY7Tf1JQkUwKds8UAL", + "0wsjTSwekPC8sGtZGpBUyE8u9XY8+fBazdlfUQpnyHCOibiSgI1p/+wMNHqG38S9pkXtUug5S+bseasb", + "KITRMgIaMSg5W0MLvEjz8Ifm7F2ezbyIybjkBYV5q3qj+0CsRCIDOgBKjMKw9f1O+lqJMgIw5QDmuBLl", + "7ASAlqm1eoYbm6m1c75DX9nL5w+O50ffMbmClilRzdlZvjHxnvbY/1leyQKEPhpv53czAPOcuVY0fTf+", + "p4LS7Q4pthQJci0FsmD0ClYquhLboOZw8vMZw1fYldiy1y89y/KV2AIgPlPbrOJfiNcpLgXcAld1MWev", + "wEEOhqIqZyc/n0UnL16cnp1Ffzv9r+j1Syaya1nmGSD9XvNSIliVG8fGEGzzupxhY2ZXYjuTSU9IFOgF", + "AcfyIx+E3eoPNDvfqUdzvuH/zDN+o+ZxvvlO7/Xv4N7Sl83TPx8dHaF77q3MXr9vGrzahXEl7uKqxZGK", + "3PiHB58G1M3BbSfg7PTFx9Nzbx4OmAT8iDcXwXx1ATsdDa078Kyxl3gqwLutOKjYLd9RfQ81G74ywxYF", + "rYgiUirdK+ucYvjW2dmbB+dvzuDbqKCQJOHnfdion5Ofz6YkP+g/MRvHLqVR4baQZef5mTqgJy3CAZuV", + "ZQ0fTa+9f93QzTsDpVZ8qeiO7AQ5mttT7ZhaQsWHzCJ9x68gWsxec5AnNmdvSd3gxNh6jegEl6JkWc5S", + "Xl4aRmNAK8Qbw/ZIV4wytMdBRS5jCAoyyPyGz0sq9ubhrBQz7/17b3lVbnO1vuJMiQ3PKhmrqWPUgYhx", + "GFbmOq71fr4RlSjv63vL9wLDaix4qYTnqO5eRxAMKrPLyPiwG4EE5M5uLz0sQj49m/fm4cFh5NEFVnAx", + "cc267yftQe29FKxREYCs09p6tOKq26oXeVaVearYOr9pz7C75dCaqYXjGZpHbDzjRbawlS9YQSqBCnBn", + "8DSd5eUMIw9N+LguX4gS4n8XllIZKEuVWtVpEy2PLbSWtEC8fMBRSC4yLbHzTGTVAxubZ5sHZeCLamGv", + "av2QlsaFLidLgcICdbfJWugPnWlpcPSHunmhR/OLzByI4NvlhOdA/l34KSOGm3xFv6r2zzTIqtqmomHT", + "Um2jlh3GnV7XM2xD5RzNQddrjyPMK6bGGEB7K+yebdi5EU4Hz37W+uxnM2VecFlRCjhwwnfHS/ucmWWN", + "CQ88lZyS/bm6itCeT6SaAI3yALnBNrzQC+Dj6fnH16c/nbyJ/v7p9ON/PTOvGgf+A0uh0Cnw8v2LT29P", + "350/MyZzC/2WJSxOa1UhFA4VRM2CbMi2bUxYNBAVICGGtkLsfqNVBtlJnyDup2Yz4AfTiJ5Noqqy7uGs", + "PufqqkGSgfYfW2LGb7Qa484nuizu/f1GZI9mNoP9/pSVIksEYhopmQiHmObXx25KXhSiZPcuJq/p56fs", + "N++NrxcXGbA4Pf0N1srXi8n9ObO3pG5fazan7GYt4zUg9lGw1yy/yQAsEU7jZw6DSxdHWN0sz2Z21vVE", + "ofvtmVPv4AL230BDIY2AOdr8ztHl0e8b6/P8gSURY9wQcM2Ot9MWg7FoZoHtkCtcXbaXrNZHn1mlzRfA", + "CIAA5igp/IfMOLv+Hs8adzdMCQ4P5xpGx+BD4qwb02KWBHYTM3QJboTxffxma3XRmE/bM+9qgRe+o5St", + "mczYjVjOSlGVUlwT6zZN0jTUGqmcMaDkN1PPIoyrxQ0dLaWmPcur39/ereZOppPutyfTyd8/nZ6dv37/", + "Ljp5d/bz6cfX736cTCevTl6cRz+dfnz96vWLE/10Mp28eP/yNOpW+uLNydlZ4703n87OTUVnp29P3p2/", + "fhGdvX77+s3Jx9fn/7U/MMy48QeE6pB87UJ1xgjYFOBGnIdFnm43eVmsZcwW9rUF0bw7gdgJjCimqoFp", + "RW+kqvQt51pwaAyJXZr9cSQoA4WkE5CiQpIdkgcUxhFrCO1RC/FFTQRFQslofnjjX1dig4k3w+Ng8MCC", + "qJcxJ9W3imVR9WbDy+34voMx7oxKjw6KsdUcEhWzfzt1GhiIqK0zkjlhIVViE9hYKpANCthDTkb0U8K0", + "DiCSvscQChZ61GbKg/f86gKQR7t67pZlN27WbhrXWQMBBcvR2yR/MTtkx3DEQeiUvkww8yUoFRS1gsG+", + "70t5KTPr50bkO3B9WNSygLNgYFqZaZPLaPoWKV2UiuVA8juKX8v70ZMyPCwHzMUNVqWFwkQDuZ7ODd8y", + "WlxhhNsqOHAfZAE+cwbPcQJsTA76t3xockLv0teMXsCiiteW/zERtAQgMNf9qWWkyKG/mNf9X7CE/wtG", + "hrlANS9XL3zk9eC0aoWaFFNiiWxpLBYbDRTqvbnHBg6M+uYWGI5vM2NuJ1xSz6UZiBIhlbxxktVp1U3h", + "9Hlu2jqjFhHAZGOsWE5yMIIkrOaGDGFi7501CtINLUmDaijx9sLdnw7cUb0HXTBn0ISfoGlGcw4eLiBO", + "OLipAryqVp5Fd1BJKX2KAaTSrgSCIbezs2z4V7SbktunHYQWzafwcXgOVvAakxcznI6QDbYV29yJRnP5", + "c/CGycFAAXjadx321neun5q6CPx1z35rtrD1gd1jZC7LO8pzvtM057vMcnZBiebag3xVdwz9u6RADwCR", + "GHf7GS1g5903gIARfCrhrOidS+yVzKRa9yG7fnQIrp57m7CcfXtXlReT6SRFV18jaXhqcqUjjEIB7McM", + "4Y/1G7tB317Rqy8t+WroYjDP9DbnzFRPkgDP2FIgb5MNITZy+wjq2ZM25+yNrh2WrPlekjdjNCY/CuJX", + "RnBidiM4rACZQaZR3KtVhXlEz9dowDG3gutoDv1rfPtSVBF9Lxjbatw2O8K6Q5bb/zh7/46dwa1mblZ8", + "YWlWtG2V94mpn+fTHPT+QHVIFwWjcJYgxuuUifnlnJ3xjL0qeRZLFedT9uJkb6Sr/UhoLwSwJ0sZNx1R", + "QUzBnx21l4nEwKKu6x40zX72OJj1nbvV5Gwg1FD4dhin+qCCaW8IP1hn8eL9u/PTd+fR+fv30ZuTjz+e", + "Loiz+7IUl7yi6N0Zho42cDnGU1LdsRrTRgtpR8WMVmOa62DPRHYk631MWaHpDaN/YqyRQz/SJxnZl9Q2", + "i9dlnuW1YgQjhCj3hu63ia+tX999+Daa05vRuaF2jko6cj10wEvDE0l2tvC1VoIwyO01Vnb88AeYffPn", + "PigJ05zhc+V/ujNAyzzZju6RGe6v00lcqyrfBCOTXgAb4UzVEHCWECb4SoqSiXidg9PGpBxDLCjZFXef", + "lu6DU2z8mIFw9uKw5fZWE6w7Yea3Y5c1e2Jw5UHAtmDatYlbnMO+uhODZaNn1l65x95oPjJqQsyYdW8K", + "f2GFQyAO6461Pffotv8typzQD1wSklZ3LW0MEUpARkQVr8ORI6W32EZuLiq4a+UP0GGDc9gZ5n9j06zp", + "wIs1r87Fpkh5Jf52w8vLsfwh59tCJBYOpeBKOXot48OLQWCmrwTQuyCeLKrW0pIk98TXmDoUSvjknWU8", + "Y4tWJQsbywdRgEowcGhCuIqpRTczL4BODwkfHqwkUmWREyte8ywTKUUTp1tIk4C16WJ0oEbKQaR6v1PG", + "T+2H6NxOuRsyk7mMA8fvCvQ8D/tx0IZpaIeDbFUAeaGbYG4eylDvru00vyzKfKlCCOfA9r80KJie+Yfd", + "y/wYM2u80vLc/UmvXOcGzBNERx0ZBgmsz5jqZL3mSA/cfgRN1ZbZS8E3TR+cgYA6OztlQO0E2V73A1zj", + "ehJuIVIh3gOupsBtGyMlQPjY6rlRrOdxhxfRAZ3wau7QDOZ9wCfTiSpEXKdWhxyZZ266e+ZqMbxzn7vL", + "aeBO9byOBzaH3I9dmJa6WuelrBArGCOFeRzndQax/8R/jdj6kcGcogTfCBq1MPxa04tM2QVmPWlMbCRZ", + "miGCMiezHByJMAlTBqqa+VmqNf5+kWk9nVIa9MPFLy/fvzv9vGBKC6WZSJ+RRQZW8JpShyCxbEHLdUFo", + "OHBOHjj4IRwyK0eZZetct2ajDN+ofUdsIlIUcUfN9Eso9XV61yf0QCcQNnpQ51+a/rXNb2nlYGLBzWKW", + "VcAt2wM5i9Z+Uwk2arofnxgHS2aXUW/NH80rzdqnljsXhU68+s07g759KxTHXWiK2ELVHEsH7Dg+qsWg", + "KNq11tF/98/+f6g8Q7ufy3huTm7YaEm2QngYOr3h8W1tkJM9VryQ1S6vq6KumFrndZqQ4Q7AKtM0v0El", + "BNs2Mi+hLTHswg7tGjwda51B7jTL9p5ekeQosasIAoaQUfH+HW0YDASyL8KFUCEJGLe5nO4xScf/zvup", + "iU6KUjrmB9DQ4ag2DmC4JW3l/xN4pW1LUNeuhIj2g8cMJpbYHECEAwifGFlggY1Siz/dcXS4lkRR8Lef", + "EOKJubIMylLiY0ISAkR+pim7lpydVES1yd5SPsn8IjsTqYgrYOivl0pADN8VgLdkCfAGAZk+wbToWt6/", + "OdPqmtG7GLe1eirW0fzoIfsL+/7oCzSwxISnKTuaH7O/sOOjL/qf37O/sIdf5uxdnaYPclrif2FZ7nVq", + "fpGdr4WjO4LMBsxn9PgmMa5YQeAxbFFqL8Q+Lj69O/v04cP7j+enL6NXpyfnnz6eLp5dZEixYliUvXGU", + "COOyFZ6WQ1kUX+K0VvJavDV2bNxIh0I/mgWQhOOM+2f915pnlfynjT3Z8GrO3uQ3OOOxYUdO6lgothGb", + "vNySHx94yvn2IkMyXl+tISpzm1sKrFMzRQsEbUV6monF4d7q+Ancjz9++DRlq0cP4Y8XHz7db6WcHD/R", + "Ktmjh2DQqX7QfxQ/4B+Pg+qEVjwio8NHV9YYMlJ56thTiJRRpiKJkIN6wJhbJhUkPMVyc7YAun1dSCQL", + "vQk4GPvt24ZaAhJ0RSov5TIVFxlWYsuqOVvcrPNUzOBIXLgcJd7h0PHoeiX6VPMb0gzqNL0A7HbbTLXm", + "hWjOg9fgyXTifTU4B0nJV1XUE6jaGSSrAF4LBhFbcGuZpL05+8AxZR0Pfs9Ejk4jtQEjOoOP4ksNO41r", + "1qpEVOZtVIiMp1UzM6yT2X8CAs21YPS2W8WU4WmqI/J0CMS/l+aXsmKzv7DO19gfGSh693fivT70Nv3s", + "YWDXX5Z8s+HlgIGlNwGnppSJIPVSiWqKGmLFrwTIH7FAlFtAtFkYZZIy+hbzi+xHrGoW51rp5BKzOGiu", + "pPKOU/z2zF93PdMhL7O8FJHI1X5XbadvS5HF6w0vAfoPUn7QupiKZOYdS94ygr7DkeudWxeZRd5SVV4Y", + "ddmYPEWWzPLVTBk0b1LY6XMy0/eEVt1TxjNMSNC1XGRxnqEwPGWqjteMK7bY8C8UNLSYEtlpvBaJs0YC", + "a2qqhGeWxOxOdvr+jK15lqR47QaDK131jdF8+P2TPsQKCoLycKR9Pz8UdKvy+PGfHv/w6MnjP+1DJyCx", + "N8hXn12LUuHEmNfYvW4iAi67+wcASDrE/1DgvMyiYveWfyuzWcGUHgIMhyDQ5zl7BcEvygwaEq0VDHFi", + "GVTN/sj0JBRzdgYu25LxFBjMydbDqryIitugPU+ts3b/zi+6XuaF4JeiXLArIQpK8kGADSJQAoAfvCSK", + "XCk9F/OLbGHukoV+z6cCo/so5VkGoN72krA83+YUsFxJeNcBNm/jfoGWTdwVG7xWei6Ud15IjUX+o9vC", + "wxWPElk+cMiAD5rRNpBy9eBSbDZ89mh2vJzJKgwppxWOWBx4g0CkL227Na+0vC0AjZ4D6Fb4Iml/k8kV", + "XiS4+gQ7uuWNQgGOCPu6f3Hh6yaPzGgScPoaL81yCy4kZd7Fl2ycs8sufp8hT9sUbIkk6dDrxrWoKr5V", + "5mSGhcIKfimSmdMeYOHKTFWCJyxfXWRa+tzwSsYA/YViqN7RnuTSFrfgkrAoXAv9zlPm1r5UF5lNQbxZ", + "y1T4naNrUCsB16LviPbHOQyWMWisIQew4LFAJI2PPv6Ki8i/Elst5+lG10o4FACs0QCM/e0nM7BQKINz", + "4qPJyoTAKIjHaXb1ma61g/qivwjYOgIULLzdYCg2/ItBEYFrJWDWKAQeHcGNddzF6TLvu60FEJxWg0Ut", + "81rmtUq3nlWGdt+94/kR+4sBjEqmtJX0r1Ch/KdQ9GOsfxRZnNelvrDuj1Lldp3lLTHrYC/4KyzedKVE", + "MU/lsuwJzRsmg3tVYPzeU7bI8kwspmxRlPlS/yMv2cIgXmhB0YcWopdg3xPzJgrpdvEo8DDjstkIrmrK", + "U6Z9zJRMUaq0E4VeCl05tATrhtg95AbNV6tF82rR76Ehewn/p8bu80U1wCPGjx2Wfkr9npo2Tv1GvuwQ", + "tBIGqK/LMHdeWktCk4kZap5MJ/lqtbtT1yK6ugN9zIXkYzNpSxVlXgAGZiFKdi1Kl98CRM32JCBB58k+", + "QRIt5/tVg9crkupB/OCpNbySFf5GpilbCvBiaUHcel5VT5KS73sbvSnR5+vTi4hNoZ/UpdgtLJxZqdOV", + "AE4RfU6JSpR6uFQl4ylby0uta/yFbfJSsJJnSb5pQDkdzf80VCLonEwjvKRoYLXutOG+DOeWzvOQ6dck", + "BMNjds+EJKv7ngU45hlYh0GK0Je8T1/igQE4MzAdtB52Ki7K8c6YkHqhpfur3VN8nhezK6de3HOitN1e", + "SCIqM4PUc/Lh9f3JPkVsJ3Dg/kks9qlF7+o4FbVyLfeiOw5XZw71CBvPr9U0B5rhh2fCQ6STC16w3vVd", + "URrhBexVQu+ye85WQsEvx+NV3Va40O4AjxZEYCa/AGWkqvimYPcoBPu+y+HxGn3DFXM+d4de+aejx8cP", + "Hz32hLlO+Eg7/aLO5K+1aNvvkBWtMdwNxUw/izdFOuPL+Pjho9EIHjYh/rA0+PO1MGkSrWS7VoBLI+Wu", + "9ez/wLCXO2CgGBq4MSj7vkfm7fjU/qHyLHI+6VHd6HjI7a4KBHbVcQXCKnmhN5gxYBYA0MJMsTV2CPy2", + "7U3F7CHdCwxJdwl0saXBXdOS7WZWtkuM08cq6F7gVvfMayoWXeM/1RZ+SHL8GBJrJ4jvgcPH96aNFnrt", + "GTaaDfmtB73HWW8gwirdagkeBUeR+F4xG5bVHcZGUNegXCVjvYYIL04hXVB8Bm3AuKxdAVzzi+wkTcng", + "j85/YAaLeVluIQZTqyfGmwpVh20ZgwIoPvWET4BhINqT9/sBDQ6kW/jsRAEbiTFltIiLvCspwKs0NNO4", + "cTF1az4wfxmL3UUCM7tHxpn/5TX1/vi05u4QjUl1NkQNgeBiwxuh15ahNujuhyBRRqsUu6f/9dRSmT5D", + "grbpfD5vQslPwu/Jn56//3hz9LcfL/P5fH8ejW7SoE7vpNjA/ntY4H0Oj8AZASgVNDJDaDL8K8umXNtK", + "hl0602CJQO/fiOs2b6jMVnkHV/NNfnmpD5trUS5zJastS6Gkuy4TsawvMRFcF7/hJQiBJvvZTSu90J/y", + "1uH+CMS6A9bIMsSRfw9AQAicnBjsb0+NPx9Cgj+E25fa5/P5XspVh9o3TETTXhbQ88FLwuAl7eAD9mfB", + "j1TqR8Az3iF9gBtiC+u5SXOOSGClQ9qydvqLjOxlzpUF1ESBhI3Fl5QvmAlE0W9++I+P5w/+882J/T6g", + "B/KtJWXh8Aor0vpSkm/hWvKLbHHy7vzVm/+KXr97dfrx9N2L0+g/35xEH958+vH1u8WUdR/rauzzi2zh", + "/R19ODn/K5kD/dd0i2/E8rKom43+mavNg5/F8scPn2y7ZQY/A7heop6xgsuSScRFvsjIg3FhfWoXkwXL", + "s6Z44iiTzSDTN1qxJo2x75og8SYGJJYMsykwnymuE71svqT6v9gt2N9qsztzFRbQKw9POLh+dONWHLFh", + "NrxqryOzggh70s+dvbysV661iq9EJTKFgHXr7bKUyYD2vdaHUYg3yCa78CV6RsiQazOzZaUgwxksKDIE", + "f+U/7DcueG8RWzx8SBl88XtGlAPVHkL1iarFEqJN2QICpUARxBg2NKTHeboUZeVWIUlbb/mXM0nBA0zF", + "eUmcBcNpWZBWor9bCJYDYLedriFXhfLCKrSqpTcfHIK0oeCYXIxo1Ne9p9nfZOgoM7G2xJV2JbNGirb1", + "NuvfiKIOsslx8EEJrs2/LAsdvIEMdPBPjqU84jkQdYh0bsBC/bsX8LdjOzXiAofvpV8fR1eT6eRXjM07", + "fjKgQR+1JNM3luazNXFoA+VqXegWAReMxXqdmUbFPil8976+bZztSPecf3Z9nU6u5NhPw1KDSJpNXm4j", + "hGeINsvdNtoPoiS/9snj5yZsU2TXIs0LSOZ7K5/P2X+LMm/cK+YuMTfnvYdHj3/Q7zp9Nicfvd5ax08e", + "/fAYHtuoWhP8CWBtLz69PLn/DP5nA2t55jWkynOM2INltdIqJ9UUb+fsPcRQYTcuRb4RyD1ITCNwDgDm", + "aEOkqmVWQYCoMzsjuc4+rJ9wwkMT+dcQHHrUa3a7G/sljqdIYP9PWa2QgVZmbIEe5PwmEyV6mB/gL6Uo", + "cvxh4Un/TtDlwJSSLCkiRTxchmS6X1sbe/gaaxwJ6IvGOYhGwUGcPF5+NCURDaIlPsLqnw5ARkGu0H74", + "AWBCsyxN+5m8TJZInErddJsmss5vaH1Zcqk432x4lvTwWNEGUSNQCaAAEeBi4DDxje5iYB0+dyB3dMhW", + "T665TBE7pcGIrNg9sllbntALpDa+mNwP5d2E1ZuA0ppy46K2DHVOtnzQhiZvyAi7coX68H0cUIQhe/19", + "hrODsd6OKbPteRAaTscO+zs113zPTr/vC/2OHss8+87Jj9vgQnCxcr9Py+33Hrx587ZvsJsBfJ029wIm", + "dpavhQfw3DvzIcjGxAb8+4wJfuzB+xcf+waEmhMcDSvl/16N1Z9zOyXcJOJW/n2adCZ+fXgmfmXms/2j", + "SM0KjqPP+fz7tNv74gNVCBGvZ1U+g8DQnh74bQx0Yg8wTft69e497/ryV5R/DjfOisaR50+42zqtEd0p", + "GHwoRSLjqjeLrodrBnG28hVbCYykIUT6uVZg2fHR0dERK/Ob5rV0OHqsj1al6x4M207dy0uUOC2Ajolx", + "qXiaIyegi6E0eFgeM1vBq0qUusL//cvJ7L/57J9Hsz9Hs8//6w+TO6QUsHPRJ6X1I0QUWDTsVdO6S5nf", + "MPcS4gioOcM+MqFVjCalxCKrNxG6X9XiIrt3DBJIKS4pYZA9ICPsFF8FBVuoheMDgl/ut2KAbrEGOn9z", + "dTWCe5wWwrku1TdJUGVzNIfMWF6GjVXnJTcnGXv7xoy/Hh9RcWO7bgHR4H6KetjjHSEM7Tvkt5UZmXXy", + "MhHlOHuRnjyqbaeDzXwxztN6kykmvhTEB7ollxv1bj/Fpre0dn2RnP+UMGoSrQtACNUPyvxm/7cQQTda", + "lXwjbvIyhCODGLv2DTSNQIRzntSxaPaOvf447+OkuaPVSKuwMTHNQRu2KM+pSQHuI/B3II/Usk556frX", + "IGpy211fWbDdow4Rk9vs+g5CEWW3lcq2cIdG2idqfjxUsHQ9vLWM0dz2HTmjb9sDIyVumKJxL82HCxRe", + "L/YsgnxTVC94vBZ9FPzvKJnJ5vSQo9vLX99p+CNO+f3BFUSF2Zd9Yb+HqRSeouucUz3Wgw3/gmTSbfvd", + "98cPO1GWeSJmN0jbVAIia16y1LVjho0QGRjF5gxy2ylzK5HXUqv2srrIeFzmSqHvJpZa3zESOXbD8OrL", + "uFLEOCVUJTeEWoEHP8bZp/mljDly9z/AnHc9GtCjOXtu/Gcp4PPq9YNutIrLDAaLQ96yAXOdYry/VCZL", + "nrM1LxOthzIwOefAD0uU1KlJ5AJTZjt8/E/f//nh8Q9Pjh4/fnz8/V6Cb5mFciOfPJ72QNXSxKckfJi8", + "pFVnDbYJm8Oft0l7xk+OzPRrrtYdb/mH1pqz6K+4yNnClV1Yhtb7yLilf5wR1btILjIwr+r50kUUq7NE", + "mA64LCQ0DLy/d3wf37O0uwt9SnwBvCmeXWR5AQReBb8UM57Ky4yAD/BjDF5mVSmINUqteSkSk2cEFlC1", + "loVeexdZKvgKg5befPyESxEZuqtmMrEzLK/gJIJdqRcmGKQvMqOkzNmpmSM0VVvPpt6wXs6YQ5x45ljC", + "rIM083yjEgxmzl4AnWqmQc3ZQsH0+H7oVGaClzNFuVcV5ErPGglc/3/23nS5kRxZF3wVGM+5llINSS2Z", + "Wd2ltLZzlVuVunNrKatr5jbTSDACJFEKAlFAhCRWWl6bh5gnnCcZg7sDgQhGcJFU5/TYvT+6WsnAvjgc", + "DvfvA85OwueypURkZwALAsM4bVJA65+W4DtRf3TFSt2BFy8jmIDWg6UosgDMHIj38ZbSeLtM3dVTLoW7", + "bILkmbVI3SCALsXMCLvAJwJkZYMUI7WQBWx26CKlZ5mYFaxUwSMXJEFg9hd3uTSisc1/OD19+vRPp8dP", + "v//z82d/+tP3x1v48jc/5H0y2klJ87nVzSIkcx2GdJUSwiNe5PoDn2tMr9+7EaYAPC6dZRwwh3QuFJf0", + "h9ElvuZNRWp0cg3uVgsBjDZcFQujc5ng37NsBU6yyfVmFeVS8LTia9nTfdhlRmRZYD5p3qh3YktaIrNH", + "Ku6GW7hLKmWNp49AQvKlOQzdtoFleyQ6uEuR1wNPkUHR3bwram5gon1Nbmf2DNl2Wx3PJi7lz5fvLDuA", + "uFzkSrAiKQ0Somb61h6Oal7t/yR3t93c1L592Tc6IHYHqwwT7rjbgKL9R0MInByfPtsKHrA5whvtRx2h", + "3ZEhtHpHW0qnDOlZcVQYnRgALR7kxlWZtsd3g1vmBjZwdzkhiVgjIoXD1RZcxW8ebm18Nh9fXZ4Btza5", + "a7pL6sdXl4ewxrQqC/bq4+XrMzbq4eOgHSfapIObU3whHPXIspEImQM3gSVeZMz8Wif/+Pt5nD3Vyc1v", + "nHLTb7BFpFb46y8LDkctkIvrgmf/gemOmgl9dq7srX/DHPVc5W8zjcryaVX1x1eXtTZ/fHXZDx9fnX/6", + "fPHxQy1BwnMCDHBFfpJ3p1fEPTvqQRtBBOtZNc4S/UH+AxvxXmuVGsGXLsNrJLIQpGMuwX25V996W0rd", + "3Vq23B56RFJ2C6D6Gtxh6tHeiQSzz7QSkVUBO7Z3yFB0YPyh6Otb9vCyHpbjdm07ufYfRk35mEEuDyCU", + "jM7gFrRnkaUbzQAtYRcdESpYln8XxNv9RqnFDkCu9KMdfjgcqbdYDjf49FsIdwEArTjVhbvOVWR2CoKV", + "mfWNsD5G0+P1zarnSYik8FiOVavB+Fjful97S6HKoVoClsxsJpCjQ5XD3EAEau/fnw6fH/cC7Hrv309O", + "nw6fPW8liTFi3m6MvlBwmS6dsHeKASXErk51iUM11Xd0kTYi0XMlfxeoSAxH6lMgip2u0NI8QGrD6Bnv", + "4Ko0K95nn3iaZsKdBveKRRV3xSW0r9X+LO5ats+bsBCC3oOMakvEv65E5YW6ASRwCDk4Y340t7oMu2q3", + "LH04ZM4921dlANoIWd94a+B340QruiyOYxKS6ppz2qW5VDkjQJtwawRPH4DmBJME2losX4qRAmMv45nV", + "zMrfieb6Vsj5AhwttblmRKiDN2C3BvBuRCLWHXVR4K9KIQrvjAmeLIIXbqKVLZfuFqgiIJRMF7DkeAFX", + "aFvA51K52xZCkvikU52uoH0x2BW0LtM87eMtHFX46aoQR4gng7hTHEOcIL4am4Fx5sWCq1AhDEmAASE8", + "GpelcNcaCEFCHBBxlwhg8ZQWDeW+qeIuF8rKGxGBiSy5G6YFL21RjWnENaeVMO7eiW6A1INB5dPAAoEc", + "i6H9mQgBcP6JU6QD13GWCAkRzbxgJ9+z9/LlSLmT1nWRhprVRxoO4uJW436xQ3bOPKRE1VK3PEYqyfgy", + "tzix4K9K/Xxi42bgLPh2FJragSG6I1UNHEGYCYmOTZRBWj9iIu0D3ibAXFZYgEZEwDVyCX7/hcjI7vP8", + "+Cm7EgYsez8rHh6ggR5eFGY1OJ8VwpyxkxeuLR7iRRfM2zbcamZcMakG3oL3Wylcc69E4fpz7P5DgX9s", + "qouFb7vtV/M1UjjkAQsdAaQgAItGdZAKI2/cgnDjinvSuy1Ky56eNs4Lt/nD7ePZ6Q/Pfvj+T6c/PH+c", + "uPlvm+Wb4er6vZP84Mb8mo7bjVjCmwR+hGxGwS1dcdbhopIgtRDZMtJI1Q0B2NuJhKiuL/t0tvM27lvQ", + "cub6AYJHOzIocosHFFjLajFUOTf3oIbvnJTdOdaDDhsKafq8tNxKyWGifi+9kak2Tuxmv90KdTq4ORke", + "t3t4ijaWcXfuM/hGSnQmbrg7Tsghv1cPmsJjHLQilpbCKVdi55sONqEfTd8Oy6GyTO1nW6Kb9Z5mJVfj", + "0HW9A9UHvrRcteJRi+DJsBGtQH0NH4DNpitf8w7jtZGdbaNF5F7L7+X5+cXRdC4G/vvg5nSwfNptEWmN", + "0HXZ3f0hli5uy5JlDWYFpDQ64SKSEVyHw2uKP7qdUoTKtCsQgtJ9iXh/OapIgyjC2Qk4BGmUNqhRjSPg", + "n70LVcAjuefZJAJFd6gbJdUc40BfC5GHn9isVCmHJ5fMuu9f9vFW6NixV4KbZLHznm02E3HN8EXb7r11", + "/RzutBD3tVe4XLSDOgwW1X66j8Tutlnsa2nwT///K9oawAnvvlKZ6MvdobOnZIZq2x+lnKBoW08uSyEU", + "SZKKG9+1oVpZU8GXw/0ivDZKamzOLoO42ROx1aDgR48EIxSDDpvugKkLrM/uZ89r9qMwS67O2E8iy3Sf", + "LfQtqN8rXf7HflJpy1Zpd5Pd7CVbCats+dv8aJZxNSieD+ANc2B/K3k6+G2+n/fhrhOwr4iKl1SHkLqP", + "dIr31COIJ1fc/6LiSbfxrbtfPa6vx/SzoB8gAXcUbEIIUhW+dNT50sJNh1srbcGBy8SubCGWxHO+5ZkV", + "HWy6OHN2CUvKjUgQiqstUOmdmPNkxbynATLmgc4TIM0RuBbjmVJuF1PNTWqjWKaRWg9mYheFvx2D2HF1", + "EyMRVynP3OpPRZ7pFSj1L0ZqyRW4SES/soW0hTYBRrdgEq70cAcestdU64BqHXkDAvH+WcYxOH0ATkpW", + "FOCL5IFVyP72P+Tc+zH1GdhNyK0raHc4HAtumdJky+nC2OW5bIfWPf90AQi1sOEgHtorkFwxXhYLdyuG", + "efKGu3N4jI/Mg+efLlo9K12d3fAluIDXCgthRUchXsuiHaYmWxdFkZ8dHWU64dlC2+Lsz8d/Pq5FPxrZ", + "1ibyjBnnRmojPZ7ufZciqfHRhEVGU3JB81GlPLjlROdH8CvrV243PqCXloXtE7A+oTLH0e0vWhYFBJ21", + "rIn9pDj5sH2iYXqjCtPqy9182N8dlB4zXlG+ClrsWoh8zDN50/ANe75c8wn7Sd+yTKN10GXzjwhoQQQy", + "dYz3Rd6ejFt4a2YHH8EbJQp7OhyOFIZagiJRh8kuySQZCAJj/xwIzAd8a1mshiP1sxXsR119xwV5xka9", + "58tRjx08Z0upykLYwz4b9U4W7rcTttClOYQg/VHveNRrgOBi3mA4hCSx8VCmmUBRUrmM0VCAu5UtCOsV", + "PouUPN6MQB9reH6ypbuo1Ej/l/xujD0fh3f72iaEKVnbYpme77wK6IdBNf0bny8279PX4Xtjr/JMIu3c", + "SE2CZXUYtuqwo87JkH0Qt3VvWjtSFDLrFhIhViXC2gGA90SGdtq+Q3YxQ+uuzQUIVjtSbo15cvM+Qj0v", + "S9i2N6JCHUfr9sONtzSqtams44gfdzuyFDxrbCvyo+VZhPDgarTsIIQ+9SMkjeA+qN3vPmjKY/IQ/wh4", + "CC5lETGA9PFXwW0xMCIBz8dBCQ6cbsHHujhXq6ot0oZ1DoaNa8GMBicmb3d3x32poD7Auo332slxc8S7", + "xpNwCLwD8+OdIf7kAN/VcCy4U64fzpC+K2oqlRumv/0DjwibGIjuwnc2GwDDwVIMpDI2xgoYtQO3ua7B", + "Q8XYyt/FHn37UQ98yyF/tQO6Ohg4ZvDNZKTEHbyTtL88elm+X28yd0Nrh6Uimg9XqDvGuASLUmfsjy2n", + "PkPwM58cLbOjm5Oj7yYj5TQpYPdKpU20U7njCAJYp5Ov2J5vR18VX4pvRxTFgZty+KvVajIcqfqK/J9H", + "Q/R/PArjcrTMmrK4PVUnLuzYFu7KMvf6eoc7RZOHJOO/r3r9Hjz3d7CRdMFveFQSqngV6Jcg9jD3SDYY", + "F4UU69Ks5arOacKDRhAcnkNKphVLhVO5h6zSI/yFwVKZTnqMVHVUBv07Ol3paLWQOByrwPQWVq1X1LRh", + "OeruS5i8XxCsHvvjNLHQ5REO5KgXtxx8Ki9A8nkJWwR0l2JhdDmP+Cv8h1tulmUOKFxVfnxa9vpKH/Dc", + "UamvRgP9zI7ZQS6VirWkQ9f2z65R6JzJlvIOgiDATRzCHvjvAUJcRvQYudbZmlNKsKWj5UOowc3J8Hnv", + "LKylpTTF4ihZaHW9GiyXU2EKSgvvSO6oLHk2OOmd4bJre1qsjDD7bHEPB1Xfz2tXkcZmjgKMYcomX6va", + "v0VoBxM2YG+a4AgHHz98+D8P17P54xByvWoiVHRkql4vXK7LRnh5Z7Yq/hjbqAoIPjECUXLpEKl8jSL/", + "ASymrSXe8uVK7Ioq9w3aQa5RBnelvpGc3chcmEPQoYCLTICpM1vB9RniLgCyo8L2cKvXP9bvJCIbEfkx", + "vK7O9j0C73lF9BJlpBAOmuVBauYyFwDMmAPkP3klSeuF1gsWXwBlscEaUANnBcnVAfJjiSUmwEo5WRMJ", + "pSH7MSh0rg+Ms0KqVY0+LvjmaApPOcIAFPResU5pcV0oc1x118IoV7FXqtGnCBUZkSLWIA+EH3jXqaqr", + "AyRdS5WGkkZq6ppROXBAX6A8NKHzAuHJ1jzRAzxWQO0LtgWCyUMIqxqaGUIntcMU1VGJEB9sfx/2ABTW", + "chlvElbtGBPZDDgkA6awxdgNtPY4AvfSAo/8SqCiurVBUurItQc2zwuABATHnUzawg00MRkBuW5gZgYZ", + "AA8HuTZVVe38ivbpODECvD14tgeLXpSnaeP1lq6N1tyrnBsr/gEHThsRgIXvBMhAcVXc8CwDjMRU3B2R", + "jx1AAPSZDawe8JVxmwiVttHTS5W20zEgPf3Fa9sEWLjRCWqmK3ZA9YTia6wMYU94sLEW3ObGKsWw8DYe", + "RIPP4HBukqDA13CeLKiTHrTJ+5a1N2ZHaIImMj8NU2ji5uksVlnDRIVkNCAr2qF+yZ+XcDmtK2HInvhs", + "T9D1X2fagNtsopXVmeizJ+5i8MTz5vvj+a9XHz/02ZNMz2fL4knFoTcQs5lMwLPgWqz+gosm59LduZ8o", + "rXMqCTikhjV0/tB8V2EPTDmzJbCca53XIYejxN2vBJ/FXfGqkxZe3BXBdwmWe7Lg4IJrmJ7NrChsW+xy", + "OnbJWpZPyIxrQyuno2r/VgtvXEhPiYDwwp0SB4Gy7LDXTrbETfFIFUJZtn2LtHokg6CErEU0VLv5Gdea", + "3q+GbeOSjqarHSS6NmUBLwQ9vsJzYX3COroWFbQTLHPDeXozKvN2T+vIQXydf36qWx79X0Ye7uyfdyd9", + "tjrps7vTPludfnETn8s7IE/UJpWKF8K2vMJHhKUh/KwWffas1cI+AyTBVlct8LFHHp2QjrxODo4HJ4dt", + "QjDj07YH2Ct3cS5kwuA7nggZXwFUr+LZykq3X4bzYR/WY58VsnDSCeJ0D3sdvgydjfaBAU0IyTYimvYV", + "DhO1eZq1ztqO2kLrrKLPOkQ1cJ1Cax3khXLsrDS8pQyvxUziPHWzp7jt7gOwsIFVwDfceUah/lHPXXQC", + "s9dhzAzvm7jrZmnJ0DGSr3iWvRZZmy/Bhbu5oGsYtt0NH/IT0Kkl+LJNNdl7RH1DqpF1Derw+v15jVvp", + "AEYS7o3GFsiFFzvTpOKudSl3uAJdgFJCb5lVx2k54x7u1om2rgAorK3Ju8/49imtj+TGqfVV7Ty93My7", + "fJvjckMyUGoYNb9lGtqhaX0PECusa7z2HptA4LfG143WxIW+jYQGWOuBmG/IiA4HzEQjAIMf9c4YIuim", + "IgFIEu+5UGgcTU6b3kNXYF6llajyAmej0gXlUCusEJP6vV0lhzclTBqHa7g8Lgv2+4y9hdc+DiRIciaT", + "apoLzaboUSpSuExpJSievol0j2yiQb60WIbXZm2nah9JYsBc+nXSpmA8ptz80r6U3kYtr/epfVl/JlJj", + "L1/i8aHjaXNLt8MLfw54g3v6QKLzHgDKb/J9zLial4HIqf5YW2DojQbCu0SkLCSO7xliCy9dUBCpJ3kn", + "s1yHJhpGoBn4B52rtWXd3ZAVOuWr7eHRrX6Vu05Np2cltnAb5QpOEgjrg1/O/9Fn7z897bO3785f0Rvs", + "Ljws3ROJ2zhsYp8QD9V4UtiBDxrqE1B9KnKh3Nl8uMOUd7gNBmbyeKdEMJodLps1WNCI2ttqT/7PVcpS", + "6eqflkXMhh6wz8P7R4X9H/qAMCdHtwtpc2EGhVSrXhOccjcPUJzknZfKfj6gn2sTVI+v39/1c02aPNj7", + "s76r/9fzAIURlUIVryjY7w2QP61faeBgILThgl67DHHAAHOyNtxEj1uDEJM54zKDWPR1W0t7XVcIzESh", + "GAMjeIoI2y45S5DxcX37oqfqenE/lUuumqXESdqxpm0bZ8f7ZpuiGNkS2cAD/ZcfibFPg0Rb9GPw4+nA", + "ui7MagzuYAFBqg0vDeciFRkHhXQps0wSr+xW/EvIiwSQa7ON6zm4rnLcJ26dVPMakfNSn136L62khfFq", + "xVmvJiyMdtyktSHYuIobXl67PyZUyxUcPVlGdtQ6wGHtRaFUCIzW8rLAZFE9lwm6suXkuDXJ9HzisSnJ", + "c2Z9S2Se220nsYBMcGBGJFvxTtnQstxxQbEFz7I3nha7UzXAbo5/DdvE26i/fmvbUKlZjU2p2iAh132J", + "54bTjW6n4+FVgNNvOw9sovOtQxP6ewWpvwGwWAeZamM5Y/lfusfyPVdytmkMsRE7DyWdnjsPT+gbHlmv", + "RdJKo248b8au5ZKHfld5MC52HGxI+7cX5sKT8qwVv+D5Ho19zQt+5bK0N7dtTuO2b5hekYYGP3iXwMIX", + "6bhJQ/YI20D61o55MRa5Thbj9Qe275/1NvH17DRrF1WY+7d+L+fJNZ+LcSrntAXW/SAoySPU0r1p7y0H", + "bKB2rg52GEk0IDlFAIK0iV/ZaV15yimeKDU6zynE1J16Y1ea2G4BgKFojMx6L9dG13cytLpt2f5VS/Uq", + "46VttzxVxx6c/L9q9GhCUxy6wmo0K5EHjEfuY1IpYfqANok+F0bOFwWUQCA+weu9cljD4IVAb9ggFddS", + "BW7OTdPm+gTgkm6xAgbS2OXdHaAxGpRvX/pd0A4Er+SKrh6nBrd8hZ0cjhRYVKJUrjcQO639+wNegGa1", + "QByXFPvfwckdFPydCeW3m6+gy1ql4dEApmtcIWA1buGIQlVoT2dU2TFwnguCco76ORwp71eFt3eR9sGt", + "m6CxWkvgRvgq0jr6FWK5evCrZj64njerr4d40j4SSy7BtuWkx17RnH6IMk+ksm2A31LSkLdoV7g/xLYF", + "JCLRuIJcX+s3/6S0hV4SncealKONNV7I7egmroVXlGFNBsXthfXUKUzCImqTJ/gJtwsvAL4X+D/YVBS3", + "Qijsrm3ThGe0GjsWY7V8AOD2IDdyyc3qsBrAJXKRqPbxG8tWJErcsngv3TZ4H33a+vbZ2mBctwdugkW6", + "tcFt7WxMVjRY9ZZ0Tdrbag03WwofXGuQZt/9QS0kAAg6E1ooKiDvGPfpetF/EyvPGY8pA5EfniW7mAmp", + "igD2sFFB5bd/h3Tf+j2IxWi7z2MYigrUErA216WctFUjd8E+rq2OmuYnflvz3nmllzk30mp/YKDJGgVA", + "4vcQ+gSL3yZn7M1vJW+85EwUfPmgC+Rsgd+yYnLG3gkLaH+KfhLxb0ybKP3cpf8RXERNlWUOWeKfo1yR", + "xQN6puC/WQH/cVM4L+A/7WAEbpg+GT2TbULRfYysok6nkbaQSYuw8NFwrcaSzxAlmbuTNkDBwbKD8W2Y", + "TWqrsFslhg3n1srYJlwpkW6iK4E1RekaUmt95XfXifv6YZVGJ+autbr6QDDtUB/4OVDiaojDEt6nylJR", + "OWM3UJtqhoGE6t1hSczPIEkPgMlBzIojQOQHLe3wfo2AgdvYfxjZ7mbA93u0I5zopRXp3id6q2AKKVpM", + "r6SXI0ZQc7OgCJoazdOEWycsXvq/CUM/HGXuH3YB8fJuiriZi3BlYOylIOLKVC7xwkV6AEPIYnZy/P7l", + "kH08gFJRCzn0AVq5MFg0wDxP4MVrjOQJkzP2sxVsCsN+LVbEqWDB1x8SAiZq1AC6gNyI+KKCKUmHvxbA", + "h6Vvff36RpiF4FS9XZSzWeYk5E/cLgYVczvEZFKvpqtQVr16GJhBNDx0lfAPRhZvXkt9A64MdT6CMBH+", + "3Y9GwV0AsVWdQreC4vfnElzf1o6mz+QtAq0vtBt7t2bPaOSVMJMzBhCzRhSlUdUGIDEAgTzRUOAhJGZu", + "8VxiFgj0dNnQIyyIxT778PO7dwhsq9WgUh/DToPC4J9dpUXyrqu4ID5qY+uHA4QPyd7W0fybWF1y1fbw", + "4PQd4z75EF4MhgJlgpzW11Uoo5ftVxHdbopc2+Bo2f7R8Hzhkfb+7rWlTfSvnaHOMyPEwE06m7tCve6G", + "jPAVy9fN8fC0EZ/i34iGvc3t/KBTcQX7EJWljWC0ewRpQ5Ow0ZaKH7KfVUVuQnw/boeTqMZAX54sUI75", + "c7LhsgDGp6B7NJ6LeVIwpVPvFVZA7NdH4iijpy3BFQKkIi5ZqGcnQx+M2SdeLN4Q+BpR7XQQDOxgNXDj", + "CMgIq5Ye0VMJwD7oGXYNZeLnXYdvxxv27i3dWZ13rQV9HhcACQUwD4yNmL1gUt3wTJJNAXqiTTxzvftA", + "lVIrXeVkLdg2p24L4NWLWMGpgW3erT66C3oF6HaYHvpZWsEgQisV3j3s353SMS7EnSsREo56bAAp3Rdk", + "N62XRBkDVopdy7kUZu62GAaTtOaeOY2lpUpgLQKG+RsA8ozzwG71eYaoDMAeGSi+FIHCAIviDFBWaIvj", + "VsKsLYE5u07bt3tY3Wi+I5H29wqNcGdxhX3HoRThiBiCRuNkBd4NpVYDZIurKmKvr961Xce9Ke/R9x/d", + "jsc1aNnGQ97+pYl0Lh6npKVbXsm4Mt6vuYbC5R4XDiZmmDgA5nhbYobib21ywMK4DoW0T1udxuZfPe63", + "WPs96ikQQilh2yw6/hPzFiPPLoXY1H3kBhWITk7Hu1+ONDg17se8nGbSLuCxA+pd14v2bX9Ls3+M5yYE", + "7VOrmVaF3jg1KAFrh89G56gak/D3axujBEdv+o77eNcewuiOp223LW2KHTrhNKz6Sk202VNdeA85P7qm", + "1Pv658fsas4N37FBILc+YfpvEY/yjo+dnzD9VSHyB4gxvLOMAb5H/DGSEqOkYB639alLIwYOWadpPbyY", + "Hd7SqtnxL2oQY0Yitctg/J+6jCtFqVrHT0/vOUXtzteRcP7yKAoBMs58CIyU99UKQP32rlf0rKpYGV1q", + "cMlCCHqLoVTkTv6tOUsDRGNC5m5Ys1DTQ7TflMpcr+0XzzARkkSWsFSX+ORUDyvbo94I9H/3C+/bMssq", + "7OsDOWNrOs5fXL7D3v2XQ9Bv1l7JFLqqQwLEU/HoXf56jTsHTXLa7Lxr3qRz8QDpJW6qyMDdx/ITN07H", + "95mZUDci03lgtbwRChYbxAESK246F0H1esAQt0JQejuIu9/17q+quB3VFmcQtmq8C+EqSSokokF4E3y1", + "uxCFSFq/1R798HEVjztW3RtYa0As61uHLetq1JbjePHApVatizZ/1LmYGi6TltUz0xliMfp245nTx21E", + "lyjcR1wxHkqqb6g/YvQ73vo/3oJHTcU+yxkA3g3Ij0Gn4gXTZC9Ca2Uqwlpyt1zpDUkPueTGh57bF49z", + "zCHcfkWv9limvHW9Ag9AL1rWTzoET+nmkwSHZjDUp9KlWEoFD6+xXdM76Fy9/psdAl2PMBbmhkm03zfa", + "lXqc1FZ7qBGZ4FYwq+GQTgDh60bq0mYB3AZodiSEKiZaFVKVICZSkYDAWIgKACAMAvhC1qjuYUJazdVk", + "lX+YEkLaurfdvABrDBrdw5jgQt09yIM0+veufQ/YdGs3//vw+Ue65hUWtIY79+Mm00Fg84+ur8MHnGpB", + "72+aAqObLgtLG0E+4oUHaiIsSL9mdp+Xdh32YWdSd1/g6x/WF3dMPUSga329y8b5ZaEzMUB7ZOW/gJEC", + "dW+DF0zv19NgDMw5SKbY6PkEwoOvCRZsh9flfXoOxI4PM2fCBAPNGAoMt2vA3M5hg6REZsfncyPmvBAP", + "sr3f56rbAfYBfX/I8QjM9O+FmYtPfC6uOoyS+DvTM2A9X+YZIJMhrT2a24FKf8iuyiQR1s7K6gggVOlC", + "L2WChD5ugEc9i0lHvRchDAtSouEDyb6UVoPTuztPX+dXWp35HstpPVGi7kXRIo03pLgXwY6ND/ArlQAA", + "J6JAGZFo431DxZ07EnhG0VWW4K/RHYsguQOqIpQUbsQhqxPIU3fqaoNv6VgWAGlexvyHU2A5KNyQ5BmX", + "CqEC3Hn2u8wBypsI0tzvI3UwISSbwRuV6FSq+RmknBzGRbOpTiUNOuEBu5Z//wyoBhHv253tWDgEG4VG", + "FQteIGSt53FEBGN6aYf5enbydMje3OVcpXDiFMLSNir0LTcpQtQBI6MZ3Mo0IN4OgKmSPBoQB+9FFMdn", + "eeHdwLE6O1JQ4fPjpzjGk4ilcBKggN3qzTKdgAc802VhJagtvKBKYHC+++4nfetUqFttru3Zd9+N1InT", + "r1TaugxWujTNtTBSp14jY2VuhSlsyATViTtpAcAVcoNHyEg9DXlS4faX9cvHZyUt+zo83UNZsLmmyKAb", + "GkRFPkOZrMRdMU5KY508swAFP8Y3srFMg6OjQmJbmAEciF/OLz9cfPjx7LvvwHfP8pmgoyDwpIb9Qo4V", + "+kaYjENgQdVQ2+ZAvyHqqvEgMoNgv76fbHbrun0LOORTQYOVBsemJQcszGTha8Z9mXBLj47/4JlMeSFw", + "T07Fgt9IbQKDmF666w2Rj79aiOSa3S4kPCDiNNwSnrwRS30jUqBzd7upmhdACOZOw8TdRqowGW2sb/JY", + "gvO44Su8D9KIgCmn4R/eGY5Wn8uW16TX6OJkC2o/LhKv25PU8895QOEOwCH0C7osiWVerLztbtRDlvWr", + "cmpdIhUS27O11VavzAtwoKkXM6mEZZm+FQZHzLU0Wtzau6YEaF6h0GMp5+6SEaQ0eig5+QmwaQ3g0xyp", + "3s6Oj5+2R9PCrG7SxjcZl9ZcR5qP/jky8hHZwMVrZHuif3oQkq+jnv9tLNPxyah3xr4Oh8NvfVb/chq+", + "fPOCXCKQDK5toCRGIeL9emww92TiTiYaVB+Z4MOaywO7QysLGJO4jBUA2njAK3CXA/MLbHFEPQZf6Qo1", + "mVD9+I12izotkZhOALyiO/K9wxWKtCeW3OAAKpeBSjLjiVjDUa4m76SCCum943mhc5AoAFLzww8/DH/4", + "Ae00lPw0Sv5elxAJTKlP1xI/jRL/TayASqdK/ydI3+bO5ATIeKeo2KuVSiggthnKQOuvzRm+pr20k8KT", + "KGkHGnPSyotHbdZF5oGc1YROOwxgJK3a5AsdabfCRAW31dasbIi+eT6epvl9HyK3fs9pkHvEcb6F5B/9", + "4dVW4rVYjY13nttUVnCyIy+lbgdo4EyPHIBgY9bcoOnEJ1MNCkJP2NiC798FK3AeJBaTCrcmYJDgfh71", + "bKHznLDWF04ToxNrNeq5z6m7tpWKDZjS/iBlS56KUa8VHyyS+LucQN7PEEXAATjeaFPTP1qrcTeZfNNa", + "pwRsKhJe2oobeMHtInh+H5QKu5R2ol7SFWij5aH13hTdw/e43/bOeu7+PXxdVqsR1cZNnZUKU7idDWGd", + "rm/HbIct3QwoxrZHdYY936sGPWyx+nRThzuk13WZo9E1IvPfKebxFzGtZfzWX4P/6eL0wmqZp/Y6AA+w", + "gr27+PC3nz+Nzz9djP/25v9iQt2wG25a11n0DukdkJEfzaRrPshE4QpZ8GjHqxXsZmxKfFWNixF565UV", + "YXLHxZoTtI3M13YdZRe7Xbe7Ri2KW9EoCNWL9FzZW2F2AtL70t8anEvtofE5/3QBmjj9KgGifFaiZelW", + "TMlJr08XqMoFr4nLP1JGkEOfu5/NMn1r6dp2JYoyD/c1OVeszJ2EWxRFbs+OjjKoe2g1XM9+FEVYIzBZ", + "gUQPS3utE3f5C7lTndhhVESr27BdXhXabLCjQDRVUrB3V++ZJ0QLVyiekbF2yF6LAnabU48JowNuzsKQ", + "NSVAunuOjMgXqrFRIIJgPC2z67F0krwYTys7e0UZ1x3lQSXI5bKEl51xpucy4dkYrvQtOxA/041fKhZy", + "ujs9RR3wWw7MlnDQgNcWEhnXbIPbG2UETylWcP9cYzj24F/j4k7ds5AE55SCTO9TAFFvOU3gnkWAD1rO", + "E/HQzkD8/D3z5kZPxQPqhsvAPvlhTfM8Fyod86IQgP/vtsl9CkDks7FQhVk9uBSySd6/nAc2Y8azzMmX", + "caB8RPj6xyhRZPp2XCyMsAudpY9QYrgtugP9Ecor9NhLqnsMI21nFHTHTpG6V36pFQx+pueVnNwpP0mD", + "JUHz7Jeb1l8s7e+1M1rKuf8GaSnscUpZW+Z+3h+v4Ies9k0Flyog+Dy40HtJm1lW2sV9spCCuteyrOXc", + "e0u5K+oY3hfcjtyr4nrWvWuudJ69aq2y3WOdV5nvMUVVZq9q3Te/0QWGre+Vf9/5uc+kgD481jfCOPV/", + "z/rqefevW98Kgza/PSuOMu5d65K7f4Br2TgV0yIgqOybGdyad894nxMo5EqlQUrk9VeLkAZVvj3H4q6y", + "ue6U4T47+AH712e1iud2oYtxkml1z+obZTxGO3LBr/drjJ5anYnCLT34P/9af9/8eLnfP/suKyskBvfS", + "vcarltWOkSFyPF3R/eSel8Vtpe5/e9xa4j2uk9vKvP/9clvJe104txW29w10W4F7X0m3FGgfXA4B7N23", + "ICOSjMul9/69RwlktxnPtIGdvNq5nMo14F66GKy8IMzW1JS9RNqWsvZrV7lv5fv23OpZcU/9t571HjXD", + "BcXbDferOc66d82lCqGc41ue7W1zNKWKjIxpaQi3G4+Q+1gcmy1yighYUre2jRs3BizzbUQ4gsdsm2tP", + "shDJNfj3eWtPS3QT8a78cv6OVelZpCIyaRllb4kgXq+KoLDRxtAeQGTBy/RGVG5+3uXIiNwIK1QBLKnN", + "VkHR9xwBbBaAj4+VbQM1VJr8XFmpCplBewQ3mXRT0NaSF+x3YTThY6SlYErfPqh1XUDul/VRwWRhzEIb", + "aZY6R23tZck1Alz5SiPG00wn123vi/EScbfnATr2BUx4t0Aqbi4qpnupILs9nF17CA88d0NjI5vaXkKk", + "uxhbiPxBZYkbp2M9pAQ4DzVPH6cQku7udrhfUQAY7M+Vh+Tfr/o2fxqIrzjPsgBQ0eAm1NpuxQ19CYl8", + "MMmYZ1l0UfBVNdlGQtIvXa36oJV4xGYBY9SO7YK0nQ37tDDcdjUNqAt/R/16TRzs1+4AHroOe1n+/rv0", + "8A4bnW5CwjAOObR+O5B9LXXnWPxXjgK0cMeOtPdApJJvJCC9UECGPpWKmxVbuvTBxeUAmHL6TC75XBC1", + "UUuEdSsPToMyicp3aXeCPV3KpRi3c/q9v3j/Bkn9sH1Ht/yG2ng0lzP/Z67mUZO30qFCz3emKvO8M6GV", + "7YNv5iJm6Njs3EDxIrMSDkRw3k30copw5IC8nWfCAw95aL+12TBcXY8TrWzB2yiCLy/fsmvmv7ODk6OD", + "6//D5Tk8HLLX6BYCoQbfHw+Ph48apm4jEMZN+wKGrcJ07PeIuHqTN+22Vq750H6A+BtPiR3C15ASG6LX", + "2CSgSE2CnrQGHwUZXoxUAI6KikCVlACjKizGn1UAB/e/MnLIcQN/MjwesnOKhfdQgm7auErZ5dXbNXfW", + "0Mze2fHwab8HJLLj0KDe2UnbsXgrVapvx1b+3saCe/WWKTekGbHrM0zOXPL6OpmAJjcZbmbfuTcWld9G", + "nZi5FEaHLucVYi4M/WI1NTKlqbo/hq4rG5xKCHTsfgC6FSbZQrbdaSpgVfcd3YfWlhvFE0Bv2/kBLXEO", + "71GLz7JHJffbyp2zW8dn9b5pxszWPNLe0zxHqK2VkPQTVCEP+MGgngHcU5gHWhcjZczsjF2KROYGSJEu", + "ubpmb1EOD6h8YRG+xbLSYl0huROf4KteZnykjIXCMrxnXQFxdCjLb6qqtOmKLaU6WvI7T9vM/WZzrQ2V", + "33oIEcxXC1bDgTJ21ur7956CdCu6lJ9VcPUK7Gdb4tabLJoh8tcKcyMTwVINWC0FW4kCQAskRW+HKIBk", + "xQLNy6qC5pquwGmtzNc3aELgMdUZjZVWdDGrcVn1pbX3HZRrhPG4aKNKc4sDY5wmrgUo/aeGK4BPjcPO", + "wyhEHdvWoMi/uiVuBAcl6mDcfwyNlL/XXmB2pFOLJdTT0/240vZiOyO6ujbSs/XO7c2A5hfzO7B8P8JC", + "9r61CUQWuAUsbMHBJMd4WSy0CZHJMYNJmPkMGrLH2sUM/yLrFhvzwDX7r7jU9ltUTrl+6S5RnfG8ryLA", + "EjS1RkGKILTdqZD5kGRE2+Ce6GekfjS6zO161ukqYnMZsvMsi79ywElhAWfcjhSvwgkh1sLVxrNsRcdS", + "zK96OsgX3Prk7OD006tDcms+h3yyWH333Rl7I8E2x7fUzcAjCOETmdLKyfxhI4iu9EGRLJPXxBo7EwYC", + "T3gtWBBoswN2OlIs9UcKWFOkKrg/0QH7tZAQdFKIOexGahoRqbSEgd4rhorwa+6NoVFbQF1Be9Vkg/a8", + "th7wosGLAB3/hicLBg4DbKazDDkHiCXXx8fZsPxohcZNWbsvQIweMVFDGHH48ezZ8+9hjyP4AQTA9Xul", + "ReqYM/jr7OT0ae/bt2/4e7OYkMJtZWAdOuv9qhfqv1P9w0Qve55frfdXvVDstRauvHXdsHnzxsnZtn8r", + "tt61ewLGMKwDEDUmYcheaViAlgHRMHHAEkBvy2LrcMt/TU8wtP10WSTa7fFPZGIHajQLpv8ItelGWg+H", + "oc1IAbp+InN3WXfV8jnex6RltpBZ5oETnIB3Yp8Ew9iInEsz9gM4GakKgpt4gzBO0lbSpM+mwOFQCLME", + "vB2hjEwWACRGryxMCZHakQqsLVgPqKqBSn/qQy3wfjuTShYiW9UBFkKtvX7197jqfXhuij87AXIjTNfH", + "WpfbGcsfY4fTElvb4p/CasEVZUIwzPrKbl/DYHB8qdMHG4R3QULuBlD/1u8FvqGthjN3OHsGpK3mMx92", + "SO2j5F82jkeHARYsYuNgId14Da2P7Zr5NCqpsykbTeL/tVZvt1/rc717iOtGVlAsuG1IIvT2LnxUBHGj", + "oFPEnisMvxHG8oyUFUCscH8gqI+TS9Ldmu2+RFtU5XQFkVeU6v64h/twbnWyT/wnwK9/KJfCyAQChf/g", + "lQkB1VbeiPGSwwR0EDfugQrumRujopHB8vGKprauW4c7qtjdph1qkOoPraHtqPiYZXzJ34CBV5jdHxjc", + "2Y95WWWuJlONwUhKsGdVJAiEXypv3J2TA5jGDJ6LSiuG7OfLd0AzNClNNkEQLnc1mHx89+78/fn4p49X", + "nyc+JpbuHW9QDWTvdSoyCIVU7iIygCrR0nnwp+//zFK5tId9trybckkfEZDj4OT49Jn/zLNs4O6W2ZId", + "PP0z/dweaamh3079PEKz338kf6lM5DX9eOna5qaq0TIUSjBYbjqgwF6/VxqX2NVzdnQEpOoLbYuzk5Nn", + "T5/1vjUFGRX+tcUSoSJKTJomSO1JLijQ/kmzXU/67MnaQD05rFMqtnRmnZc49K463ambbboUdDy213aM", + "QZvVxS2delfPP10wQfQuQ/YKkEutRoyqApZZtKzcqpJGK1BNb7iRnnel6nBnYyovAiO30kyGIenTzH3p", + "3I8/Ipylvt+GJDBMeSPY+UW8KdeOwiW/Gxf6WrQ5IK3TwWBKt4Q83Ga7Rf/+67I+7PD96fDp2Z+Op4+w", + "xgqxhJtGadrJmwujM8sMV6leAuODVGEotWIHx8PjwenwuE49N8s0dxtgiaPVOzuNTFTHLU+HhVwKXbZY", + "hWoIcoySATIbeYG5WY4WeMKzzA63WsQKnY+v25A+8sG1u/nnGeKdG74UBRof2wvJ2x6AkkyUtqOY7jE6", + "2TJGQRrsudX/mB2ZC3V+cc8TEvJ2nZA+uh/OPYKLqJ19n958OL/wiBCTdkEVcYhjbQOPg5kJNzq2eax2", + "np5OlA9CUwdPB0hK6IlS++zk+dPv/XG5lpiO1KfHfzrddHbmGS/cHA11LhSXcIimOrFH81Kmwh5V+kL9", + "IA1oGj17PRgOh2HKznrt7W6csFDb+hHaCdLxuZo9mqaOg6Q2R51nSfP2AOaSjypbRYD/xOXYIoo/Qlhi", + "RPcYXC3DwjoorbDs/eW7BjGkEWkJluPDIbsUiV4uhUpFesZOn3/fZ89PTvvMaUE4uWCNdTO4Va77Y7pz", + "7LeIfhzZbtHfWe5uhwDO9s6KhluYPJfRmtyqaFRLo1I0YlcGTPDEArAIrJz1RfPy/OrN2BW6mwbS2so/", + "St7dVwPBcdldA9lnA+60j2Zg/lQJ2PV4VrQU/Qk/YKOdTsNCJnbgjng3gfud9IPTlmPsD9eu/Bqe58Xg", + "2fDk4duuKqhlnwkrVCL2GVaf51FHdc8N/y+i9f0LKlAtIuwPEyiXuiyEub8Shfm7FKkoBf1uGWelQr84", + "10NXVnB7rEqBRpOPTypn+D4YyrbD3XS0y48/f35zuVVPa9e6cOkebVG+DvudCUHx6o/UXOt5Jo7mwq2M", + "KMXx8Umf/XYr1JH7z9Poy5+nHfaNXCgDwznkErUznssj47kzd1PStGnoaRt7uq6uYQvuobLRStiqttWn", + "7b9QdZMzFqA2PGMRDNrhzifAtrHdOFJ4HiBt60wKE6xDG0t90mdPNqy5ptFoWwt3VuxoXXx5FNn0iSfX", + "fC7OTSFnPGmDL5Vz8uGoumIX/PT592d8mpyctkL2evaU4CJDgXNu9XCLj9YALJXJqeFg3+fWinb3K0+c", + "VNUv7gpcTMNfrVZbhTS0hsrZMAavRe6uBippsfx7ns2NZFm+VRdhIUFQLRmj22C01wOyKHjYP7x6JrKq", + "70hi+5eT4fHweGvP2xnoqi7/rTFPYWBbJ4IyvfeT2RLGUcyyFYRdUj9anz98Mqk2J6M1afegdawv5pYn", + "uuAuJ4UdB0eBnWuo3D7bCk/9CpJi70ZHq6+15Eh8tYxVtUtbHpsg/mtbMy4wWZjcaBfv0HpYSDFsiDuk", + "osldW2D26OakdZE9YKcZAdQBFW/Wts1WmNIGBNdtiRHBdfd5/RnS+/G8FLO2ee1e/2tRSS0DS6NVFdP3", + "so7WQzX77UIABWt9EwfCt/8s6raMWHW/PowqvU33QGbidfJ5djCtRVQd/iEEbrrg2RhdzVv4Wsol4M7z", + "W2Qv9NEzPNNEAQYUnMwNMCLBo5uTzLK67xoQtIbwG9BlHjPU6AEv6mFtrLubeKrGvbD794iOAozahxBj", + "I2Xug0og15775q8Wzr/IVL6VKo1ceptqohEBI2cbpejrkJhoTSHsz25yRaEorwQ3Cyb/A/bsde0sOIkt", + "DsfHjxKUhVaxdQDrk+OosuePWdfjLyRo3B9SbrVxm1znADrtBXib/H74Tm/cEeH3P6RKHLnxkqIatp2v", + "buMh6fF7IM9rosPjmIWefHmMbd7OZRHY9/4z+PIQzGimS5U+SDlAF5VxIZeiGVPxiOv1v+6gefhqeuhy", + "iYqrGWWclFjofB0K/zybayOLxRJ9ooBCWIITMnDk+Hxn7GqhTQFQKi7NdMUWOieGsoOpETwtFoMZ0CF5", + "eHo5VxAKSPrUIZQXpOAZ6lMQgUTSlhHLjNPGaprYgS2MBmwkliDHttSKJQsu1aFvZUupKLaZRf2uXmKm", + "b7E4WzAw4xzGcUTVaEViuyZrv9xf5rgWNuepzrrtegAJIL5kbYbaJ+i96678XUTvOHCBaA5QSEfjgn7s", + "wB6oxBxfyuKxak4bPBe5/NFsURm1IZaK/fO4f/Kl5h4fj2w1mLVhftjIFsIop9hcFSLfk+eSGBo7KHJd", + "31pire+tbLkew0g01Zx/dWXjj2v0H6TJdGi0b4CoTZtA0e1pt8QN8L167fYFgyCuW2kF4+5WZwv2/TNW", + "KvlbSfovgwsmbNYqwd/ky535bMHP2i0R5C+FCaQghu+frR3bWDV9x0X9n3HtqFidH4tzGv3LDEKDNRy8", + "UcbdO6ZlG9vx3gPgbd//ObaYh8/UowtAW4iWWHaeSW4fojOJdC52pBQPcp3sTONZCNXYlDsK6njQuApD", + "qG7r2rinHWyNz6MLE18GX+cqWtL9DZeb0gqDP2jDnnz3BOP5MuAkw6fOOJwSYv3mmZ7yzE2XKpovXBAL", + "OS54TXLGwc1Ek7gLu7LvgpdNu+SpBquVkTmMV7MpGyKnGkWu3xJXOXiX5CFdHBruNFT/Ytrr93i6lPC0", + "Uo0ZpVh/a/n/U4TUlsConQOhPkGs0R/c6SrsadvzaSMKL25nqYBAqoOH26PmAHKKvhZqMOVWpCylmCaW", + "m1KhwGrf0XZ8LfJiE+BMSMlcSqKCroptAwbwRbtUbbCSbYVjWlZoNpMFOTUhE3OHm/S1UF2Nf2MLueRI", + "YQ2eXlJh40N1m8rsanVbqdTqTeV+65pWs4tXDg20x/jxUDk4y1ohrAz7reSZLFbDkfpJZLlleHAwXRZA", + "GmZEJm4A0DXCQro8/5HlMhcZ4NRMVywVhVPf1XyksNQ5z0EUp+JG+jh3XGh5CKxmn6iBgLGrQY4rJPNG", + "+Z2tEJGIEHBYqpdcqj6tIyMMV8DRDMBpnH4Qhkk7UgkNh0ghiokAjvRsZkVxhFCkQP877wq4gCrHc56P", + "c2ES0YasdoW9KUpTwyKC5hQLGAQ2FcWtAFy3Cr/W4/kg7blFFkuqhc+DHyBoztR1pPA8WPI7BlfWw+FI", + "vYZBt2zUE9lU39pRD+IQcKg8KAQMvRFzbtJMWKDexxQ24eDv9FYbRpK+z54eD4+ZLXRu/aC6PticK8ue", + "Hv8337KoTQ2sgafH/Ue+U/Gp1VlZRJwTjYAMbtLKngENCzQ7Q3bp5wQYhwl/SWT6Fgf9hmelQIQNcUf8", + "sfXBggtOtJWA0rtJDn08PD557H7j+oPtvN7pvwmRM62A1L3ZQUbOFyysYvYda5TYnPfj4XN2LURuQ3ku", + "Md3aMsFtwRY8m1Ur0+/i4Uh54DoQv8qvWEu7lDQxgjiEgELqdbWB10bz+WMOJh2XY4z5Bvi97WTxQCns", + "RiQMCJLk57m7BkhVuSl6PL8DQO0L8G0j5fpKSIAogQ8BqcezVUqVGMEtekmKRML4TD1KFohSPjcCnGuC", + "FKm8HwOzJVuKYqFTWpL7B84GqMZ0nIqbiqKqY9HF600q9oF5LtJY1PsdJhDqAhZEc8mdDI8bS65tCbvs", + "A3bynS3SVNwMgYPfj2CA/uOZO64yKYw/iLQC7HYECmxhcz95zCXWdkoH9VArsQNP7WdhlpjlW38LsEGF", + "j7A1aQNPYaeiY5V+W4Z90q4hKmzL4LT51Y7tiDTybWkvxVzc5bul/UVmacJNultqSHUFavpuGdZj+bfl", + "cMtkj+SveSEg+T6teomiY7fEr7T6tVSw4XZskrT7Zahjee+UusLY3toanVykuw/LW3dd2y35xac95ulH", + "oV/qEox6L/XdznleSyd7k/0r+aSz1XzXCfhR6KsFz30tX7xwe1nKLBWm09cCAIcNl2oTjG+b1e2j1w/E", + "HSjMGpVnKit4Zidam1QqXmjTZ7ZMFoxbD03bZxOeZfpWpGOEnZn0R2qSg2v8mA7sCVJCT7x6gLXxaSYm", + "a/hZUCp6P9QLQTScdBVh736pNI6qyIa/TXRQpHT0t17SPVs148CpbX3MQ/A/L60wrt85N/DYlGRSqGKQ", + "cANYTIlWhVRloADZyUB7PheqeE3NanMHpIEZe0NBm09bNYcYSRHSOjVxIbIcmo8gMFNcSWjOo6ANoB23", + "MEMNcg9Q2e2Q/eLuJ/59wqUDxRSshwtuAXYcwK2UZtizPnmimRthMJRHWFbmrkGIbEdmD14WeskLxNir", + "jdruTmAtozb3oXpbHzwaMX3gHVu0XkA/8KI0PGMZV/PS3Ryjz+Gu5lGroYjYqvdWqhTsqIGPhgH0GNBq", + "T3VZsCVPFlIJdwPAK26A2c3clWAluGkzC4JyhKTMNU27bbdfzBj8DKXyOQBt8hXdqlmScSNnRGLDQNKA", + "ggmXUyVE2kUZ4u4+0AzFs7EsvHW4O8Ivz7iCLhZaZxi1zw6OB6fHh0P2P4SBKDxLggcDtxGxDfIJM2Tn", + "LNdWVks0ACRCWMMAX0PdNQmWqARE8MLfiQjSTTAQrSRTmS2nniwF0ZOgeq60Ap339dU7t/FBBLh9UqpC", + "LgVgIWVyvij6qFHf8EymZBUSIp3y5JrA53MuzZBdFB6Z1+03dg5+70wbH3rmBmQAHumZYGEVQ5VKuL2E", + "Yk7gnpty69ccYtVXwXfHreFwDUpGgB6sTftOYZn1heLWjl1bUTioYQm7mmoNPNnewFanhCDsAkb1Qqqi", + "7aAasqtyPifUP5BjwMI/cSLHHVoV/Lw7rzyUtfuCuOaQBuxz7q/CCOFPMXiYmgzZz+pa6VsVl86TROSu", + "xpk2IzXT5tbd1uqr2JXhBmnmxAGskEJTsxqHYc/92LbpkZS3xWUDDkx3iGEK5iMmugYDDgUf9eD6Oc3E", + "DXV0pCYxUtdkyD6v7UgswEf/BdRElCg4DBUMGR6Yzethr9aG9lciPOL98Y1osy2+odokgnFYbIMZTwCw", + "1d/20UIChqglVytWKiOszm4ARRFlF8u5tXtvJDxKxhUkYANRmDzNcZvCvsbBCiClgOCdOrH28UYYA6Ga", + "KLiwaCZnQRepPa/9E7kIAKXRnziEltnvVbxnHHxZdsesswLEIGCSrhu+jQHsdXfpr6LjEPQTyDbdwQZy", + "IBxMWg3ZhWKfACz3BKdAWiaShXbKFW0A2L2wQECTCHjncplnK9IkBm6cGLWP5cIQ8rlopyCpwj6a1BTB", + "tuYRckEtIhVp5mTHRTXmfTqO4ilxHcZphKG/K+rvnv5o3xqORZrCly6rRlD8270va5K44qTqesUBDbZN", + "evPMCJ6uYCmWSyTGwHmKJrGDOV/N3EJoe3OGiN4nllVpsNxwsgk/5BBSfjI8PqybJv/cajjaOeRb3DmF", + "gXu3qIYpuwHxXaV1I3ULGMFBa4bVCCCsixWTBbvlTtKFGwMM1S24VVRr4Mo7Lri18qSp3T1xQ+GJgHAl", + "nX94XSkHuI9habrG8aTIVuxJ2NRPWlVBr3ptWgMNYVdN+e1CZt6ZDvUf6c2F7VPvBuyet00ajWr4w7kN", + "Gzvj+Kimp27fU3jdsNeySQKG5O6N+DGsPdLWcYqlYhiByl66I5DAn4cjRUAlJTzXgmKZQepYexw2WS8Y", + "yIfq8Fy75SZkzLFgrqS36UiOE/Bqr7luwITgkwdhXwiz7J1VMh9NB+2jFU7asx1MbBHedrgQtJwKHTcH", + "cZdr66WJcPdMYKTAY2KAx4RQNyLTudjvyvx3qqPt/DJiifDmu11KLn3yamsgeyOCT8AmQfVyF3lYVb6T", + "cl1V3i6Wwa6yV/X0UjHea64vfbbmpN9TF9hiGlmTXBDLwzPZRg1wFb4xeIcgtR/2Yzi5g6iOzFORgh9p", + "8toEBb9+Zlc8T20NDCDkW5fmFSaFTCJvMzW1boNgfysMT0Q15dCrgTfbmHJP25L3U2tuklsUJy3NO1cr", + "Jq0the0zsMXh2oWR49aWyxw395KnAm0DHhuGLs0ehDrSU8GYy54kbuq0WT0BzQ7iOPAVHU1GeO+uH4pS", + "2ULwdB8NthUTu1PDesVV6q7s4iU4sbwBNwGR7sZ30vBHghLGRvyK1oaILLabf8Q7huF+TXxrxlSYoPbc", + "gzeks26rTdzKVBTAJLA9ZbcLVSPhpsrDTqKePzs9/dJfJzVpcc+JZ7WNlIQj9EPXNHS1smtAOrofutC9", + "pLSaZTIpwhpKpdtgS7RLoM8LUKq7PxdSGKcxrMZJaaw2wM0quvb0Tz71K0h85dK+oaEodK4zPV+NkwVX", + "czdMHQ9blO4VJsPsFabM6gOyRwj/+26vqt0t2/rQ1tqe8PZRwSDch4PonFm8uHmGBa9vgIRF1a/CnUET", + "uHd1isEbtjMQhVLwvWKsdEGhal7xilPkZTEutB4DVFFLCs8RLNqyuyMQCeJbP5e5LYzgyzH1GTYHums1", + "s4bfW/JgsR6cdX8SJXDJBnuhTgUSJ7m/yF1Ozw1fgumfLbhKszp3+R7cSP6W/3Ayo84NjV7b91t+SAwf", + "PP0M+j5ZUpQN3LmRDYUrJhUYkF2yUlUASIoi+BuwDM0jhHKPmwa1qKjGt7ZJDULeF1uD63fHOrV/3DxZ", + "d5s0PZshu8hYtVp4ryqvdG+mwXi+jsGZVXwIw15/yyKhPsFcfWl7y1g/no6P+/c7osLtbH3N+ZOkMRY7", + "LcYgaLcuRZ5lJLiVbuOqDd6mc6PL3Ms/C7iPQAmVLEC3DrQS3AhmRc6dEInCAIa9esehuPHULRYowQjV", + "+/LNHQOt7bjqrjDhymmK/jkhF2awkAVzjbYFMJCmJtwqarvd19vv+cR2UxvOGYVHQNsrexhcYCyTBTwH", + "/4qvSpTUP2K2BKOEKsGDey0w251tYZDWjHqBWMtvQJ9rmzERM35p/hxNR20svvTXFgReO9HKQUy7PhoN", + "nreCO2E1GGEuyBAK9wSo0o5UsdBWhCJ4wTgL+g6D5rJiYXQ5X7CJb+ZkyM6xgCeWKXyymFARk5Gimomk", + "ybURD+kiW1UGaJV6yl+LtmUjBDKyopWL/CoH0ISRIk8Gf08NZvnC6AzSVm9f2DOALpz4RTZxLeHVzqh2", + "0GBq9C3caiL6q4uCuZkFI5SP8iuVLEbK20k8gAstNCNu4A2jz4DLAzQVl4EmRen4cSFZlOoaH6pKK2AO", + "or6yydgptZOjCcXAw7PIhKHuiX3mSYFY3H5m8fXRTcQA5oGuZtVM0NtXdstXYPTJZAKxAcGpEwLsRspi", + "uK5IZYL8a/CiFCBRQdpLy1Dg0TDYxkOsb+LZSLne67KIlg7c7sO0xOMV/MSpcGmZWObFql9b7SMV1qoR", + "4RVryCZh40zQEzjxKKnVRqD1H73Voh7ujagjlXCwcYIxk31cyoJNwm6YMLdijMhWbtyMu4Fge2+Oh6cx", + "frlc4vCOFK6OAXQeYxvAAXPBc9EKKxwLpZ00+fOQYw/GliB5d63lVSWqd64kFp47VQL8kC9XDyOGoeO3", + "/fhAekpU6WgqFrJYm4VxSo5kLecxv/U+zD4RIZfDAk2FsmJACZwse8EyfStM/J47FUUBALBv4UGudqaB", + "op9iAImy6L4iVMpVYfvhUc6tuFC5nqH0mLqzFzYG3op8PnTTdpoY+qQvfFXogwxLPLyFK60GWDGyRNuw", + "ZOlBeA3at9VBmKgNwyhGE+qzAkH4zkup1bR58dqrnsipOeyCdqHmXLzuQVFw8UOX/Eel079CL39gK3Q/", + "xWz6INnQdHZ5+TZiY28NMKUGQ1YsFVreEe9RqYjoKk6DspBFv+LaducUW8j5IlqLeBQgc7dT3of7TjA2", + "qH12Q4PJUmN3esqA3r6mDC2V+U9QujZtiqo2BSIfYy+DjXYhiyH7xK1l3LL4YGUhHmGMQVEjVWiMhXKK", + "Aj17K3FXHOXulNelm+S5GLJ/YBVwNpob0FikYX+9+vhhpCDWf8g+oacevpQRrh1JHleGjY8/PN8H7uwa", + "KVKnnTLk1DMxmwnwH8NSnBiYjGU6YRw2eogZ8fbXjlhyP5Zu6HAMfSTy3nhuoSgo4Vu/MpLtNNM/yaK6", + "JDW1Ybffw4LfcMkqdr9n3fOcXauoOm8jrNPdT+sI4xSUQLtn2D1eviAr6sq4fCoH2C5VJOipOsuQZ36n", + "94nQcnAkd+dqm//rTeVdsFNhb3wGwJHcgaJ4bRoCXzHqgun4QVNBCmWbX936gQrDzouCu3r9sFe6b9gG", + "Q68A+avx4w56zo1QxTjVSTus+FX9AgwyEXQBfasowjIVRt6IFERjK5Q1VuGuMa0n8M9KFvGrYtDf3f1f", + "RoNUaMarlrjy2usDf9E214iPOf+t9O/9AyXKwrgLLqXHd0aIgKrMFNUdNpgqOkxPeHFreztdQg3C752q", + "SJ/J9ex2IRMkxg/tAQ9WvK2BxueuvMNWP6Ptku08FlV7iLhKtO91jwDMXCWLe2RsB7jbSXy/6zCnUFtI", + "WIJ5ThUImRs2fL9XizVYm2BfWdszasCYxTuB29rrlqKF3APMOtw/2k5gvXRl5MUqzBn+9rswusbY2aXr", + "vXRavmkofHzpsWeDS2eIKCelNLr6oxI4kHZAF5Gdtb51DY9I0ncbEfsZUrf1uRVpZbfBapTeQhhXgDAo", + "CH/OY9gOW1Acsg7HrF8WogiKM1gl7nhSILAKXu/CDq8Qs3FpzgvRuiihqBaHt9BOSFAnNpEKr00toFkt", + "AxpvRaytX3XxywMH/JPRM9nmuIkKukgjX4kc0kKwBIwY+loFzZhMKJUBKpwi3qJsRWHZBIsRZxAdMWkz", + "niAW0VIURiYtmx3wnRh+ZjMj7AJ4c5ysjohz6LaCzCuQdsBvufFWg1ttrvdD6noPhfgBaznIf9VyqyHm", + "r1qqqAgAgdjK+O0SRZn8M+J2PyNMF2W1C27Srd29glRxNrqhbcykTRGydB6IUVAdPQ1wtaJ3k+oFpSm2", + "o7tBq619zQpfZfkSvULEKciDvM0yHw6z6qkmDiOJTK7ezts00Q7hGX7v/tTeb/6I/jRffP71uwWXZJri", + "+3TMqfhexxuAcK8u2pWaB7Vg73atAWNPLBqoJ+PqsuALjW/0LZrI9rHqV52vdsteRl7A33/pbuWd496P", + "R3iP7oPNw7+mBxAHrdDgeHJ8jG8m9+05haXG0iE4oh+7dlbYk8cn375sfR2OO4xl79VbvAKg9oUqQ3Gr", + "B6gE+EceVpR5Ju7d49iQdd/5Rif48xmC3m0YglplX3be1M1Teuc3VdD9H/yiuk18VGulQ4L0G+/Da6PQ", + "8hUtib0v3760PeGS/j4gc7/rZ3VohKga9MymleMWC60Vp3jS0vqjRWv72JBPSbPrTf/6TSkQdWY9QezG", + "svZ1zatlffAxQLCzAZX/ldN0M16I9TQ+kr07c8s3xNHavhza++tB65qf/QvKuFRp22IL3/XNhrUoZjOn", + "iK19Bi1yjLhg618xUGntZ1LD1z8E9XLtS6yXb/46xkLWE3HFs5VtmxQ+nxtAotbKdhXuVotsy+xNUmMj", + "3AC3NT5HdLm130FrX59PbYScq7FHsNu8//vkFyXVvA8xdrfcpEe4lPwDBLgUoMsjvBoUhivr7oSWpRrE", + "BM9zfJDudgoaoqtmwwYej9w9kXPPqzKigISGkaVKw8KrBgQg6GVegkc63dkQmK5MrkURwccNR+oNTxZg", + "wAQfDggdTcVMKoERmiGyOOpSP0Ag4K2dmEiEbSZkSYwTCE4VgKQwWTfkgg8JYS/04zIsuY54vL4Qvj5S", + "biqBJddrx94VgGyK4CgnC3rDtEP2gbxpfKVD71bj1IiR8u4z3uZe9/BweqoRsCxg/wqnt9Z6a/QtesYE", + "+nW6Ede6c2DLZZ/xm3mfLaXqsyW/62OJfYh6s30WOUoc9keqmrZ6QQAb2kdovD4Dr3n6eyFtAQ6m9HP4", + "tytuLnSjoLnQC24X47mRaZ/NhQ6v9Ic41yAjAMHzwMq5gjgdVYyhAYfQZWIxPRupyWTyq9VqpL6OFGOj", + "Xm5kIsbQs1HvjMGv7nenebgfRj361vdf4JKDnyDzqOe+fOtjeRREIUVXcdCq9uJ8AEb01crfId/JMVQy", + "Ut+gCzHAWXVhDrJy29716SBCtDUqNWBRGPI6g2tLBcWIpio9ixwPAqyJd/yhaCcfeMKTouRZla6OYFYJ", + "PfQPkzbP+KoSBlijHbJX7v8HBPiHMmWkSCASVAZXbS+ufWY1m3jtbtJndSerPsLUTWrn9iRsqsqjlaxU", + "5JFhyibuYychXf3MXn9xIuDG4OAR8OfogQM1HGblUmbcyGLlvQnq0I4+/0gBcGOx4CpCeGQxwCPsjWgS", + "8Hc36EpwM0D9My3dPR+84SBsEq/CqbTUkNqM1iOET/YxMG9zBWmoRJ1IGI83fnMjOMISbBhC9g7swD4X", + "OD3CcHmni6q+5nAHHSBAzEax2CT8NwJCPnxM19SgbkiPn7hKMzHlxjKvQjO60SKGiiLOu/Dc52PFCCaX", + "lLHhSH32+RfujEoSWKe6yoi2KaSw/vrVjfwQf8L/+8CX4tu3CTFUv0ZX0u++O2M/W2HZ548fP7CDzwDj", + "/NFIAWfhR/Qr/KAxXu6QHhSh5U+PB98f/zcCSzYiLeFMxbNiETo9Ul+/IpXUZ61xOVCjvnmZ7FpzfsNl", + "Bq6eP4ksF8Z+9x0wsEyqvBM2YJcwWtb3FKLNl7nbbtB8ah2sx6CluEJxKu2ZOw0GbIJUjO+5uRZmwg6c", + "sDk8Y+dpyv6N4dUCXmCN4SsSoIGgG83ph1SQBCfdCTuQqjg8YxfwTxTHNkesjirjqc+VCoxVcXXjUjg8", + "YxhNSO62GiNOCz4tnaiAlgBXzMQmppz+VCwzHI6lvhGW/fT5/TtW8DkqNOIOfEndersrINNSpJK7DL8Y", + "nhOy1c+XF6hV/ijU32SBcKRLnfLMC20c/99cvjeE8YzDbaQlgOhEK68D2xrxuZ+/l9zKBCbnzK3JrpUw", + "cWlfRVO5ITWLZ+8vsCUZzsNfjqmkzzRurqObSgoT8ZdRbzQqRj3fktIWehn26xmbfJZFJs5YfVcBUMm3", + "b6OReqnTVfPrVKcr3x7D/TDRGoVW/Zvg8BAeDcTXr//9Wqy+ffOFQelfvx65lFDYSMUrHYENwEIIY9dn", + "7969B8G1BB82AszJ5DUpcSNVYE8uVIEMQ/Qo/w90wUTT1kjxslhoc8b+ypVgr7UYKbe8/vlvT7+cMS77", + "xEK1zKItfOlp4/3cQ0OviNAQNsVZoJ6fy2JRToeJXh4VWqsB9gf+dll/1OzCLaOl306tOXmWXwt7LdXR", + "XGPOWuxz93Jrc2tosXTsJ9ThAazaRFX8XYBRq9lamIdQ+Oy0w3bJPnGtnpDDRAipaiuMThR8BIXz8vPb", + "wZ+rciE2z2leFCXnSicaHTpdCe/DaWkrXbJbcIXV2I1QzgDwwaJeVmcVRDSDd+2vpS1A9qAnfNUGS6ed", + "SMm1M9VFbvQyJ5FdddyIpS5EKH2BZ8Jwy1mBud47YcdKk/0FMICT0mQIBuwE2VtBnvBOTrquWV+XXPK5", + "sEcgKqPSPr1+21WWO0O9n9nwu+/qZccyGBHDptqoQSrn0l0EPr1+a6NaPrtUOzSZnNdCo6F0GiU/j5xN", + "gn5CzaBnErCSoMQg3yPXO3xiAA8ccLBjSzd/UxGgiFJU8Y3IjXYTBLED0VhPguUAoc5ptvDukxt3oonG", + "Cqp4V2BhDlvPDoq4dG3EVQ5Csz4tKCNByF64CVxPWS2HKO17eSfSMzZBcee2biVw2dev/yYxaMPJ4/ZS", + "vn49kjMSyGD9gMsOmUzS/qY9CkEaTjagrSHPICjCbZc1+dVed6vbNpLND5b8blAWsz8Ppisg8f7++fOn", + "38fS7d6WqzfBZ2jNYPXJiAHapkRayT3YyKUVjWuEBxCqJEYkKruHjQQlsFrDeloQcDpCjgXf4gSBbYR0", + "SxGWkdumygp2AFrUoVN4uSLlTs8YXAZsn4nhfMgm/3S3L3Y8PHX/efplUisg58m1SLEEhjSpflL1jGWy", + "KDIxECqVXGGxT08ZzAM7+J/P7thSG+EP6kMs+ArQF6lJFMQDonAC1yFhJ9RsAjmyhwjWh77a8UfsxiFY", + "QLArX0c9KmTUO2P/POmz5312cnz8pc9G6OCCH46HT11n/+T+c/Ll2yRuWb3PjSbip3HVUhqSEkJJ2bs3", + "1FhKF9pMyfwIvXtz6EWXP32e3IgQrVihakUry1sNEVoNANScwn6jZQpXEXQ8rdYVoHEO280/zUeSbd4e", + "/NbDHve2weORmlloL7gj5x10N7uYNSUHzzKfrxauRYe0xqtW5f/DIP4MDodlbvQNyG1Qp1QCsWjSVkCG", + "3vQOIz1SLa63lcGWWc14sL5SWDeGrvIkAS9WIFjBcFXXhJHCcK+QKQSTQpw8VxidRnsPKAzAF7ZYoWEq", + "hCFCo+sxkTyzumr+DKKFfOP961xkv6+L0gpTsDQAgVEulxzivhOwVewJJ7iHpaD+XtVF/B2IdVZgtMfU", + "Q/YRrXUUv1oZCiHAASShLbih7QjqW5CVnlKDWGs8F4dbiS2rxO2/V3q5RM9glnArLMhOwNwfFEJxMC+Q", + "3eWMTUY9/PGMJ0txNuqB1PjZCjMIDOZ0+Rr1SivM2cnpU5/stbdXuAFslEoYg5g0duZzcrTXGtgfPYDu", + "s3UbD0/3PBXfYjFXIaijaVzLQf2vgv8LzahqvIbTq0ayxr2EEEmgLc1ESiIQNeQh6Rw8Ltj461eFc0gw", + "hEAVwjj7VUv1xLKJAR5lBCo/JEISCPLhblGN1EIYQlMmEDmLJL6pD+0RdwQpFuB6o8Oc0H/RAw5WFuxm", + "j2XsGuFWok9WZ5ZCAo8Kg9D7X1Jd7CCBmzlsGXj+6UPsNDDlJrzgmZ67o7xIhofBni6B2xi1C15gOVIV", + "mppAuudVMFTHTIAqTBYMlz377rsaAl7fg41DVC+8osEFi94kh67kD7EZHApoPPv3mypPn9WeYesvZ/3A", + "UlXXmtmA/VXLgL5Mo0JD+knbYm6cekTj53qy/qoDnfRvNJifHlVGa84K+E6Drg31d5gzGAyjlUzsqOdf", + "eNzE1952omVIVc7HoXnRW46mbL1MzIpx/O5DiccyHfX6ocQohfvwLX5mauz6Wnvq1YcP9WeonAYytM99", + "Tq3Cr//+1YrEiOIsn49Tq77VUvmscZejCjHhtx3eq5pOIeuniqcQIg0Zoj2mqwg8juYQb27TVXU+wH0y", + "44ANK5VHQh5GFoJCk1jgIHeqQ34A/t0UZDCYgtJXsRlhU9Ai4KEARorQGNyFEwCVoT0LfoPUvzHaHcWW", + "F5U6Q7eLgHj7xDJEjB8pj2dbr7zvAw/BUnI8PGVTseA3ElC1K5kTmdPdRrcBQAHCHH2VWFUAM4Z+iBuh", + "UIetuJVgJP1Q+dGpBkPakcLh0AahHxAOJqAArE8ZBkGvT1g0Xwh0bUQiJJqHP39iz05PSRsMdzEvXe1C", + "5nAtVTNt3AHiYaUhUrg01vUUJ/386vNISRvFyb5AlPdPFxXkQbIQybWNlEV3rolsBa89Us1LaRehB7Ta", + "RgqMJDjmqA7GsEL4S2HE2tNO9RhDMIdLfvcOLMS9s9Pn3wOarv/3yYabM4HfDyqwe/A8If/WNijUQGym", + "lWidEBxGP1Q1JEE3kC9qvUWrUa27mBI6zWgyq7ncg5lrzc9sDz2p5ou0u/t+a37vrbR7MehaX5XjHZN2", + "KeHvlPYekbaVB+fOin6gc1lnQveX0MhcUaGhOFVv0jj/J7j0yU98Ut17J8DZ6OEGLF7RPX0c4Bg1n9ud", + "Zmnwqght8FAOFSEKXK9GygMeDNkno9GP28lCkWd6hdcOunzVlz32qbUTL5xkMzfCwHv6kq+InkUW1gNL", + "VLQhCKIQxf4XtXsrsbQgZ8tIxUMC+PwercGLzSo3KrKVJWDJc7o6wQxUdiQQLHHfvU4KcmmNrM+NNlGL", + "BpiCIbuqOpyUxgCAz0gFegtPYA9sMWEeQXxEc+muZt5J4wUDV6dfATPGN2KkLF+KwbVUXvMEp2lXFIFT", + "VL/CYC6R05Kj53h9RWCVbglUs2E4GHIhR7LQ2iKMFqWdrryuTQ79rZftMbAaKb2USa92LaozHe2O4L9r", + "2M8rEL692Lt9G/dH5WwWuBo1XbyH7G1kvgwKegwtUuj8muXc8KVAnBI4ZYNXMlLFEuZThRrlI30JKWrd", + "FOP92F6MlDtMuhMMsQb06ePFkNHTPrvhgPM1DSeJ06kOilWOzETs5PhwOFIkJ2kZd1cDSz56Pjo5Pm56", + "6Vl8KSCrkYXbqx34/FWEG4KZupE/6R8fHzcW0Gk7c0rsfLtLJFdFfkS+xpuZoP20OznfcK0asup2CKMs", + "7ryC2ydwpTHPMvSqi89072k1ZOd5bvQdMDjH+hc5d6Z6pCJvKSIZJiunNEzfujb4bKAjA9hzVUJ4spdF", + "C5kxq7iMz/wFkVRU0E69j2QA6h2pW6lSfYt+giBcrW8VajURJ81A36K87CBjbZvNOOSmBY8EMTvQP4Ys", + "pefe0g2O/WQhJc8jQ3FTI0Wd43kuVLoG+YFGDsJOLaQYTI3g1/4xDGPw/VEzUjuEPW0JSHoBh52GWPz+", + "SLmWQ9UhLhYjpTxRiKdw6oyeHwYv1MBCA0Ey0XpkOyxHChkF7wOl1QBHKRy23B2SOYSkQiBFwjPuiUHQ", + "QEP9GCkiF0h1MsA+JfpGGMB6ufKdJVhgkD4oRK7F6lYbt3cUElgeoSg+ogXSZ+T+d+T+H71dnSg5cn8c", + "gVAp+DInPp9Mqmsn85LMvwtFTZ0E/R5+xAvvxHeAYmNfjFSVDpo5YXahjTv/nGriSiaLEM8g9sp7N1I5", + "sPB4tvpdpGwC19Vgwo98I7kdr3Q5dhVMwCG3X+kl/ZGaZnraZ4timfXpnYXcdNFaXj0JuJoDZ4Nvwgv4", + "a6TAEbQ+c34CAoY8nM9DmoWJN94N2WVYH2Apo/sbUHWAAlOdVqEKWgCwJ3NeLLwv6bPT05GK1YgZzyCk", + "2TPj2EK7y/Zfrz5+gNxOblQizr8SIrE2BeJbXPfTVc0XEVG5Pqz7mk7Ac2vS4W76T5Q7Hj8mcFVU7wE+", + "cK2ZBJEGvn3ZMabZ7QPw7WrFJOkKCw+OxOHFP90UKN6ICXfrMXWrJUSBT2pPd7kwgxyYiwq5hFlxcjDV", + "t8r20XJbUSPb/kg51SsAAVQGSO/ZDtxf7sj1weBOeHLLllK5c7R6aGBOOiwET9n/+3//P+R6nOjlUsDp", + "QxDKwSnJ8NlMJru6CVPcydk2ol9IdT+SlVcBs4bwj8NVHqLzZ56KC+WP644StxFB2EgRNW4wseF2DpBj", + "9FrjmQTRq8X1cbgsLS5lb2fw/C7oPeGTESEEAwuNGmxO7MocK12sJW+COVc+GvReNTVcoUoI0bn4bM+X", + "3rbh9E7uroxAWBGzrREDW+PJdVKzv/mx8Za26oxBMyMx7jtp86JusMT3UuI/86IKNPTKFlp/EFtjsHED", + "A2B20GoQE0hD87XnKuyd9Y7w4asXICl6PFmKHoRNuxGlINoCzSxeaLjLEFDa3BXAiAPOEqbKNYY41H9+", + "7YEZ1Mb1pcIpBSiSvn1ph/6owEZ+tVoNL/nt+wDLfDf4Xc4H4XqVDiEJuEvcB+WgUqubobSNzYJwPjP8", + "HzAGxho1AMu5ufFhvR5+rsYPupDuSEP/uRqWnGePiuHgwCwLip87hkGniXS84Uh1w8+hcvLC61TwnoI6", + "Cbhacan8P91BjZs4fPH/RkGIF9jwkf45ZDQe5HihvJOVEXnGARybzkI4s+0Lpsosc0WC66/XCKgOt1ln", + "UkmnqPtm4uJH2NbhSL0vixIuebTXbgQpRKjFT/wbYhXZESm/fX9JgG3VokSDTcUj7I0UquveC4yAnCD2", + "Fm37Pn7dU345HQpLDQ/Wj6HXvqjdokQ9gCEy3CbrsTCELrroUvn7jPt+4fM74dQGi75X0UeKiCrCw/3k", + "2fEPrJ0QY/LCySt4ww8iNFQZQQHXYY1jtQqg1YPnlCsTx30zuGG1eSnOdtPu9XGW+21fRLeP9y8ZxP73", + "9v3f2/eP2L47LXeK7I4ppSE0qIljRzjYEA2Lb3PBUSHhWXaU8UKoBJDyUqFns3YfR1CZBmxyPDyesL+w", + "GbeFsEUfILAAlsyVRSmeuxRTnnHw4KDWwbcTzA1hUVEuWgCtUw74xo2yGPadHUBlh6yEOLeRAjwuMuLV", + "+6oDcatbnIE43hPKZaudgqx2I7bc46WliVmwndM8MKCGDnbHssE+DKDSLdFv8ApcveOB3zT6nCZ6OYUQ", + "a9ipay+G9EwgEpkb7S4Rl1xds7eoaB9cXr49JG+ZTZ7DNWr5yo84IvcOlr740UOnIgPmZjwnvLdhePlg", + "GC7QFiVwwt7Ll+jAN1mPkZggDnYUEVCFq62Rx85kInkGCyjL5BzeipE3ukESDxZHTy+4h7fzyfGzPz//", + "0/d70uQSvtpl7Fw4z/SUZwP/XEZSptajW3ktc5FK3krTiwRD46XdEEVCRE3hPp8KnmYSXtXYUmaZtCLR", + "KrXea8LcODFN1li4aXLL+EglmnyW3DXJl8ETo633AfM3yr7f26HSfj3YXZhSWTJVQnpCUkBX01zbYkBR", + "AGCwucChDJWKuxzsonTLCtvEtRGNmxYf5OAZ+/zTRdCSnh8/w9e9SqKFzdcktPK9wdcGEkyGXRx9hEsj", + "Z0+PBzh0Qfj5FjaW5fPj4y1U2BsQ1nBQ2phDg53Aj1zzFRF7VCebaYIrRsXvDqRJnMpt9IGd3SAS5goU", + "Z8daXnJ4ZvvaiipIdvudX+qpTNvW1BZAEAhP1zOnlYfBbgzqsNFHaG6LBy7YVT2L6yoqD9VBXkDgcY0D", + "wdOttczbo4OE+PnchBGCR9e1WFX4yjXMj3iLo/9B0NM91PJIvfLxZSGUw6OMuEsAoFVYYo+DiAe1Bgzi", + "pm24HWPhXkNC+TvHA74HbmcLIY/s06tzVGsHVx/ewLM6OWoF94FW/uUOBjigSmPEd8XkLNICZmCabYVL", + "bjjAhF1xn1Goeca0D8Rl7LVAJBvzGMjTCJ62d9vD9u6ETduwXu8hPAJ45Zf+rliogh34oAQ07dexTd15", + "Sev4ENff7spkYEZrYNS584noyoHfT8fTHcmYHRjUOrWRXxAIG/WQipicJQDRY/SyVbPQ+roFRta7jtfa", + "2dQjGq3tAsetrIlOjRn6steA5KAp25lDaxwV3doQYjNDcIIKNDGetXImnFIPag8vmDexoKFBEV4xPgVt", + "RLm9rwdarQvdh+kV8Ti38f8Rw3NF0BmAqSr4luA2CpZ/KwF8h73GXUx4uu5uAXrEIF8YDnB3OVfwOswA", + "7XpGhNJyvii85XVGKDKVcj5gqYCeW3HGXhrBr+n9+c4nQid5W04Hgfu8H7wG8JrP3c0pVEqt4qao2NKh", + "JluIfDzlyfUZ8zz1bGo0d0qb+3lu4BmcRKk07o5cuH0eYklIRQ9VVWzsSog0ECzf4fPGYpWKqKrFKtfF", + "QhR0qFvAHaHXhL4PcF4QqkWYgqg2PrUQRnuUaJWI3I9k6GEEY41zBg5PNLo9pMyG/vf6Pde2btT1K/jn", + "3/3TVIOJWevtpOcvIVH8vrUZDHILkfRnscy14UZmq58V94HX9yMNhQUPAXs404GJFlx93bk0oBokYE14", + "cluYG2mZ5TMKgCvMqpu6tpV4k66YA1cL/ACFsCTj1gZ4ghiPPNXJ2Eenjcuq68jJnI69C8g4N5JiyWzB", + "VTpdjeF7PUsq3fBPy4qstP7dFtrwucCsRTXmjWRIxWQEEKfjXbhJk9udObDidiXZwnnaOCDLJVfVcILK", + "xCiubjtjaWNp4Pu+O8tf0MTEHuh4EXpi2aX7NADcVZaKjK+iCXOZW7hP2/ly9+LJ/VnRjbe28nfjj35z", + "x5PiSptiN9Lo7dzt20oIr3S+kHdyKfcso7qTtXR9py7E1MLbMvxcsd7Wie2kmu9dAtS+R0ObNNyA+njp", + "h+1VcDbcQ859LAv06gp2CTCmhSumO8AlOuJWoVqAezFdxUZ5oVLQVvuVttlHQ16fJUakBFyJ71B9Hy1L", + "asRcanWU6cRDNHrydkTjzmQiiemLqyqSReei/gzBPi+EFb793kkxeP4FG5MRYK6DiyL6eON7LfYf4nIQ", + "+O63Uhe8jYlgWhpbN8uftKiqwZT87PSHZz98/6fTH55HxpuTVq9Rfjd2pza8PSWrbmdgqQazzKlMaJri", + "BTA72D5bQKwRORMESxiZs1quAPdtKcJFbX+FB8XjEyYm6SZsYce5MOOlVCXi1jxGiwDZq1nw2kuJFeYG", + "TY/gZYGuvWfoXBgPJaNnO8SKt8JIdHrxUDVT/f9x9+9LbhtZvij8KhkVjtBlSKok2z3TpXB8u1yS7ZqW", + "LI1KtveeTX+oJJAk0wVmopFAVbE7FHEe4jzheZITudbKC8AECZbkmR3nn26rCCTysnLlynX5/YotoCUQ", + "SGJVtqbv0Y5oJwjZKOfVbK4+COqFY23F1KIlZLXO2LlnxgBi+hiZDrsLJU2x2GNyt23F7+KFxAQ2xDsj", + "iYYcT4KM8bm4CexDX6ANe6lHDemXaM9q7Mv6OHTH9eU8uwevLxd1rmh7zTboVGo4+o4MM1vVWC2BQQcq", + "kHLJexAA0ZsKLu7kfKLW1EoquJCNOyypZuj3oxJaan6X2UeswH4QK3FffRHr2fPFJ+wY+5HDhjU9lzIr", + "PoiKy/rSmFa8kaaJuDp65EWUlu2kt9FIv9SQBWtbYdI2k3BEYhh3kBmMoryeAGmxZZxVtcSUa8wiTJlx", + "AyWnH4R9BPIgfCZ9qA9SfJNu7Uaq4qDrj1IscNb+Zt/o1nbQifHt6WRAt9NMUcl4VNvRZwsC943XlN92", + "YgNpFWn1SXNYadvvf8Rnk94DfOJDq46RhboNEAA0xIqblChgPDux/11mvxKBGVxjdi+WalLhKi4jfsRH", + "ygzD4X+HvyI+VSHuX0aZw72YqAZbxa76JIDyx9euirdGZD7KCRa6aTfdP+Vc5aLMKKEkowMmeX/5nF1A", + "QrNnM8B4OkKYNAl/0ABcrRCpoZj62Sx5DhB38cEWioiVdp0oxAJuwL6mjGSZbDScV/tAIU2F/I5MQ3QV", + "DTQscpNgMLpaDyQ19tlGL1nJsa5D3YoaY7NWmgTheOmFS7rhcXcJpMQHJXVHJqJJizJ1D6sQGLQzwf4b", + "VMnz00FdEiV89ZUKdRgrxuJpmDCutsSCc+uqPypRg1ntxaK7yfarpuenh3UTNpTkofQ+VyxNicVvKZzr", + "lbshkmawdhRmmPnyFWcyxfgfHK8wasU4U+JO1NGIKjtFte3B//9/P5/+9ff/fTr96+9Pv0q6ub+oZv3o", + "W+vLHIzQtAuzNY3YIFSTqQSy1dVtxycUMSgSHUei4x+Ez134oMlo7qrjG7HdhzFJWTeYqYFZhpidcx2/", + "eB2ImgGcBWLxLnSODOedl2MUxtZAjlkPbA7zmjNZfPp09s9/tgaANdJYmVplmJWcoA90Xu0YR8V29dXr", + "N68/vgb11hCA10DfraqfItWqD1S6Rx8Zdq1V1lYFzEHFm3Uv+euQvLjF8R19V6UqM/xHjhji5c9Xrz98", + "fPbL+1fnR4zUAZwZdv2VEQ0urLgFuC1dthvFHltF7K6gG12IJ3/KkHHLZSNSZ+jS4DNo7mrZiIBpX+s7", + "qxAJ/hP9r61p5sphuoF10ZfAqhYb2W4yK3kmJXV3a1GL8dgF/WBVPLoBq9xN1JWngu0F96EyJVtL1Rwx", + "7djaObz7k3310+SkMAnazFcRIBRgOj3OtVKEgUZA0Z66gyB9rv7jjYdhmkUkFdcei8ZqDdvap+uAEUVx", + "NCqWCNkAHpXJvok1WnNlxVHdylorsFWgumNRChMZgtd3vMxA0XxX6pW9KF67BAEcT3+x+0g5yRN9R0/S", + "8Ux0571dGePw5lohipRLi6OBdcidYdTvf6Rd5lG1+hmVmMkcUmkpNrXkZWl8Odu1LK6ZXKLirfUdAPkq", + "TWmy+IXZXJ0T3Ccl9D0Ws9VsQkBFT1zuJm/ohUcOOd9V+xEOz/U//4kPfPp0jZfze88Pg/zLN2JrAAN1", + "QKlf99cDJvSz9TwnGAatuuq+o/ZYrN/n6vqrVoHa0xVp+0jBP/JKETQ9ewwxoBBsROF64ublK+xt5pbZ", + "NgoI+fBnTHtGhMKePOwWB+rKSqkRDZota6jzseZbKPxB/8rv/+1HTzTrqRMohkadK68CIsl3jAF9oUos", + "mz+s7MIdd1odmF2x4bKMqqpIWuHPnz6BOz/xFhZjxm/BXzovIPlr/IwrZRxut7/WgKTz5Ze6iwk2Jl+V", + "Kj0T2h/9GkQ6Ajupv8kHT1YsZ4QOZ67ybTxEUPy2PWh2hxH1NnoY73JDqqFAzyLW46Kp5BOfZwQrv41A", + "yzEFuqrFlN6JP9XFfo2deXCAZVCwkOVtk+aYgXPB1eMWxOqCgJt4Wt9xSN6BIBu6OrAEglqEbYDZqlBQ", + "gVUjqO04HFbSANC5Ubwya3CdYJliUwu+8YWKu3dpsCpN0jHvGQfwGazE42qKHENeQmcMyMLgISasuMPo", + "ZGO3OJhc11StwfiK24sebfeLVxdw0nFw5ig0/yZoCxrCNJirkGt+HVtg1zPEFq1K4fqXc2oj1ELV+i7w", + "i9FjWItAlGIEv+IuSOHgeHbtzy7iAZMrBRXn/8//9X9jcAJHjCmSGJTQdyqii9vVWl0Decdg9fapK/ps", + "JCRUYQGpe9wapjtNLWshxrZjn8XC0WOVEd5HU9UxpW4Ob91IaJh949D+teKNBlHnvmB3wfsfI+hSM7id", + "5yrez/abxtlMWrGrtkLFp2v2s9DqycAmDzl+RxnsV/ian66ds3hbed3cmRo0eNmFd9aBAy3GjnSYVRAd", + "6iYQuYfsFSW6G7m/HuJ2hV/xerFzuoy69ETXlF13Md5E7PUn8j7sDN1BIEJczOE3R4X+gLOF2ZwwM1Kx", + "H19/JIFABIGQOJ+KFuM1LOXNCWAC+EywsFE7dxX9/OTJQKoKN6n2ezkv+Ji/kPkR+lyiRMO7ncjQ4tiZ", + "6/eXCKhhFWktVyuBHjnbQPdwmTC5dBUzizLlHMUhFe9UuXXABocF4WogM/YD4iS65Fi9HBIDwERIr/Jc", + "9Zd5AhFTqejAf4ZqvFff14tqrEV+A8HOZJzOnqqQRXwrMsoki3HGIkepW4gNJZDtUsMQ3nGw0nYjAfiF", + "DBLKkk+UfJVhMm66F/Z3ciYPPWCaDDNfMzL5M+4qjAYe95nsie6YhmBwMyu3sjnYmGkBfXrfcwAqkvwe", + "8YAU2YFlSxmTuxo9Pq92fyUjKgu4X7s9paG7qEHi0LCW16HeopjuWYcjt158XdjxPulq1IUwRivTlTmj", + "G/QE//+dulRG1PhvvHnb/5Iqh/+vWrOm/yhL+A9eFB/1FT22kQr/n99fQ96F77nHfbffgxvXNXu8LHnT", + "CAWJAt9jPnGw1hDR6MnE3iR37u2P6cq+c12fsNiys5rlyY5vCW9yu6KVVrQ9zxCCt9hneyWBYRrdtIkm", + "75YEujvszqfpFtn/NlzM/QESreAFVpRyVspG1Lx0PHETqmP3CFlPwD3Xu72HO36jGQ8XfaKHYsGHNAOy", + "sWvW6LlS/FaukCsKXCa5LijH4/vIRbZjceihHIc4UWln5JBFk2FVkUsw48VGUvKJ7cbdWlprsSz1HdQb", + "3Yq6JCQrFxqc+bpCOHZcTpEJsUNrbN5xF2f78P4tli/f8pJSTO1VYK5cjtJSKmnW9jgKuASoCey9Dm82", + "UFBktx1A9bmjJbBWkN8fElRFwWIXrmjaCm99tts1VlEy0qvLtvRJRI49cMJ4Q0lQyCjiE9BcBTqkSWF4", + "lMZVidq3E7loE4liRB9dm+a75zP2M3LmwF0VfUWrltdcNUKwf4has7bCqWDfvPgrcuqHTEBkfnIEUb2k", + "+HixTyYnYckGAmiHU1gPpWR2Fac/BV2HfCq0BzzMQAYzQV9J50K76OtQDqFvjTn0RIBHCnlRHvXXZZUl", + "64cOZDSOzMimXOzAPtf79CMT9TcXgCOWNFrdC2MBgt675/tJ3zT7sES/p5wZ4aJGj37z4sXvIwqsehqJ", + "mnF1fHG6tx9LWM1DOeA93KNOrVvPIPXCNMAOvQs8i2Xm5dahO0wJeiDgx/tGEZxUBLhITwPBLhWLCgw6", + "oM3g6UBEtwjwk2CIwXDfcLWNvjIhdePRPSkTEJNIsL8k3oEx3idJSgNUS5R+4pFoMYeDaPpdXqfPdaYS", + "Lip+QzBbcCrBPdJjOztA7HI78RR1HosaoiaALAP5LbzewiHqpk63DZJEgX+BNqfPE8XknwlbtIBYY2aO", + "rBtx/5FkF1ZygUEEoFIuiWbbrRT6rmDpMfJJmwp7ZKInyQs3YRux0fV2EqWKo1afOA5BpFmxd0F7AGCz", + "Yd5oINFkwLJCcgiLS+MJfhvqkaxRAdTL1L2X7FdRN+I+Si5CtRXk5MXp6SzAxiIWtSMqQMw++Eqg/sGO", + "eRiAnZHPjsplOYpuidI2E0Xenn+HSBGpRNbTKmH1l8OwTWlCyL7fk03jlxEeJPdYN6Y4pIpdNyK8Fpeo", + "8cV1sj/uDjv5u4UQRyzEcH5LijoVIhd2TwUzHHkzDy5Jo6uMPDWOijON3RioOpGCr5FUZeZQbuBjBBeA", + "thPmauMCRh7nAPyNegu0DpKwLZHMyxWtAiI7aj0ZwnT4lAMgdMCRqS9fYBolGiWsd7ygdlzoZh27xJmD", + "eBlz3I/c+n/OZu0d2l6iE2fw2GorVFE7QIX7X7rQa1GLI1/Ceeu/9PtuSfs5axWGP3YZvngkKrFBWEjb", + "wgaPXyvbBOALtgcM8agJsAY3RhSOmIDJyS0M8bgJ8Ib39mdwzoRV/dRB2fRou8hce+JVK1nm09tvZqfT", + "qtaxyXbmxvHpmNqHoADTQLsf/BIE0IGAfbubS+3PCyolF8U+XPlwuoCBcidqb7fFij0y7wuSkCRy0Edw", + "vFZ2nwfReUCR//BZ9sHBp3qMXzzLPJ2IP+PEvbUmIjz1+NB7RzwUIOYucstKgL3yXGlf+GQ7QqGc7Ly9", + "O9Xewo63J5FOdZJCcUf6XeZ3Tvpui37Hj3sjSjU9dYbe8gmAuUwCaTLkzpMp/vjR00dP0KUTSJUhAu1a", + "YY+ePrL9Xtm7fMSx7RRTeA0jrPHgGqodth2ArFd69GRy8rQboXJPpobc6Fr8u14k8unQ7UI5yiOkdsHz", + "m7ai53dd/lgXQJ4OET8U3TIdElWB7vyhy1rYxIEXBUkACQE0ZO/7+kffNMB6WtMX/E/weOSoCk5lyNlv", + "axe3GS4JS5IoBLBy0hYjZhC+BpZPRkg+Y6ehM/goYzzO6JCG3UojrTqwlymaKG6Niykmsel6y0If7HXJ", + "WuauWEQtZb1B1XjkXAxHOgiTLMxSd4C/KHnPRKXzdUeNkpmG1PtIfCfqDRh8f+gFpf1jzYu9DRViU2n0", + "yN2ILVy3AB10o29FEfQh+jStHrWNIOy2a3ek4v5DLwZy+iGVn6TJCeAUvgPFwksp6hm7QDRyvDRCgSMS", + "DzXIjYGuiVmXRu35X/ssaslE/n+eTp7/26dkNr+PCznF8vdWtKI4mZzULUC5ASlvTs43DKfBf+CWLgc8", + "cn5nHiXDfnNKsD7cDtyGjT5j7/t7HngICl3BLQ7dBv7xCQmL7bW9TjxAfOsAwza+1NzVhHnJxFZm7Jxh", + "SK/xNFbElBI2noPx2gAU7vAenqt92/eOY5Vs2LnsomwNcRrbSYbIsK4bDxQmXDjbzpvv5mSuQrNT6twE", + "yHgqmGBVEMgWrSEstUFvUiCwMYj9q5cePCQwEly7GC39KWvqVsGN8LqT1cUbFtbA+/fB0cFrAp0rtVr1", + "PoHGXWCPOVcOFsy+ujs4lxjPbwiw1e5UJGW4xveuX/palBgdakk9kehxWNUA42onCMEPlbjb0UeelHKL", + "hTkBo9qhvyWKBmlp+nvrSMkOI08kF8WgXjgfe04Ta8BTmc6oE8Sn1OCEJzXI4ePwyOHi0n2BRiJJ7UBA", + "PiTu0afETFRb2e66iPoRj4dNlDK2dkow/FeCcz7ZVspm3/D7S5yEf9vNYBvc26mQeUspSr1tHpT97l6P", + "iLZ1TWg3nQ2fzs5EHTgs+OgGQDbEafdwIl0eG+NOb2IMDT+clGpSm58phUPYeOdBk7uDh5KBcDh59wSI", + "h+AN5JPU1oP0a1BswQRIDc9PxGcNMFUtCCAosZ3irjU0pnR3Opsn4aJsXOrQkIHywd1Y4sMNZfRG6Tvl", + "HOneRfEAC2MnVeWgrdnbvWR4TuJbm5ux+GbmrL3Efeygvh0y6YZubf07UH+Y6bu/u5C+kaY5UqX+oRfj", + "EXmjq29Hf0F5cV+DKXHfZAfK1ZW4E6aZYkUShafwqKRKenaO7mhCJLfLtCUcW1mWjkwq5rKwarBt1rqW", + "/yBgldoTXwP3ADzu7SvbMEZ34kuNcPRThBkCj0nbwJrbfVP0LhQvTkdfKP6aulDsSubepY5AFY5Y6o6v", + "ocfO5S9ULrwefAT4GoY0gv7zfhL8efri9MW309Pn0+ffTm9f9G5bL/5t3+ycT/+TT/9hpyf85yybDpRR", + "h+yTxCheEYJyANi59qmOUl/HqStIPuFGVAteXM/m6h1yijmgKc8gTnDBFIMUS3mP8FLmZWzE/fLhEmmc", + "jC9EXAglIOkVcX/c12utyUaN5umb078eYPf+NDlxXUsBAwLknu+8Y7lwFYO4UB3o9UI0EPuEwgaipbKm", + "gmM6RI0aqimDUGy4kktB/PaujLyft2a+Pnv2bLPFuXtGEPfYDfMMigPQRHgWhOfoGentnFhr+6nqSE16", + "W1E893wl1DB8Df0QJUi7MDC3782YY57FGgzna6H7tvLsBMC44Ih3IaEBXBVxxAzl04UBgDLdId134i5Q", + "TfL06XtZCcCr3+hCPH3KHlPMdcI2/D5zBBcZ5N6BYH93+uTMd5fXgRFxrjwTo2Pr4Yq9efOWNVqX05xY", + "D0utK/o2zJnM/acHvsjm7enp14LZD1sRtE0WIgc6DpooDRQleq5yoFUJHBAOwBXSFtBGE8zjPCZm2J4b", + "hNO/4VvP94K89GSTED4c0F2UpXA1tlyqxpwxQKTMHK5nCeah6VJlTOEZKCNytTuqS0dAZTgBYKoTFJ2r", + "DyAnGE9x7HBQJxQw25dtWU5fXb1xjegavTVEdglID41sWi8M563tRyMxNFzrO3/6wTKjPoPaSqx6cjAq", + "diJcMWQxVz53k0kKytr17eYR5kgy5KxnIgMCeOJnVmLnyqOxgNrZRjg6BBYDvoQ7wW8EodVg0MWKAuP1", + "CmNN6ZKJvN20JZhGNMBUfpKCKdv6OQBVhng4IKNYVWEcMqCdIgSMQbKQOVVdw1ni0Q2F860EMmFIYsIJ", + "DUVB8O+5IpGAMUl1S1opnkk/72ypy1LfTdvKTNIT2aG3oBm3pwmuxhEwCghpeVWJPFVABSKdWVu9FEUq", + "Oe+V3nAZkqSZfxQRR9DOkg5SX6jmkWF0TFa13lQNljgv29LRZN7KGPeY8QXy3geiRJbrsvTEyJ2TBjJV", + "w89uwxvINHLvh0A1lHDlG+HYOZlpZZNG4cnXXCYO24s16JOlWzoNhW6jJh5efSPVTWrerUY0jmphB3Hb", + "SSJiPptAYRRxJNizxN31kalzmvO6poi9N7BnY7sL6v0VdSvZZZreDDNeUrHHVIoMpsLYN+yqh7gyyUwc", + "j3Hy4mjgAVbg47t3P3sDxa4pwBiK5VLmUqh8C+qVUoVNIyozi5pstNVBRjS7qDFC5boQHzXBnMwwqz8N", + "G+MX/yD+unswJBZ49XPbxYJI5fp48ly/myAgQ9TEeclrD/kcIYkj5hZEINLeHMISBU4nxMYcTvHE32Gm", + "o/2oiIgUlVuocWOv3HrOFWb74FHuACMpw6dVqDiXDkIyjXeasCa6jGeTg3mpcKxMIwmoMYfRA756S86q", + "dLWdK3foF73D3hoL1r4oS/xPhMGNqUc8Nw86sSniQ4UQH+1E4YsYgXHXt9KejsBSzhv24hTSuHhZEuXa", + "6Rnr2HhY9YtGRlADzk5yJpw1FcDQsuaabeb5v5yx2F6DZoYtMYbMbHG+1IvTQ34Zu2KAdNCRSzMme7gr", + "ycij1Bd5MsHdlcR+qZfQdbCDmD29j7oBbQDDwz99+l7M/AiYi8wpR8+xCF1GH0SDl6qOawNJyYCJkrwQ", + "OL6OQXLEadI4JNGEciaJ2B3qfwSTmsSI6ubR5MUjI6oiR5p/yfGu2Lmq0F3R6kmQ1kvFqo6sPh6+hIBK", + "I8PLk7JH6DCXCieH5NU9XhA5fnTDis1/3FrufgCd2qVtZjuszfDaxtuMFL/x5h7yzXlVMcVVCy5M9j5Y", + "htRBBGBU28DYN1fu/kG25h8aPEyPdc1aJbXyjJPUN/OkZ2bOVddwh6YnkQ1Z+1sFJGIvRMdmnMxVMCxj", + "PlhvRVJw0lmiJPWcFhrqgjqmfGy+h15EXGA7uAO0TI6WILAX2ds5ZfLafbzDjHjyk75jhWaXwcfD3tkb", + "z//vxJPS2CbNyRFwArSesUwPbaVEyO8XA7moKsnR2LEt9vZ9CMjDGYRDrNQIxcmRHmsJNVbMveNJB1xV", + "RKs8m7jbjoSJOTtOxxP0dcdqoMP4m9NdrMmPaDygCxFQN4HTJb4OTByRSTijUQ6xEMvx7AMi2ALS3QmQ", + "ge1aMUN+AqfGESM7GJTwgu03k8tUc9IkTMVvDjLtWfmF8r+ky/VC17Uo6UQI7leMOA0eDAHLW+RrTd5Z", + "V0CD5j7MWqEFJjLITQX4Z/WtqKdQMkKdYpWoEaUlT998wGgevXvAqriCV3yx8UG79rWCdbm6eh2AYdit", + "QRxuZ00mTVewUMacjdbeMs7qTlLFhENyjGvQZbUk+fsiErqOZ3DXe0Hl7oOeix/IXxGybN0BULR1p0zn", + "y9z3O5bX4bQjuGimrDUHiJhrZVrEgCVcIZS6dI5wl7Xm4LJ2nv7YYZ0KLHJuD9eiFLfWBE+tGv2EOapY", + "TVbEOc4xMe7junudBC1e2O3z5OTBNMGdpMeBXL5GboRp+KZijymX70kcoqL62TtuHLjUuLw7KzxZSNPa", + "y/pyy8sws2gZtFXmr5oJZ0W7WmEAONgR4Wban0rX4pNYmHfjzj2ZDSfF7vd/jKwhmiCp2IbXN4U1Z3F6", + "AqAHFub1vARzBXPqg0kDaDzhhSyIRBLAin7ruTjiPv5ZMuaYEPvy70QeJRyemhxBzvqTTFpKqePuFyX/", + "3kbievkKjrpSr1aussie6pg0EWymmq/qLK+/lqsXp+tvm4X516/F13xdr9JQ/i6oHica7RvFpX/Dc+HB", + "+IfEiiCQgvXkn6UUB48i5APM+/Tew2+iwXfcRA6FiNrYgHnzJ1xMx3uYD1xNBwo23rx5S0GA1jju6rDR", + "Y9lYCSvz0xez0+my5GadEok9KupiwG2GdSCF26WOSBdnxF36iC/3OC/qf9A3UrOBKyTVatjb1d269HgQ", + "RURvkmr1DCJ1LSRBd+CcDx/EoRejPDihF2lboNR3tISHP/1AW/mAi/uB2GlkzTq8NG8K9w65pGj4qiur", + "z0SMK+ZsQkJhHy841q5OwtwRCaddrWK0qe5JPcmOzkB7ZBueohv8CJXjwUUXqRv7QsocTaRqOYCH8VeJ", + "X1Bt9Cx2OqH2ELSmLiTps7hTupgw2h+Z4MYCAcBDyu2thVjzW6nr2VyB38w+AmlKa34rvN/MRwjAwZPv", + "Ro2eUDaTqEIsDcP+VtVeLqEcTVSOvSlYIxMmkUQSUyOt6eC7j/SDdavMk2Qm9GeY23ZGQxCja/Dsr0p1", + "T3ZbAJkZYX6GN5yxePDCQ891v9c1GkfEbHbGHER9/IYL7386KLO/pNFQwIfCYCe5gx6r4UJx5+D9Myil", + "XVng+VoUGRz9g0GgSzAMyBsCvgTyvcDbyS2/v0HUKjJuliIxvCwhIAEqJtlyWW5QY+27oNomQqwddVWy", + "NaRdO9BR4mY7tqcQ4Mrs+hxU8+/to1fwJLrXYG1N5m+lQ13zj7IlkDRGnaO8A+fdSDPiQSrvwOCv2o2d", + "S1ymf+nOwsm4/OO0d/U4fvXgk+2XWza1EN43vL+lj7UQV/Dk8E5MFLuHRBxPQN7dX/6IgIQtrpI5W3PV", + "TdqKIy0OT7gXa5mxCODBipAAOC3k/sZWhekhP0hDFWAuoBiR4bK1qEXAHoaMHhIQjHCzxz1v+wRyjzJw", + "ouBfntA1zU/6XPE6hB8dGAg06mjT7ANgyHN7p+V1w5QuCLwZZohQ2iipIxaZSG0efYLbM3PGfFMU0aSQ", + "C2Z5QcAC4XwDhl9gI7PPA7PoNg1MkTpYH+yVHN43IxnYfUy+x7hOS3rGfoVaN2bkRlpTvdm6hcIsuxCC", + "sa8tNi++PWM/tGU5hdtm59Hv37741qF92Icd9vwZizJUULa16sEQQut22c/YJd5NbkVfDHAnIdoX5XDb", + "t0Dyz9gHugeYtaymC24ly8ewkCp9UcvijF0gTxvSKlQlII/b6bIbDT7w4cMPzKOxdCnPadJOJid2IgBv", + "C4dolV4t7EECvQH2c/u5gdL4pt4Oye8HZOvekeJgL7pjZTeHWy+XGQ1LupAQOdlf7GRC+OcoqojSCLxp", + "2NRev1KaUjX4lSgGmflumT7t2E4s6NLRtuErCCvYx3zogMadDqYZOFTEzke/PpwNQpTc7u2h7IFBAt6B", + "4X59ejrMs3ZovKNOU12Kc2PkCohrduEHap1EBNElpHUjQZ5pMce90YxDUxPWmhaOC8xsd4D/trEz/95Z", + "N+sffiQullpwRKA7wB+qhwDpdNsMAEh07x526VCx2DbOvM+GPfZXpyXPm9ad1k9s70l3PRb3ValrjOzj", + "r9GWdy3ZuyWaE8kdre+ICFw1SfbZKN3WnR0Yz4N4iVb2Mud62jiMhO4SBraKdGXHPz9N0nivODEQOYOg", + "UYhiQGU8ERFA3hvVoHZW9J+Exg/AU7xuUMBOkPJW+AXdWbwBno+PobzQXnwfPX2EM1GWlK39pCtPui6S", + "VB6pgkursXCWktKEcN6oeIcwv3/Wauo5oXylH7hHKcyGqnn2h9HKk1jM2E+66RT0OnSpkOFdi40GRCsw", + "xKaQcY7FyeYlGkBt1fmr880iE6HtPBj19sld6RgR8qB9AnBebZfeGPJIGr0h3uNQzdG1TvtMjQPxozU3", + "CfThN/pO1Dk3gl39dD598e1fXGcQ4h27lJrkxbaxU2SNYCwHe/4XO0UQ6Qe8P0ekgYklyIm1lKup7QeD", + "ZJRhrx/Afdei1LzIqA4zxYkCqQb2GyVvrBVIHKFFh9qUKmttW1jST+aCm2jIXADLwgPVouG7EDlvjehK", + "ARnfVL+eyNOcnNxP9UY2/xC1DjBQ0VgcDvyYNaPXCKB3/HsgnMNT1mEKD2UbdtanUO85XWldBDIYzEvh", + "biqxVyPH/ml4z78SeZnA5HEklwOg+ku5QsLtDiPbPz8NASZ2n3OFUJldzehE6f8dq58yXslAATc5udP1", + "zbLUd/Y/udnY40cWIuf2NMV6rORB5Epz9zrN7huhjNQq1PbtqFNoJ6VEr65ee1ziPiudh+YIGJgghDN2", + "wSue26sFIUbnvK63VPqGUTZ26QCX3LPwGR9PiYsiKDg2ocDGhHlYW/J/NfUWs6KyjXmJljNYAIRvAYU/", + "0APYaFDivqtRRXqY2C3qApgRDt0iX0slAipxrot0+sxYWGMsvY8eOcDg4QTMQ1dlOU1ljGeVeezxpAB1", + "Jy+RmosmMBnKSZN1dnKYrzfCSh7WtQ51RGJxgK55vfVAvO4+7vGHrNAl09l7oo3rOiTbtyKV33J1RRRz", + "YF8aJqgemKor+k7VkLIkVIGUCtElshFVBpoe8jHgnw7cxP07Bg/o+eTdimM8vBMA9f5vCOufUAwHNBO5", + "0wMWRaFVWoNcXb1G94rrQcKYxbjDvihoN9Q1mtbmN4CkoTpcbljDb0QytSKK4qet4VRS2XkAwaD3o1oi", + "O6aThFDsRSm8irLnojHLjRhxj6PFyAb1wWtSANhVykQBm1QaNse35yfJfIfhPAtYO8qxyKPoaTfP4mGc", + "4y6w5ujG0zxbV1RlaTsC9wAfzu/0B93VcG2xfYNUwLZe8lyYzw7fisqFcFPn3sTJ54CSsO+/dzv2KAl8", + "TXoD+Qt13XhYI7hZS3sPJJgwclR+XAt2bXt1jQbhXPkwt6vJgIlskJGBGluIUqsVMhNHUh824VzdWmW6", + "2NLb20o4AqnIi+vK9QwrRNWsgZIjU1jtaVoqqGqQHSQZT0wLQEz1uLfznYtg7NI/dBvcZ77Q6tdJ3fY6", + "Bnrg3TAvW4iVhDSQoZDZkFrrHezRj24aSOctBMJlodYLY8f4hP3NbhPIDe+5CCU4VKOEeax4+BLKQa3I", + "8do/maJ6fvhpZDLW/+GqpCdKABgwSi181Lp8q1OpEq/78CFkJgA8rGG8W0FPB3ZXtDY6nYNBORfuPY+G", + "HRkcdF2ApBDywmcYsUtj/GhdmhH4ghg38VGkx1UE9ESIEUCJexhuB8aWnNScq7+JrTmIu9BoBGK5EVuo", + "E3WVNF6p3ogtq+1FGlMnlMZ/QZqEo+2cYKU0uKJcQ56NNqXdqBIHqh06d78khKEv1XRYM/ANzyT/qNGP", + "4jA9BMlCJcIZg0bZY1DJ9qOsrSpBCXlPhkhT0YmQcMZIJNkhJ0O3Mhs5QwlgEWariWZpruDWhCWmWIV3", + "I7aeistAeOlKAjohNn/G5ieNbEoxP5mw+QnmGc5P7HM/YyowQKPYx6BojxeFPQhm9uqCj/0myyLnNTaV", + "5etW3ZjZU/ztta+Hsj9O/a+zoAzxQWDe5GXUqUhdYtcyCi4J929szL7fKVTC4ViNAGOJAkGzhmOx0vhs", + "5bj+7BgyYissSZWIJd60MCToboc89kJrD3+SWCs+nodr2aL8M/BPo+c/IQborDIR9Skcj2C6xDTOKWbm", + "s+enpwNpurShwthGbSknvL0tZVt5FCLzQ1sqTAnShcCWIvDJlVRwiAzuME8vcTi0ZFxFEe6d1KS6XQXV", + "tX4gsTZCLcZihATfSXBH2In2NMp+3p+fHoxfOSXWrcMp9shS0EWfK0ugeBuNgiTCR4dF6EVKhFIuwCvw", + "5vvATC/JqVNi4R2eBW/E1F7jUlIq1G12y+uBZFRTaQQ9FepW1hriceyW1zKKe6Ch/e7965/PL7Pz95fZ", + "317/rydJU0mkwvsYnoib0pVQXM54JbMbsX3y8PsRth3yWwPG3Nj56Z3tdgR70zLxiw6frrs6GIgZj0IX", + "L/WOou1TZ1Hbw30aChEhaWxgG4PV6FjjHRhIl5aZASvDJP6LULfdPyx0kw4wuh7pWhwfuXJkBDBggCVj", + "AG5GtFNAqReZcp8bXAoUKyGOhB+fIt6nj5qNjCf9CTGaRrMjIgv/9VEVpGzdN++IkzhhKix7BGKBhk3w", + "UOZaNbUuWVVy5cuMIi4cCrvFMjJjl5BGrKwOXyJWIqXIEPYXzz22D8xt9H17zYJqdIYU6ZSBSh0Fck7w", + "E9vG0RHcYYztqK19bmIKcUtMPkXE553gEg1qOMi0rIVZj5YFojM32YhVCmJpuwjSb09OrCSBnYfRT42o", + "l9hijEbSm2Oq8sXIsAHrx/kqEFoXxNregSUhFBDUdwc3ByxgLKKWhqwgqLniW1FDViHAmSOnAW0iP3tr", + "bkLuVBzypnTTz4jWoZb7rZZNDGTZ1UUD/LJXkWSxxzAAX6gToT5WtW6QLcqrSsddgeLqKPKeHDzZsCPp", + "4wPjvmCppz0C9q9Y6u5ixJilERYJrlG1ACbWMxbIj122KcBzVByKsQjyzTzLtcK0A101ciP/Qc6QW0rx", + "o4SXN5DQ+MI9hPYtpshVulmLRua8PGOuUpLxzg+uHB8LkAGSnFeV4LWdbqqpjZNMHv+0ffWaTRl+9ev+", + "V6OzkkYLyXPhewPnYV2JupO2Ozp1+Dex6Ob77mQPkx2V9PfbO/f5+0uwah/DpDbs6vWH968/OEvOWn7W", + "4kubYfDlrPFJTS5c7D2baaIcDxXHKlFbQeikJdKrStxBmeYGSgcnJ3DsQVBprZFXK+nwkRuRVaKWuhjg", + "YcIfKX/pjBXfFXw7YXff3QlxM2Gb7zZaNesJ2363FRCgdh0rTiYnd/ZyfDI52Sa+PSLRejedF9dgVohb", + "9qPWVuXhatpVgTRmWiRKPgbaJvBG1IyXUBGHuaXavX7RmkZvXCv2fEOuZdFIeLKqZe4qUQU3WyQ3JozL", + "K/vfZ0+fztXzGbuSK8XaivGGrZumMmfPnhnf27l6MWM/isZLD5IycrNeaF4X2NwrnZuzp08Trz8rdG7i", + "O2ikb9a8LsIu6OWFbhuRwb3tkPX8/bYRH+DBHezU0MbvQ983g7xj8DNlUY9jHhsy8vBDqHno8KYqsw5c", + "eMKe8rzT+xt12t9f1sOL5XY2XCgxVINhqG1er0QULwYdPhuZ2mmFCsK/Khfn5UrXslkn/D7+J3J/b6oW", + "XPeNqDfMRG3AwfJHuT5j//7mJ8b9aw4ZFk6CTdu0vMyk8vgJZ+wt/I1dhr/ZJ/O1zMzfW16L4oxdrOWU", + "/sGsDW4fqESdC9XwlThzbsHwJ6xTqKWxjUWa44/S6rPdXtjrUvigFRzfVPqc0HXzCkCj0sjIum4IVCpQ", + "FZIhhbBVaGJ9BzEbR9WCBtN3jLs/pUH0bOM/OJbWI7huzl0PDLCuA6Erctq4jia4+oTJD/vLusP9nLEN", + "sM9+XNO8eQZaGMNiOztoTmGLSfWi6yZSLkfMJIw3qJwKG3GQruT7dhh9JmZlYYXkK6WtfporihXN2Bt9", + "R3oM5DhAh0K0RNSgLwCzudD5FG1Ra4/PlTV51MpM0N4splShZ380E4dPqutnxHPsSrfhN9NMoex9rqDe", + "ykwovDKlfGKC3/UoZ1OYc7rOY/3PsuQrHOmNqJq5shc4vWTKbquSXb36mwOIQuvSQwVAoxu+dRYeeP6l", + "sj2h2vJCLNoVK/UKqEQccTX92QXZUsEThFrKkGRZapWFRKZkjA+fZ/55Sv6asb8BbQReKZ0jeK6uoToJ", + "LldZyRuRQWoz0jGAyjbXE/cUCEJmZy1iyceVgIfAD5vxssxiYlpYMmQX7j2FX002ZT8iisyIFQCjmpwr", + "/7Od7Lm6jvjNs42oVyKDU8Q9NoA1cpCR3VOER6Z5rpWRhSA0OtPZLSM52sJ3UaQTLmS44AZGXhJ9jwNE", + "Z6Kn8TWYFmHnVd52KFWUVpDJdG/tBrXKqObbTzzImNzwepuh2xH3CWEpWTO4O/sbsVmI2qwlJP5YSag0", + "tGyQmtYvAx7jwHdpd2nW6Cq7IaYObQaoW4YILi6Qj2LjboAEAYDI3yeDDdFOEcXBBTbMPWrnlugvwjk7", + "dmHxq+TsGBwG/c5QLS53OJlLPrxF7XpeT9g19o03urb/8ntDiBv7785C4N9wo7SKPi6KoV0Rv+vK8hOI", + "1eEpOq9cBX+y54w6brcz7Ezwr0I6L45G1wUyGuP+xQ6zY/uLbQ8sNhmtwCGUywqzNKRiRX8ox+5n0F2K", + "EpsSG9n+xCAl8YGLDR+w/0EZg5n/A6+qWt/LDW/EMWsMZx5pb4DGTaXSvV9vDTgvwtPMPd13l3sSHwa1", + "JzSiuepLAT2XbaQBFWTHoHRWWpVDQoyHDOTbqpXXO6EPKC+3QPBEjtUM5sX4GYh/dr8NzAR0N1ts97Eu", + "wTNk+sQ51Xbg1qiB0zuTBWukmC5qwW+E486pgPqVAgTj4jHeAk6Eva20DJ72lVsvPA1Krohu+cCGlIXf", + "h9Kri51TF/5sZRLPh0LnGbYIml3BK7muBSp6eDTemTfZHd/6vX2NJh1UCmRg96Skl4wCWHTi3MzJ9wtD", + "9IUSjq10rows0Tu/0bcCvQXku7DzAYHkxLdfMhI3z9vi5Vw6hkp7O1vXul2tSeCdTUUmsVSmQUjNfQCk", + "lFc06PPG0QXz21n7bhS2YdJSISCwe9Vw6mVIFf4k7T3d6aCo4Btv8hVfiZG6z9N1H7JF8Wckvim5UqIO", + "HUBPv5ParsBau7kjsWxIRI4XORT+YI7BRqd9EFlFYU/0BDxSo8ntMryLou3jLzLw4dBiBDeQpR+3mg7I", + "VjK/3gPt4taitfLWXfrpAQmGPpVyKfJtXiIORxqXz4j6FoXVP42QC4nDAtecN2tk+9kG7GtCnYaKQDTK", + "qv7+fwk/l47rI7z1d9+UtSEAQYir6BSbOqCHWGlCLyLDORIycDAjAvUJnZ3EsnhLfIv+i5Dw6DuVZl+0", + "ExmucJiqvmfbOD2DD7rNAdOAt1aX7B7shlZtAPweBQlmBvcN/Zn+ghYGPARS1n0afsp5yeuh5/2jG3nv", + "WgUvfecEj4TM4FDgFV5a7dIVWf97d4fcSFV0LAYDrNRoNYTnaEtC4WD4M/yXhG4WLZzGjUj82jt54uMN", + "BsRS43GnxNB4c16RNwG1hz3hMydK8JKLt0d/tTPfdP692/LOWsDwolcinBF/uMMqdbrauetFo2F+GulG", + "AxgXnfmlH9zkuD+DCDVtVQp/dcd/uiZYYoV2RKbGCsTMiqtUK3zLo1H2voHCB390S7rhUtFBQfo39gbQ", + "WuMv3gKLf7Uzq9qNqGWegc6PV8H/wT61lEraLuHf7IjhiIjG3bV0FT6aLQSvoTG42LuTqJGCzMfOa5GR", + "n/V9B3gK2FMe1hmPMFFkFS0kLTFqb82tsRsdLCn/Chofeve5zokI1Vo7j9DXok6mn9v/gFttVNHYd2sY", + "pHqL1v3OB+L8UDzyA1o7PNBjK0g+A5sIjw2hmnq78xUrwnhxLbOq1k6l74zWyxgOBV2qGQnUju5xHhJn", + "WPhlji0Y+K2Qxn6k6F/50QUW3wMnSSeZndTMmZg7z8SbJgv7b+A5Ugj+cdjkn+mg6x2VAw70q+Dy58bo", + "XII9AN7/rqWO1WB8KZCnw07ybMRnDxi2sV0ypXvJSH9r/0BmyfO1Jx7dUzp59OI/7TFCQ+EFqF7aFfYH", + "b7GktPqOru5o++j5WBWP0VP+j+TJRjaj7imn2rLMEO8jHHNJpRkdO/A7zO8hhbhnEw3fPofFeISzZcjJ", + "uxNpIfctFuT0XbyjbkY7vna8VXQ0d55WpPj33h2HXQ/ePHamYs+1fpQfjzLD7CGVpmOF+4IPATkIqu1h", + "hyM2vKyFwMnQ5BSmH7Bwu3J2HI2BfgzacfRAEAawHcQ/7PoYo8rXvNZROfgYAu7duB9h2/1Y82pNeH8Q", + "ld+DgbPfHRU3GFDrdzQhPISwYj4R4EZsUZAD/SU6pN6IFc+3bsUABk4bQNHBaJmv+co72eGufB32Rxe7", + "jxhX4nH95RtI3o//9PzQjA0xUvwYjatfyO7gBHvwgU3NlbGCMmMXHvYQO+2IxZAvUbDo0y9ZEYAKe/CE", + "G771r+AMwouYg0OcGZTa4VAiQneIlftkcqKVGJHw1V3v/c8OdObk0+/R/H4hyEql8W21pbd3+IN6kwzV", + "4ANTOqEExoXGu38cW++04jDq/Jsnv9vO7ASTkstMxILouvNIjshlp3TjuWpL3BVr2WCWIpIOFbxqkJRt", + "X+8wrTFzOhG7F6a4G0ZeS1HbYYDTO241X8uyqIU6+X0XKtq9gi376d/tVHeKOigSO538fTd57Sf3JQa9", + "CXiEA9OFKmdg0kAC+wWI3U4gRhfJR7oCPcIO7W4qF/MCz9Z1d/jX0b6fq+vOgl07mbCXi0qoAn3WICET", + "VukGQQVLYk2b/r3lJVT9uEr2RoqOCC1guy8AhpMMX2kiErkOuqU9rYF1Op5AJ5dYtdxBHYXkJ2BTO2OX", + "dLoS7GitNx3wU89Y7N61r0LeA7l2ztg7TIeIm8FECYrG2eY6aU3w5ROi+DQ7vOMRdUhX/B544EXKzFcN", + "PlhAeCl5CJLdns5e0PQQL6BWQL6F7L2sJyQvQ4AjIp+H6Y6cnS5uPZsrOla7p62HRKN0ebtEi20jpktd", + "T+1/sHDVfjlXAaIXpiDAHbG6LXGtSsIAanS/x8hm57o9vZOFIIZSvWR/+SZisWO8NJpKKAyg6QKanr3M", + "z1Wwje50fXPwTHOUmv3Dv3fUj0ic9XZMBwXQ5RlE5zjyJCDNeGcK9p3fyFqAWLnInd50DQi6SXrLAeeD", + "I9VhkCD4XRK4K57qu+csJSMlybUoT8kXTh9jwXiRNI0sS2+P5N2zz5lKUMOnGvO5Zg1ZipEkJCahr+nr", + "eBLGxV57c0jG7k5N3KARGb94rI1jX/qeG5HIrccZq4MlP9pi71wBxmyBDw+zcXESdM1X4i0H2GxOsGTw", + "qNynjykNsVdOYLYqX9da6TauCI9inPla5DcDP9ledjDqoh+1KqUS6d9ued62m9RvPXsIPx6+5F/17U+6", + "Q0gmaeKEfSDQ+df3a96aRhQeLO6YBFg4Fylk1ERnC+oODwVm8JseTwUSEOG7yRzZnCpxAhAXvJ45oPxM", + "uE4nj2SPB/fA9yOYpxGoa0cAqFF/7GH++0H0s5zQwAgGzHUqbnKnO/vWG6MboSb1iG2BCLLxjJZY/QP5", + "VFb68GupyXQOhcRcLq2oprfEJmznw5pnrwLYAZXDsXQ/sWfewoT18rilucnaNBzYK2luiOZDEirq2Apa", + "sama7f6iWXQAr7k92HMhge6QNzydk1GazUHb02y6Yw1uMcyiPKj8EZgeq617TSXPLSzVeBu2WQ9BJ/wQ", + "oATeVYGljYCNOjUfIyEGtiqH8rZuSZf9vgaa0p5pRrqUiuCi3MkFYNkG0xKuK3PX0PzkjP3GJarBD3wJ", + "6GGVtvdJTC4AxsrHS24aYZoJiws65idQW9dp4r1Y2DX/268Mf4PHPLVD59GlR/yHGBL77fxN/JJQtczX", + "SOt/csbe1wILUASLfmELsYQCWL2xJ7Kzflw68oyds+iUmSvmTKnaI0HiBQHrWeHzL1mlDSRBWMvcmtjh", + "aWeidr5im3VJ7XRxvaYOiSKrRcVlHRyls2hOYOCdSeEl5RNiX5AX3+GZPg5T9i+dWfgXV45JXNfdGsgg", + "M64W0i8IPOXbcb8gzFdKR350B8RuqYiXxUF2wQRGmcclD9sHwbztXACOOTEhbPj9G6FWzfrk7NsIoj9S", + "0RDTyXsWVS/bdLnEBB9GEXQKi8VvhfxpYka5oLtMzhX5ViRyPksDGQFAt5LXdn2wNWfUO1hR6TOaPVXF", + "xuHdYWWHX44JhskmcwWVcRO2Ehod67xttJODXq4ffhW85tJEn0LEZ12z60wW13Ola4b5Ku56vOH1DSUZ", + "IbIV1mZD6rVPTZImyheyAhrSd45gZ4SMTX/abZM8no6m/WF+iQtEXbkkhLr76UpP7Sem5kZWU8eTMwXk", + "VFGHyvsdxbuRq3GUXSCob/3TEaBcwharXEk7BU4S0vlem2ZVi6v/eMMuXl2w6B1XBh9XnffEdPxSfAjt", + "4jmY5PqDh0fNwBU+al/CkoaHRlKiitPUupBpPK5PdLSLBksvBiA5fQ2Gk72UdQUNnhOs92uvLB26TIKJ", + "gOq5AuOuAwXv6OzeWvIhjgb38vhreujkznT6BcYIehoB8aPnryefWg/wIDWc2Tg2hcyhHvpBDU759zy/", + "aYGSqZR5MwAQfg5YJQGrG/3BC3gVumzfnbHzzUKuWt0aptsm15soDopgNivwzdB7l6+syiUkG5+qMiE+", + "APQSuimIEDC882oJePz2VJCl9DQB9ryo5QarOMB4xOwtR7TTVhnxmWdQjWQGa6rh4XN89jU86mbpZOKa", + "onFm3I18f2vv8HE/T1GDuG45b3ipVxlRB+zdfRf47AU+6psKoYbtzyB3eHH8NDbktm/ch0Jw+/s15ruD", + "MwQxlEhiR95bezYStg6io5W7OQHMZF7a+39Nwjlj1+EucQ0w5j7WM1emEojbGfmnqlpM/1OuWFRDZF4y", + "Je46f4E3sGwKbizXEBc5V+zaS9C12zwBwvca7vxYcVJA9oDfioSEBykDJJPSZwm47ZOFX5jRDLO10dRy", + "G0i41HvYW3MVbTgHGwTfBN9/KVVRbmfsByzkB7JTYiz1G3/Dt1GKEQ7gJdNwaY3iBrqHr7plhQaIOfDw", + "p6pPdwc1hLC0X30stgCE5FWWEwBikEqmZO355BtSbpWop1T26tXcZ30zT6LQXO2hLsCDLv4OrUqcWD6o", + "wPY60PaCI9PlbQLC4BkedrQ0W7WysPfd5HD3nZV0qnXhBM3JEBdCmqbgB4+ZlJqil0za7Z03nf3gFsjd", + "g91WGCh78aopQMkF6OUI0GLi4DImJ7R/7Jkdr0QYatzECATtvcB9+9T0cY7AvmM2fYodIVIUPFyn+DTs", + "1Q3gM0kPot+l5gqgNg8RduyXERoADPKLuGIH5/1tfPHpB3DtvxYQVGdSTT2uOp6RzN+ZEFyeELRFfQsO", + "E15QMM3+Z0av3K0l+OIXrSwhe6nnDhIhLmzPKnwrFcTyTR57W3EVOgEqyvUlDSvUQXpskFIn/vzOxB5x", + "C4W+fQBP0aUxKTCyV20dLFyGTiVWiIW1bt1pgpgwwbZ1tUCQFAheBPdk4LU3L8nf1H8ZMMXBYSIIBIKY", + "cODLuImHbytprKufxP1UKCukhcejCJ1ysZcbsSXjGxD3kvsnfGcM8js9jBPs4N99E2nN/gpABqM5Q/iR", + "NW+wnhIIYqpaAwpjYU8ToKh3qiHd6w6FZBoEnSbYPYqt+tu+tNIx1kUPSNNJo+DC/sIoNx/RfexfpkVv", + "0LvRgK9fDEMsRp8udD4Ad0ZIpw7FsQikrTdiSz6oIKfGlZ7aN1q1d3aByS0zQqgsRTD9VivdaCVzSCk1", + "Dd9UzrpzEwsEMUgIZ02DukD2nzFzHSy5bMwqh6tzB+Kys+h0pve6V4HX7gHdGjjYHN/BDkwknm1gt4HU", + "A59cEXcHCR5Dhwbq6NP76xK93LDegKa6jYUCKyI93CooIycUVa0XpdjM9lXuY+by/h0esjYiFVQLvomd", + "HVST29WEBJlJty8EbJcNVnXSRFpJlYBsQClpW4eNWQDUSN6EBBMfk0Dyc0dH1d+MROWBCmJgxgHXdsiE", + "4aZxkESkZcIaO3dHT/8QJtrwtz57u9lWwkJr/+XxMl7xWqgmG1Q57+F3dw521M0BzbeHGW38efMB30E3", + "MJf1IZIyacKK29MtwtRdc7DAeNvoDRGAwmL5gwhJ51IF739vhcr3HHO0Bd2DuOU7+++oo2fcDqTDoL/9", + "UM9on/bUmZNJCjpiZ6FaJZtsgJUmNn8w0KRkMzW5ruL1H/udUBAzrsIfFCo0HcTBQ+YedqN2DZ+Oig2n", + "bt+46YheJA1eoNPnV2Sw9E/X3u4fvFRENm3aY/49QrYApgJEBWP1BCKXsDKBJNpATY5KgTpGlDLQHD7P", + "Gt3ma/DpSED/opsHUkX1MR32k9FmG13v2cVcoUOpIoyMDhD67vbEcY528O/cFhIO/gEOiRACha7BU2kb", + "b3fYStw32RAEFXm2CBoqXN3uG/wSSLybODg5a9Qmh+G6D6+yV4PWZjOM3vjsdd5LN+0tF/vlv7fCHWfS", + "NAPnJd6wRsTnuKw/4rNDlNTUlFvmMEmT/t6IpNXL2YHd+u96MfL++YdeoKWWuAii5kjq4GAahFRmZ280", + "+kYoLz909HkCu4qb0QlKgExaZh5NY3i34pPE7GVlhGAq2Q8QmLMqtJjiFfgPvZh0ny9gYrassuYxGoSe", + "ZMWpMcXELUBxAyk+pJgL9EY3juEWM54BRR6OXoKyius7yNE2gGkTaEcyJIlMnn33TFQ6X3d4JEM9mV1P", + "K8TU1uiZPqATtGqkaiknKqEg3FhputwiH9QM4r6StTCfPWRpmCjlSrrEk9zOaVuNHf5SJ+tX40y4zjDt", + "+Cg/XAlRuGySypOO/6EXjwzEgaDpoiOB4aAeOEsgA2LgugU+jP7Ya2GnNCcSSaDugEtOSof9oRfJLU2s", + "pdwYuVKIodxXFMHoGjuxD/ftjL0G/aEXDkXa34RGiR58IDDZfJ784WXXX4ao2dFUJgdPelFPQeZoMY46", + "9Ks1NyJF9uYShOABl88VljsO6cDpaI8iUuMu2CAK+KuPNZBeHXDMUyKdGeCq+cWI2kHOOkXi3dRwRrHr", + "QiyaDOt9pFpds43gRDfn9iYuh4aQoaPQj2x8ez91KOFi0dAu9uT7mGUJWVJugwYG7ZiR2elyyOOM+wTx", + "Efj8wBy42omR9uGHVkWFGgeTPkZ4GL6AFUNKZBIbCZNuggiKXX/RE4YPKl8/M4ljP3U0Dm7g1KFy2FwC", + "grxBRskrZLXjXin2bSiMmljR48Wt7b7BSr9wXAjDWuNScYxV4t7kxZpvs+aVYNxYSynqW8Iuwy90sopT", + "NXu/gYICJjzEWbD6ncnNRhSSNwLKs8hZCbzq4DRNHORRZLKtfTYRKYiEFTPqeuGR+BZb4ICT8DeaEKxq", + "SnuIw0m9D9P8Bw1eCTx2KfZB0SGgbEftrXQchrEHaXDUeO8x1nO6Qr5GU5jlu/4Je/gA/0Cn9E6AID6z", + "EUDgwQy56bM0nC80aUD8l+Ym9K6c3pWs0d7CByINa+KBeR3Px4RxtWWVNojzgWDnlbWQGzSHugviBHz3", + "GNtgb6CnhzkKH6bQ9qkE1Logh8MVhpT25w39WCk4E7i7dw+e9A854L/AkXLkFdkHzf7Lb8k01t/HrF4S", + "6ILK//QSxPGIpSOyNJMRLdre2BCBkXcEndhEPOJjFyiSmh9rMvZsjsGrS6zfHD+Yu6NK0zmhgIITyB+Q", + "+Mq7isD9tTBA6l6LunV+zxD+CCSvxKKGXL2QFIC2FvhogxFGSmWARmKA6CXMb9xQT5UGFpjRAbbPc0VC", + "AWk0lT0h+rLeSBe7z7na9UyyW8njdRs6nIIZu2+KHSxT7xgIJSo+Mc9Z2b1Ogu1ACbFM36GZvhk7J/SR", + "rBCrmhfWwltCvsWIHrv+EXkPYAO4Zmz//97ymqsGsSR8PCvuNRHuPrivaDEd39k7UYsju7oQK66O7Sia", + "QnuZ3F2/6FEv5vBR7AroZiuB3hT6IpfdhxyBHroSg1aFzs0R3mZXNtONLirNSq1WgFktTTNaGh7qZO8b", + "XmGmd5zu7DI6VzpGKx40fUurEuGeYZV7oYWBVFgqduMpD98oL4q7aB4x23kpgArKqY1OZoD946abgzOO", + "UtWbJPt64kOD3SW/g/rWjlEzKkJ6dFjDB8Y722ns9wJ4594x+m/0DfigDrGMrRVFbAsc15WAHX1EVKd3", + "YCi9GwjHvbZLI4HaZlz/+ql+w5GVWG4iYU4rlMjP1cXOjhZm0nMT9TXu8GExeORNdk3PYA9HoaGeSbjH", + "RDZdxuheAX7a5wrlO4zSUZkE5npgSfVz8iflWw95C2kYXZ4OV00Xue2aWq5WhGLua3rtdLU11TJmFC3q", + "5ErTsDqJ0nFTXyRR+sqnvfb8TZhhWwhAXw4kdFTlKZt1INv155X9hElwwMGVP+IW7eaOwK/wrr3h213p", + "rXrXMrybzqbxj9Ct7qGlg6+onZDc268S2/AKTHDbUUR9ajTV5voZ+MNoRXnGcXfDpBdbxTcyz6yGBD7w", + "hFgBCBSgiGF1F6OXAK2sEEQUjHUmM/Ybplz5LlDNHUGTmbkCrVto9ahBnH5PhkbNY5ExfcN/UxomAOuk", + "mDDfWcZrMVfilpctR9ORiFsaDR2rN1IJttZ3rIHTjWphzBroeBfkYDqq6vcV9usjdSGVpiAU+ORQSPbE", + "0DSjJxlNieMbg1zcDWIcKB+HIKLWoiffs7m6XGKmzSRqw9oyPmbLFSLNsTtZlnbcgWShBRaV7kTsXpGa", + "phwPbvSxKV2R5g7W0DmebjIPowDnNGXaX4kGdRZKKM4RbPGPH98gIK+xFjb8AozrGGck5ofgZrCPI6Aw", + "DqlrtuGIsqKNiwM+G+/tumnKmWsTsQCTQH32uUEg2aaMMa+/TJ8QOfphHboVtUnWT3zP8xuhiumGK74S", + "hSubiJJtIeUN9IYrpTAz9m4jYf1Ap2IQA/Y7hi081lj6mmON4Heq3A7laH/af6y8t/thWCGnpvjDDxfs", + "X7/+61/Yv1+9+5m9FfVKMGimc/5QJQd7h1JLhHlQcwc4ycVc1cLeIeStKLcvUXJrsdG3wLuMz2PEBN15", + "oGXdBnGp+XyO0NcsGhLWHtIiXXuHFoY23NrM0nzIVFbu+IWTQR76jfGFbhs/Xr+yCP0A5D5UFdRjKEbK", + "yIY/HI0YYBaiXu6chO9FPSW8iNDf0LmYRz15BMZRs6S2hqC2EXHz4JaAwDYBuHSvSLataSPB4tm9IOq7", + "IY6m84CmzlS4M+g74zj6m5074J7bmPyHyAD/aP+X7HMd8JI0btIxmNRetKhqeNy5AQo6QTPv8j5jWJfP", + "gSmoxUqaBvg8ddsYWQjGmalELpcyD76bLwZM4BDQguE+AlIrwnDqVml12to1pH8P098FjDiuevtys2nj", + "KgpmdVQBOYeupsnfRMElA7ocjI0h1IdCKCMy347po0AhYH1pNjtIUK/vK1FL4BEuCRYow7sO0IP6XAXO", + "lrUwawbIaEQPPQVEjOiiYBXZT+dYKuthQ2YdbKG4J/HnRnHzd6f/F1/f8yCIvyQWhFsA2IK1PfDRc7Yh", + "T39bi2FYv2QeN4KbTmuNZOi+sDlUJ7mTDj8dTxcRSZFchjeOqUS9cHBDbmiUulRyE2EKXeL595YruRxi", + "gY2KUPfWT5t2Yxd5dmRB83l5x7eU2P4S3Di3Di9WIpRWxWU9cb4AqMj2obSS50CGHc3en10DeyvLLcKq", + "knoarY1/E4vOiwm9XMl0cQp+lp2/v4SClMe6ZtZq/3j+6+Wb/5Wdv7/M/vb6fzGhbtktr5+cJLMgoWCT", + "eJ2SJKtY0ekwrN0V3dhLOr4TKfBdM3sHqAm/x5W5czGVPYkkDhL7/HIaHTXwrhOsgUjTgWF1kLK/8Nhq", + "bu0O1QjVHE4XcSOs+R376ePbN4zetHZCxVfCDJTkAPtYIapm3dXuC25kvovwhxDi8Dxg+MFjZ+wHSGfE", + "HxGdvOGqsGo8whinrJ/ijL0SogJuyfDChhD0qlqshTIYRaFXO8AM2C3X1igFv3OpvaDSOu4dU24HXDIa", + "4vn7S0AfoR/sDRftUjKkP5z/CKb8+aWrzoH70oRu/cBTWtViGopzu+jtIHQOIb4WpYDxMJNrOwr49NOn", + "V6Jpq7OnT+fq+YxdyZVibcV4w9ZNU5mzZ88a6Nws15u5ejFjP4rG72GQwYKb9ULzusDmXuncnD196l8v", + "dG5mcRvJSwc5TsCsfxvggo44FOFV7xhyzDncu4yce2jGXvX+YiKvimeBJBopuN3IjXClijXmeDmEFHCY", + "oI8RvUgvwTXImXPUxQNClBt/gXMQQY8Mu76fcgAaxyvLNYMrOQ9cVRikMu2ig/eOJ0Ei803xcvuPVBT4", + "nH7B9OjHYraaTdj8xG2k+Yn9143Y3mn3D6Hw/9fNpsxcw/OTJ4hUT+hs4r5xXjRAcQQ3PBY4OJhOyJWw", + "xkjpbl1DhNSol6QCUvqDKinymzkNSZci+z4tI6S619oYdyWE7XdMTlrnMwAisCbHITkvyCkcZRBSv+EO", + "RRVAvc8dAZuQ4MTqqlLPmEgsADta9S0+8Ay8DEi3Cs2EiABSkLlRfYyKUrA80TY0V/SWAxONitahZeVq", + "gz0xMTOaIcUUBUzNXFG2AZZigiEEnhRCbQQeMTPr6OSd8Q0xqgYLbZTY2IETekQEGQnATAtyaTj/cbbY", + "ztiVi2fNlYeTdEoHPK+wN2kLTRiRRz7De/szuh5P5opk4Jn9/wkreCOsmnlm/+OZLyNGEKlSqpsZo61b", + "wG6bzBWdqtxkW92CV5lANP1VasIWpV5MmN26E/KdEsamtQLmKlImVpsVshZ5U249TiYpM9WdIKeFwACf", + "K25YI5tSzGjMIE+03rgy1jSQiny5UbYtcivMFRYkY3atx8pAkpxC51NMjjRN3eYNAdMSFfZSE082roVU", + "K+hx7X35dlS+wp0FBjM/1CHPNt7rjlE+ePH0WoFSOh87JQCvP0l70SnidNDdRcfIR/t8+oYp6g1SlexE", + "Kxdam4Opht/DQw5iNm1GinqT+KGflmif+n2ghx+4Wok/uZtwFFj7Lttw0OYDCRoP0MRR01J92ab395VG", + "d9Shob5ge2mJu28AYQWhj80QJrIJjrRGo70AYARJ97C9PJe8ysB+2JufhE9AXji+wxaiuRNC2V1voODy", + "VuCXzIz9JMrKnnISQo94d7m3Rpy1DPApR7QihaGEdmdILuW9KKbgFIUnRe2o5kL+dTcF+zTNkl/x2uFi", + "9q899JNTm6BIq5LQvslYezSfq/lcPYJf7QtACWKejOvuQA7y4FRj2nHkdaYZtxcraPf4ORgUI7xKvud1", + "Cm0WJIYumxA8sib3Rhe8RCiRXSkC0O+9DaUnhLSx92PZZg6iZMGvk/7D0QC1Li+8PdvdH0KdX8ZE9NwY", + "aQ1ya1PqEqAiEyGUVuWjoJvpwz+4560GK1KElPSpXvHiwfnxHTk4R9Kng0xSb+1O1g/RGHu9tccs/YpY", + "mnTZ2+hClPvmjderdiBUgBYobj29DO37V44AS3RdH6pSSafBhM6lp6XSpV5tCS7wIY7jAFnXUGOMAAIJ", + "QYBTeOwO66uiVHU01h6F/G1vqYGzMTHTA2vX8S1T+aAHEr4VqPlcao0HNo5uAsivAv08xpk8BNfZLYf1", + "257mZy+A4livsrivSo4GsJWsuzVyKRFCvodYa2o5UCmyx/X8wUHCOlY04E/ApvmKS2UaCn/4Jd/wLaOq", + "1GjIAww4CSzNb07/+ns3ApjMNk4ACUJh4667mkTloN+65srgs9+LlVRRGWJX9MxW5VnpKEX2RtU890j6", + "XNr5Yih16hmu9ufMX9xARY+IxXY6uqtrw+epFP3AKdR9frLTq84XD0zxBeQA7iv1bEQpjGHvLi7Y43fg", + "tzSNzNmFVug3yLf2v5tal09Y1DWiMPFseuAAXXuNgCiaIWXpcc6rBuneKKHBGSFeS/zPKWqn6a/4wFy5", + "2ki2FrywlotipdY3bWWeTFxw3jjiEYgLllso/1T2egtVeIBj0azBxyqpoABjh/0BgKuEnD4QigKfIhYx", + "QUYHL1zXzVxhITV6HKlilLBB1Mr1iDfaZdLdSs5evL8gX+3P2iVxQHwaqr2fPj2zPcC0d6XZHJd9fsKE", + "KsCGJ2cOzi9mf5i5glLDO4X9MwIz3t3o4SILmVTRwkkFPtKwcAMgpSIlLwI8YTdiG9dwwLf7ZH74tYkn", + "7pd1gOdy80gEk27WRW0tFkPxEPcQS8x1xJ+Dk31Ell+0Oz4IXlw2YpMM7D9E/VCB4YMTYr63gwz8yH1H", + "hR2qXeQznyDq04vh/tTnTppg7TWFj7lhb63BHX9khv1NZhH19JKXCj/IkbpnT2EpeVa5Av2TUC8RoPdO", + "gaLHWR652jGy/1CetYcNirriQL/Ze8ycZj6xGreuAVwGK7VzRd0uRC4NofO4Knq0zWRjojrIW2k8+TCv", + "aylq4L2peN3IXFZcNSEOLd1OIPsAEH/2EDfNVYCL2NMtoB12yPr2N2tYgCVTCm5EMWGLtpkrHqB/IizQ", + "QErVR5SgcueO9zdOSA/9DnMQpabHw8LxD/zYGbO1QhZYp/B78pr8BfYmSXMyVc3h1TeBbZk9rmph3IUX", + "YZW68tOaJwf33r7U+oSAJ8558rsWrBANl6XxwO04Xx2BH950GEnIBhNH6egOmh6Fns7TxPnAqloUMidI", + "rlE8fvcVhA4Od8J7mIn19LM+m0yGeBVjc3JjdC4hXcCdef7mMw6c0WEejGUowLYxk0YuE9lONEVZHuhR", + "gJqxif+yc0kChFKnf7JW+ZJbQmx0ALG9n4ww8DFQG1mpDTQeGbOdb9YqQ1Dyz7mS+SmInkrObDSgBxwY", + "76O3k0yl/Rgd+M2YaVcrVPOFKEGpdjgPALvWJX/FV+QEPsRhJtQM8EBT2Vc0QlcECNhNoUqiFnCvhDq1", + "HUqG/j26Kxo9cQgiMAAFNHj7DeX70S043q3RXden+YG0Qjy+cXeAdAn4AcgHQQEfL0t33PjKl5E0USeo", + "H+LbMMGdjr4Lp2QthQzs7QJ73/Ex+RfvLwjYKjEgnYPeLgZwSZP4aECtifB/VElJEAR9f84owOM0Jhd0", + "OhoRDmD/53xyj70cwXhExWsCVNLl7eg0zsQlIMHWZfU63nM4MkMwRDQfsFonjJfa3kmc/r91V1l/IHK8", + "G0M6CnBwFK6gKW/8SO3d0dls8GDiknb4QOp0/GTIHtpXswhjsE0thB2XvWMcVUfya3zVZ8k5YI/BAdC/", + "/NOVnxJU2PzkdH4CoDzGiLrx/SokhoehNqg7t6nElL17143iwEa94rcC7uPDbiTjHqGtNcaH5O8jn+0/", + "CthjfUdSp1+Hhomq/AKhJYfHmreNXi4JUX3EQLEyJh7G/pjSbo92bpUjKxAGL6a72a+f6bIbSNzfGcor", + "NMU/ayzUlCeY/mfSl6N4Zdb6CE7G3S9YNXlF7ST9JZFwdb8zQiqSdI+f01vQYTi/u+1/Grk+b6TZs8vR", + "zkX4vSJU/SQwQODBNfD9Dj/lzOfDj3zOvAQi8kNTkpqQjgzssirT4bObrRDuT8Nn0L5T5WF1SkOD/0Ju", + "f3KgpGn2ofCMQJOObjmWLJMhx//o1zxP1M6E6jsl6kzpQow/lzrqY59seiiGzCsCsydtJZZqrzj2ib5/", + "yCPqjGi54StRZIUoRSP2to4PwmAPP0bcbAefg0vJ3uf+5MBRd8FTgaQBMU2IYFe8On1PTkxqUpMzk16m", + "tOTtCkvYhuPsmfhc2LXbjpQX70T6crJ1UGaG9OUhDWibfwUDGwy4Hqui91jTv4/ojT1MvlhfOsfFn2H1", + "hx4PWv3RwTj2UHatxobVf81VwH1w7Lj/O479h6xjQGkZNTQIqQ1KYTy0fbgCO5/5glvpqAE1rdkjoH++", + "YA320JoaKZD/KRRORrAn7mEHMyHVir3VaqVffT81zbYUPsRkZsw3jenvBBgVBd7niqOzwNWelnwr6gkT", + "JTKbI3o4L6YbXcjldgrql9U8h5DoXD19ermpdN1w1Zw9fRq+56M79rM/v/voMwYKn64TKu4dggRysG2q", + "GqAhKlHDOFUu5mrRNgTuLptHhlXaGIkJXg5HQxI+hE+pQAJX8DTlMHhVMKFMWwu21W0dRYF9suxcEZAH", + "RBAlUAdjgM6BPpHihbL8vBRnz198be0I3xTcTHV1cnbylVQ5uIObtf3X7FaKOysQkDF/cvbc3kTxQQwZ", + "+wetyfGrFHcQaKCnT16cvvjL9PRfpy/+7ePzF2enp2enp/+Jt7JjnV6PPaIGZQFOsMZLOdiZUt7Y2TSi", + "bkyyPjUe7S6NgEE03TC5jUaKbSYVi4iqxt/PrBS8q1K34LayvRxR0LmkWgySFb+b5JLJBvAP1SPnJHsM", + "E0BbiuEnUiUNvR2Pqiiam71b/V21q350dcR0YHmEE5tUkud73qzt5GOhhi+C+2rWGlHPFN8ILHn7atbw", + "lbH/rWs2P4l+fdKBZAsivHskoZTueDehOAQo8N20sMeIMxmVtnzVKiOaJzNmh8QKUQlVGEfTGBE1kFab", + "seuvqrYsrz2GDMaMQSiYKAXhKnGo8RF/b3mJ2GDSYPUVdPYlq2qNJVYMUsunolgJV8pFycFbAmVGYiCz", + "lhVbbNk1JpNfz3ZEQFduHx9Y+49J9LWuFidMIDfsyLVPCsP+3zt1CRvV/hOm0f4HKp6vqtas8f9LewX5", + "ihfFR32Fj2wgNvDVht/vhgMmJ/dT+6npLa8hccZ+8+r1x5OJ/d/s3c/Z5c9Xrz/Yf//yM/798ueLk8nJ", + "+1+ufoL/e/PmZHJy/upV9vFdhg+8vfzZ/u/5/zwhRxJSIA2hcp+7FDQ8RTAFDpYp0CclEDaq1FZ4BVnm", + "OQHZmYbXDbM3Pvb4lH0X/fvJ7vV+RM0IWXFj7aABvVzwhrPHcslKzQtR7KY7jO9J8gD4nhvxl288U3PM", + "mBlHqBbbZhe0Z/yn06royhFQurS4aB3cvsT1sMqYymJtU+Y72+6T+KjY39P+AXFczzOrAMz+/sMjnzOA", + "vWA6xUp8xhga3fAyuxNytU7lt7QbQFPidzAIhs8ZCscBwh5v1jMGhwaUgSLYjSxLttRlqe9MF4AVG4BK", + "hC76k24Jqx+HgTU1o8eRPFRxZ//+8E0RFE5b7kd5PKxkihYRCgT4jcxBuIxfpZENEzxfo5QAL5xGG+j4", + "Gjqnb6Cwc0RhjBWqV/7hT5MTu/pD4Iw/yLIRNUn5glATH4tN1WzZd5DkCy8+GY/AMX5EnY0zHqLDPs6k", + "ChciKLeGq8fnzO+G3yeAPL4eIkkJnH7wEh4trQJPbEedH3+y2J64AY2kbHFXr0Y7QO8v3J9YybjuDHUm", + "0jZsCfL1BbUFJP6kurM7N5Ry9Kd259NnayixJ46cRs37aNV+jJfXF/4dZpijomMdO21kUEyIPujSIcSY", + "phbCIdiYpuaNWG1n7GP0V8Vv5QpSiddS1PZPMuflXHlTJiqcX2wZoeLCGRbQYUgNg1M+lTGyEHyT3cni", + "0LYPlbmLmqt8jbnl4r4qdS26n4nrRl8cZC4aQOX4mW88RiEeUATOQQjNbgZpktDLFy5uhc4zN2vbJGxY", + "Utt9O7SlUcdZ04eE1uMz1kIcN2CwouKDdIdmzS4h/N4XFIBMmp98VWvdmPkJmzIotscaO60beusx/h+4", + "gZRmSC1vntiXL/Rmw6dUqiyKgMUM5/TlKzNXAOOgKRcZE8Fj7FiO3yT3KALffIACt1texj9N5iqUwsWu", + "MShk5ia6h5iex4lGeDBXB2UnefH0kMRH4u/twBVPA1xxDKsyDHrYwRjugRq6htxD5Mv0FZgkJlOvmMKT", + "qLBnc+VhSoDyHaf0WpnrCbtu4X838L/4n/Z/1tcIDXJdXL+cK5pk4zF1rv96Co89X38N/wGP/msxhA8c", + "gRU7gJpOkLCnqt0v5JJxa28lI5rYWixFbW8cM0xsfSPUym7N54ckwM92Sgh+CRQFPzld8GOt20qq1cPA", + "GUORhlcumBBpVxGhSHPEXK6FS/FHVw1C/gaioIaVgpsGqq78BYOQN0ue34RaDC+IbWD9b46pyoXWXduu", + "txPY4vjPbIEyjRa2QzMG3xihxKDuxWRhHA81Jw1I4XTB85tOzelJa4SjjnDfzHSd4Tcy+safV+Ub0VOE", + "g8D3JPndSLI7+bXtopS5J49BOYY0ysDMse4SrHZFIu6W78rMzfysX7J5fMZ7LZDQMigUuHBV2HFUwITW", + "ny5B5mZPOTf+3ONqC+KPUg9y4BP/uwshm3R1wO+fUQ5N9KG4JLTl9iaD7+7XEaCcqQrpFy++bIU0zb4T", + "v9El05FyAyqkC14RmcdD4QPC6uJ2j9zSnOW++XihnRXk0MQKUZV66yBPB1hVUlsUvpiFjxzaD64NB/dM", + "Coh6jnonwF+FnelIh0JHD4vhccitx0K2RusI1tPDDiZnZSGYvlXZsT5aCm6vCX5ncITL9MA9JqoEk3yl", + "tGlkvmcFv5Rihl47luPPw2GAbz0Q4/dzNMv/MVpkr4xBXOX1fSMUZmwOpTbY1WhVyhWVKqEBfKG9qRkD", + "PXGAzh/sPXynE3JFnQv3AbPmL779y1khlt98+5ekVVrrTdyV8Orz2ensNF3+hZe/iIAJOmeewSvT5zP7", + "/3+gct51+emhzz1Pfa63kp3udhrbE0zDuXvPjbnT9XCWlBJ3WUUPdfumxN3V13n9dfP+fxhz964uYj+Q", + "f+Ugok3cfrKfBvFJu93aiIYXvOF7Km0/7cIFB0AE3jZr5hqJGD8dQF/Ban03Rf8WXtQQNaDW7WrNvuq+", + "Dh7XWTfJobCX4gYjWicCokCNUJwKNk54voFQxUCGaRi+m5tszc16KC7FXFzKPc3s0zP2+r7SxJEuDVCb", + "MiPyFjDfa2luOn0++fcff9j+54u/tm83s9nsYFTr0wQC3Q7WKDTzh16rQouDK+/f7g8yJQW/nr86iBxn", + "Lx2/apkLdp438tYO8hUUPUHU/NfzV0/YgsONsS2kZkasgBLAVXT2BIzfZ/SEJ/pJl2KST4ceDlfsxxvz", + "ZMau8M/GUV82a04YIRzOzFI2kGazbEsYwG9raSq6sxPOlwfAY8Qudsa+Pj09PZ0NMHeqzMjSXn8PdNzV", + "kOLD3Y7DRrCds8e6yNdueGbGfuQV4GbWzc5wkK1mxn6SqzVinLTCECisoBmYsKUAoHPfYDyqPWOCfowc", + "Eva5OyL0s7hlajRbCHYjqmbGrmgwrkswlkKanCNhve/gi28HOkidq3iR7Nd7jtQX0A9eFKEUFkBWATfL", + "M6FhW51pmY2sR/gVyCauwF7t8oDsmll440Z6Clbxbal5wcBFDtxDiGIdSGpKQTTTJac4EdbaEVrrhXux", + "FkY0DMrfAWqc1UJXQqUdDLciiwtOxhR60mt3vAzcNGPey3m+Fhn0X25EkXh537sNL/UqcwofX85yXSVp", + "ywP3Cb1ImTXwFsO37NUB7EBj7M4n/4dLmYXp/+38zZQesZJbumpc85IRv4HnOQPUdq2bKQTzokdHUv0M", + "DNCseZKJ9o1eAfS53BkoDhHfY1KZRvCC6eUXGTT6wQxc/o/gF5qc5GuR32BGP44LilXHyk3/5btaNo1Q", + "D3hfqluhGg13FDAyRstfaEM94J1a5EJW+/p/VDNr2YzuhC5LPH8zt91zrrKKt0ZkKslB6pwGFVIZy41g", + "rQJOxrVg2AixtYP3HMViI7iB++bF+1+wNHekZKT7B1zliZyq6OvwCKTdFm0pCnKUkR86kA3d8bIEJi9W", + "tM3WUQoe37mqKreAYtOYjN9qWST1jvKwflEfHTeFv24Kq6htb199j2mlL4MlC2m81kzNAaUByOAc/FfO", + "C+dTP673D9h0/ZeP3XTh/VxX26SovdHg4gmPWiW1RYlD/QoAkwBHhL5PbykeN3ygES7EUtQ1L02aCXFV", + "61YVbMPrG2ZE01aG4RsBWsYFvu25q+0iT23DTgQBPf2B0tXp25HTW8IsJicY4+own1LBpQscIsQQXktr", + "9NjXkXQhLINQTW23ilSNeeniSoYp1AtQxGSOH6Sd2mzRFivRZFvwXqdMeTv9jagIeQ6eizi9hayZKHkF", + "iAp2XNjegztjtvvkwsMjNridCTIpkgwk7PVa0Z2jyMZdP7Bb9uIDdvzArlErYahHDZcl2vyRjk4ssFRM", + "caWNyLUqzGf0y67NEUfgUAu1vntwG4Mz89EKhP3ZbkWHfQvTZG8SAFhL/Dp5WFjEFkF7GjGz9k3kA2fu", + "wRNG7IawyZNjdhVboLjgALSTYCpgeiJqxD9TJj5jIWGbP+Td+31HCm6OMn202E8+ZKD3+7Ss+2ZagbK1", + "LguX+PJlBOo+q0qujhy/o3j9jDmAM9isx3w4Oq4/44NwDo+dc3j4+O8Q4OBxt9Ho9bASR764dzJ3JpEO", + "cWcUYdRdQ4gIoL0mzN9tJozuCY5LzDsMjp+dWpi8FUVG3omUY6XWzsHno1XohGJSNdrppUnU+VpUgqMv", + "w4W3kNcBzOXjOzksJhc98YBpjLtCTKHMbFU+YeGqhQC7/lqc/721lv+DpvBham60JViIXBcidVCMQbcI", + "meBQvF7Z3TDEOPyB3zlnFToZ6BpkzeOKpECtAoO7y3dxsuPAg0kiRs5kuovD8viq+9X4fDeum9hnd2L6", + "IKrSAFCHnxjbPbcRH3S3D2/bE0OOVkDhvSNcAeGljTTmAd+K9NHAjeXIVsxxICwZLf7o99aCV4P82ejY", + "0rXLNMCzeYrkUAigmK85KDKy8smpVgs6xhGeu55El9XNhldIr0m6FxTcVN8pK3PtcknkLmN2tvftZQvY", + "1Uc5W4OnqxBlwzOpTMPL1CXn0j3J6BlX8mrvX5jeFxU+/3b+xm6ppbxn0PBLdDcbhD6F1BGeryGPLLjv", + "x22l0OUlL0u7Kg/tdS0WrSwb6Cwlpq1lFet9qWQjPb/hnzIIqfJaoBOKXFsptliB34weDryWMozRuIyc", + "csuEskJRTBioqztpBPuHqPXRHURnRWbuZJOvU1sE2UuMJ2oCkHkr1gd6Gy5Q3mOC7EaAInd0P+0VI8Mb", + "dQo4l6JJpEABbBvQWPGwgT4n+mjbM0zXnbFQvkDk3bcSRC0f3W9sbfimSpc0dVwfCedT5WKKQkqdfmD3", + "UhVUe7vTX/+h/rzEmKTfSEo3DKCKoASZt81a15T2elS373iZ1aKR2IFE9+2SufRE5DeVixaVA74Hh39k", + "6oEeM3+KDoDO0g15sJex1CKStoNy8Fwx5KsIc42dizXaUteCfJh4TH75cZXyVjjLi8wd3mTBZh17NsXt", + "PLQJCkplGFQ85vLWe/UIA6r35lFWFPobZbMZtkgunKuRNBo+DRtOL5fA+uGDcWiBR4qqklY9OgdDSKvB", + "gB5zJyrb8GqsqoAu+6M4MlDT3f7t/M0U0pRrwfO1yy11ahmkj0bkDG9porMMQo3ofnfvUC7qmpv11L5H", + "ZMUju++c9F5iIQowNPdaUSj9VkTI/d7R724Tka/fNoexL+6iXxviV6cYN/IEHBEm1W0j6qxYZEAugHe5", + "Oy6PiDl6DesScAculONboMWID+ARrUS3rvGdJ5hnHL049q0HzJZ71fkhjnut6OrCsaqr//boFwn7+sjP", + "+beO/5w9gh46yv7bx7wIUKtcqcxXEZnM3MiqGi1+HlEQAPlGywOVt7gkpCPfWvO6GPtOq/zYii9xpNr2", + "3DqjSytDLTZ63n06z3GR3nQOVN2I+9fAbS3q8RW0P2q9KgW7KHVbMGyEnV8GkmykjDTsMcTPq1oaMV3V", + "vBBPAMbrFyMMO0fyDWiT0rfYRS3A0uL23fNXF5SI1jZr+1cKKrMPjt3m8vwtq3Up2LX9X/OMy6rkjZ0K", + "QB26Jsyw15huyd7aTp09fcpWYiOVnPruTk9Pn3tW6Qn7+vRfX7BCbswT+76vEKRxCpqt6J5HafUGOXiB", + "OtXM2PdScSg/KyS3J6gjWZ+rKNlnIcAkLLldSHuO1ZhjaJsNVh+O45XOje3+umkqc/bsWW6nf7aCtZjl", + "evPsFno45fLZSiiAQLoV9l+Fzs0zP1rzbCWaqe1qmAHTxx0r5AYz1k/O7HQEk8qKmJnm9rDh5fOTyQks", + "9cnZSWpSMZHNDhszeTfb6SqvpvQ3/PVWFqI+OTvBzp/sYIzlQSayNOaLg57iip2/umDhBURgWlpD/bE1", + "GmQupjyHvL2Ju9j8QxRTKyxg2Ih7pN90Tz2ZsfMS/gTmBnmXHP04fGzNZYLjZHICkaN3qtxGKBp+SuNy", + "b5jd3ZIg/7SLSIXNRa7dx//6l3+bsOfffv0X6DtILYCKJBbiyWyvndwtKu2ub7drnZ1fi5XTCEEJnL+/", + "9LTKUUuPJuyRaO3KTu+EaZ4/ejJjF1wxXhrILrVW863k7Md373588zq7ePPul1fZm3cX5x8v3/3sEzyh", + "9L3TbLLihGQyHtSAeO5OvIrK3wdVGxXDd3PCBz6xW/4QbYl/7ptfepBdvhoxV+8/vPv31xcfmVC3stYK", + "LNxbXstBeqOw9/4ZMxQ14n6wMAsqzQ+CwLjx+9p0Olj6Se2+A27Bfh88on5EfaY//4wKmpEOqRQ0w3id", + "Q2qFkcJAjXMjtqB1xumFP2kTzoawF4ZoyV1yvkowwdOs+apDRwU6kII+uP1ezL6dLksOUHFjt97BDRc3", + "+kU22xfYLY3YAJReWydJoOB6bFjNVaE3Shi4QjvR1Io9Pp2dTl/MTp907qPLUgP0d4C8SLLCE44MlEpV", + "2U0q26ya3jBjpxFCq7zmG9F0uMljiGtdZVUCJqTNS9GaRDPY++cHe/98b+8friw+iJqrm2PsWStwJAxB", + "7j5wqCSBvXSk1QqmBMF7RLoEUc/QmHtfCxiekY2wVt1cTdlr8LGwV5QDtWWv1UoqYXtwxq5XYO85jeM8", + "Mj5jaivgYbIHeSWNtQmvbbs/1lw13lo+c+Zy/01ebKS6Zo99GuF1/4kapwQn0MA/r1kl6o2EwoQnODSw", + "s8FSNWLDreE+xQWZkj74HyVvhGm8yf1ksvPkkptmenr6zUjLt2PvTnlVTRetLAtRo+1L/SYI4fO3n9MW", + "h4T7Kbm4ulazs4X3j/v/A1bxFQZYabNEbUMp/UtmBJBtNeJ+tuWbMmkn4MvR5pmxH3hZGgbuz/G2dvrA", + "ObgEiSp82vDDB87BRr+MrfdfMrv/hcbjkCofVuF9SK9xPEy/iUXnxV36pcPb5ipl0CFaZXL+zt+/f3OJ", + "t5Ts4sPrV69//nh5/uYqOWlQLgRpC4ksh5Wdc+w+IrRifsOAOZI2GlelXgCM5B5Bc2/ulQd/8fpiMn1Y", + "+JLfIgXVHSa3c5UhMNf/y977LrdxI3vDt4Linqq1fUhKdpLdjVKnniPLdlZP7EQr2clznh0fEZoBRayG", + "wCyAkcR1uep8ei/grf34Xt1eyVvobmAw5JCibFGSE31JrOEM/jb6H7p/vXqyAeWNNnVc8vMWuHGrpffL", + "+lenUCw16BJhGPSv8Ft/1Y5SOwzf7d7VjjpY/euZOa0utxJdBh8FNQYvfP3XIfa6a9G84KfxIj5FJErL", + "tMkU1Ycv2IU4AS0GK+KH001VBqVlI2QMIwZBhi6fRNStTMUYtHnCwTZyDTe0nlxReB8JV1egLj0drtKX", + "vD6dNpmpZ0OvqHkbxkvEdKF4VW01k8vUV0PSmOaNOxT9fhoI+uGoWH6zsTeirnQVf+/35plbB4x1rXBD", + "8gUquRAnYVvD2mJFhZc8ecQm3DbQJO1WiIEntQ4gnaLy/NVI7kSmQjODuSbAqSmwgqZWFGoCAIkA46er", + "iJEI3tpEBP1MIsiyR+F60z4OyvL/2T3ePdg//uHlf/k/j14eHrw8TJ+83f15//V/pU+eH+7+/DJ98F8/", + "vUv/fL3/4w/vDtInq5l8v5OP9btZKeyqV6gN1rpAPJcprwAVa+dD78Twc7G02rn/sbX/oJcs9cFc8rmX", + "S6nO6mrZ+6/h17lPrDAVKKGdnxzBr3OfOH4uy9lShE/4de4T0hqWfNKhDvR7M10ve/+/dD2nAgQNYPYj", + "oC802seCTs0redwJaH4QzodnLGdihgSdG+ES+Dx2AP8GJwbwRBIr/hAoutmFs4gp/gQZ1q1dfYaaErrA", + "AZOWeKsa5ZLD4hdt98XeDSpGnVPtal6oAgKIVuxseAWknpGFmK8Q39VuQJH2tqBQa1QG2bVniH7eyMgA", + "miwuneE5ZvOeQpFlh1XChWpAtDoLaodRTOTppASc9c8aSGyGIFIgRHaNYZRcndadgFF4MDz9hXcQNpWA", + "ScNlgVBwR2D9f8fm9+1yIEJdrQVfqfLe/dloi4S+3wPFUGtf6q/twuNeipbbOG+J+yzAc18PMvcTdP37", + "ssqrzIrUfF3LojxIpAZ63FfReeKTn6fyGui7PvP/LcQclded9W4sH4tgo1yF/08qsf8kUAACM+Htw/Uh", + "6oF9LDGE1rd2umkivdRrG1Odm+bkVOjaRdSYBpt+EZ0+gCTTN94cmMqylJRDeTXi20onRaqLqyb+Yx5C", + "PZhOHR54U5OEicWJEpm/RevGjefQgGU7ZEeocBBOEOArCm+h4anG0NBFrnxvJWuzOt0NY7jkcaGnXHbd", + "Vr2AH1gh1KyU1nU1umbViGWCvBUEPBGQcdRI6SCgIbAjAHG0xeRwpbi+ama8LPXFzU1tXjvoml2n6PeU", + "dr0ZrqMJUKml8OpwU9J9LaHaWdaCq7M4Tksh8vG0M/wruEaicB32bkSIrjuTdSQRrDO+OFxLunRRhn9p", + "MC9PWrTfTQot910Xe2h8BW0fXZ/ZOp8wblnqolt7YTYgrdaRSJ0M0KwWRevilS2qIotSZSK6PD3hKiNT", + "mRqwJ0/EJX/yZIe9vORMidrwcivca6Rf7x7s4+voBvBfoMk/LMR58K0dxXfZIyu9FmMwf/cxfov+AP8t", + "2v6NLxK/8frGVP6DavMd7n5P34EvxH8Gfo/kE/x5pmv/43/pepjradoixAvC5mJWH03mQpuzcakvLH6P", + "zhDfBDo+5ltIloFAKIDXxxAYbAUJw7dyHScsRtqR/1lc8sbREt0nffIFoacj+m76qwIOfpFlkXODSMYd", + "NfW1tleWQn4OL6V47AvdXFA3V9ekjW92KVDzbpobvOJZ6sUJ1BKcOI/ILZk4AZlQ5+ycm8fX03XSlqO6", + "8+7wdSt91Mgb8vgvUj2c7PBYWn/YA0QmnAXgrpGaw6EYaCOFclFIZCrSd5ez/UieKlZXrSH4CaNTHPzr", + "3wsXVxfK1hXcTk40N0W3X5xXcjjDprp83sDHEYH1yG8/7u9uJX8Qs90avWLSr9FEcIwMQYDV3i5dYofi", + "NEEngi/9Vj7nVuahCSAtkFv+afO6HyW8LLgRZvFteDz/+kdQuca6o0iqhkouVBoAwgc4iXo3EXQlArJI", + "mGH4c/dgn8BP/fOCEjZHW8XJ1vnTEV7X40vhF/8Ef5OKXJNzL0j8+Q2kBxcS9ZnWK9OSWlcFXP5jHGop", + "xyKf5eVce/EF6z8CaSYdWLM0B6risHuwn9QF3+mdP+VlNeFPqcav4pXs7fS+Gm4PvyIoZthuHO9WPuFu", + "iyDbQo5MRSxtHuFWqN39NiYsj7icCC4Q/KRHL36gROScl2WmRgv9jKggaxP9m6h9J9wKf84xhDBWnd0v", + "MCXX7SXjReYorHuuixmZh8HW4E1Q0dbfqCAEMryrOPZ+2GMKkIy10z+22TGFGoawPVi+Z9vbmxwHVRYD", + "Hicu3ZY4F8oNrDOCTz+j4b1Jrc5wegs50OkuNyGK7FAguh/GvWjDlFY0EAQ5wZyifqY8SSDw9ZGX4S/P", + "AeHWf7L4OnuEz3ag2PNjTwMf+72vN7GoWJigY877VIU84PjDAJ7e4gB2W6kZSZ4g8A4cXBoAdyGSuhCl", + "FAUO+atbHPKhmGrXONhPCAeJkAOT0x3eiCjgmIaPI/76FkcM8XSwamNdK1yyp7e5ZG8ghSWsh7jMhSjs", + "/HIBcADUfPQD/OaWzwFGqJH4xJoUOIxnd0daY+G8ycpliXT+DdJ5Vx+RMWPRda+X7fGK59LN8Ms/3uI0", + "vm/ikIPtumTPp2KqzYygHVFhwzogUQ4zYMuJIIwKMrwepbtn6ktF+lFVyia7ymlmp7z0piZ8ZwlQNhqx", + "2rCxvBTFALEsCBG+CffP1O9+h8lgFv/4HXvl32d74dVHvLzgM9vUOgCz9AgMXIxrGSA6PsC+u4BNBC7O", + "klf+5XeWEgz+I+vBaLKef/yKWzRNC+GEmUolrZN5GMZPP/74f+LIBuwozCjOAXsFkDxiTnIqS+7Zk/8A", + "P/VaoR6E2F1vW3tdfITzPy6kwfUWxm6NcEzpHQoMtulRjike0tMxrRfbg4IzEO1ymJQXhGz7AhwcSBhe", + "cj5jU6lqCA/aAzwDbx3EEOMT7SbB8QI46n6PaXLdalWtzt6KS7dphQo6umttigYRVamFo/oWlgt2s0jg", + "wsvZgy7yabrIvZFbnyYqWhwYqKeDa8as6YQDQ9Lacg68W0pumR4H+zHJJsUs2908F5VDAUGcv3klihE7", + "4ZUgJwTq5LHu04LZlqkwUybUuSh1JYbslVfFK26sGEDFsjJkcfXZqOCO/1W+H8Z+R1TFBd7PFKVNUhmA", + "JAeXs0L4biBBBesPDNkLeCSn/FRQRi+wOHQ1DCLMVqagKkURiraect8wxsadnhpxyp1giPxXDCp5KUqS", + "lqGoBTLXUvMCqpagCpGpd4evAZCCVdrRRbnEghHgB6CPQcP4jklVSiVi7bUag/OkYTx3NS9jxRsvD7uY", + "arCtYubexs1V6OmuuSsNYjl3bdYjJsA9cNkHi+83YPG9JI4B4AV9YoMxJR0L1C7wNwb8zZKxML+4D6bh", + "BkzDtrzH8PZE7j7ijdxOJPbjBcmPTP8aPtWkk2DQDVmqBLSKOeItSKa4JTn/e8tGc2oE2z3Y7wc/98Rz", + "jZbHek7jWKEIQDl80AZGTDoxZdbJsvRSEUc2ahSETMFNV5+d1A5qd0B0Tao2sGtpDZlaU21gV2sNmbq2", + "2sBWaw2Z+gy1gcW8kXjDNdYmU4XR1UAqFmgDwcTAGiY98OjFD7bTlANyjTL2QeVoqRzRf/2gYzzoGA86", + "xoOOcW91jAXloK1eYMTkCucuzG9QGHkuYhoE4B75pSQk7939OK2KGwephlil1U1EpnKutAIQ3CpeNuNN", + "HyTjEMSsEbk+VVAcoE/XujrtsZ8puKksdF5D+llecmvlmParTxmN80+hAEEs452p+HnTMrhI9j2BBwzz", + "ZJ5RMQGVyYvvAUpsLwUAxBc+hDyfwYXA+MxMee0BUBeIgzDudSojeDHqt7UEAJyrcU0NK/SF8nrDkL0V", + "lw6AQDOVDCfXytZTYZlWgjWd1Ep2OmRf0v5uRnq/jOO6I8mdDmCF1E6XL9T/fHAUPAjxh6vhh6vhW5PN", + "xIVYwCpvhJztz0ktOye2WncBwde5XGYHhzHdyeL9a6m9AH79+k2D/gkpT9+L6ZSzrx57q93zQpa4EcBO", + "TO4MFm6LQ3Rj68Y2XHMacfVV52lAjrNbI8Yds44bV1fDAIQgooMBPxo19q3TDPIvmor37SvkoxAT5P8+", + "Eo6N0oigEWYF5EJCAPNCVNGjo6OXj3EBEpT+TKGSgZDwQwb5+RA6BbQDRUY5G7Wjy+CiZ0Q+imGmdluV", + "g2FMflYWewPXyA7764uffnz5fjRk++MUeSzGOmXqwki42oab36awI/vz27cH0TzFBbX9CIYgTPSSiKl0", + "NlNcsRFMYQdP34iNjVd1LibaCkw8Ao9LVXKpAJgU32NTATkomCaZqbzUUVXCScWCQGxE08Gmww7tq6p2", + "7BV5nwjCa0mAIPqocImoY0vFqGnlwSrXpcjUIzuzTkz7DDGS/Mmyjiv3OA0XH7IOmpiPJaOzDQN+QZoZ", + "iUhyz4jUTPILBYzUC6F6WmNoYjmLdbkm0enmKQnLkusLzAh9ur3N3sjn/ZQ1j6b88jiohMeeUSO2MEZ/", + "/ut//nkBSz+KiuAwhpUOsb5fqM8Ysbnh2q3SVvqh/et//vn0D75XSH+IwZbIv4ZsFwqiJAPy6jMOBufJ", + "CmkRAUJpJaaVI0DdNV1epAhnqtvlRa8td3s1FIZOGekoDDRBLgmlg050MaMWCWAH/aGCNHhyAiLfgsTf", + "ljsw+vtCaQCydIHLRU+hp/U8hcB/+geq8VUJg58Ed2KmWrv/zdNnsA/asNaO8LhX7Fq7/B3jsHmZir/m", + "M2YFhsPM+VzHUkkn4hhyIUvwWwL5weocR8t/lCnYeoueXFoNUZyKIXvDS39Ow5LakLT+9fZ2n0pyJMC5", + "JvWmet3E2njx/PXTr5b4Uvex4lCk0sQIgrIeNhIVHkkgrT6zALUGEUB4hokZhg6/2f4qUhN8QilHyyJd", + "YsArKSgPAcQPAcQPVuKDlfjrshIrYQZ0t9bp7o2RQw8m5EN0cWLrBm6eGqDe9LzCGx1M2y2oXLPcwD2s", + "lWXTunSyKsUcuz9dmIBlHN2ldqbyidFK15ZBD8NM7TV2BpURiF+F3C9w/KIW4rU9cSny2sHthtH16QT1", + "TfoYS+4U7Ief6YyY71itYkY+HCi49Y4wa5C/jxgRjNtMxTeAjm2jlHBQypiVJ2UqpzC+z8/N8dxzam/D", + "I2eGwcTwvKjqgIH4bHsbDFeCDNBTAaiBvCzhVt5Cf0O2V0qQkdPawvV65VkBKH7+rd/7H7xWXs6wgpaV", + "5yJTo0DRIxATo2BWSlEWts+qsrZsRMQy6jPDIYndTbhifhtBQ/WDzJQfJbfBXoYdEQAl6Hsfslfc0bmX", + "uQAs3DBXP/raCIuabqZqi3Ty7PJyzqx727o0hxhyCL/ErU8oxmu2TKtyNmSvald7vd//HMrzZgqWG4PN", + "UblejLTAgAcIs0Rj35PVDjaEkZrPYc8SIuYUtRGwINFYgQCFupB6/uLlDdmhmVoSzdDSqgtpKw6mFzmF", + "ArWlgZ5w0Ka6IKUbYghko9xzxaQqRCVUAeNo0W9oMFMr6Hepcg2LccsaNvR5X9RsGszyq40FcgmIGLLh", + "39oAG4JNiecC+Q7B12BWAIbovh/icb07bbfNtr5UpfdWNbjnbU4ffZFTDc4MrtjTZ39qcSH7awunb6sc", + "kFy/KPCxqCAVqUsUD+R+fhBel1nUN0hyltLrBXqc+JzjVXs/JuX0mREEVt5n0cHdD5fK/p9GQKER+Bcv", + "wEmpCub8FHMjT4SZ82WHGmAWQUIgEYk49oJLPVTAovShv9RcOcDwAMRwL9km/Fyw0UD+acRsPR7Ly5A3", + "RHlH2MkuJlnFXKCYKMUeQVXogVSQd3XgxTlXs9WjSnOaoLNQBoJ6+xH1J22wGRN+XWwp/oTzw5+Op1zJ", + "sbBu6Kl21BZUOVdYELPkTgwAOY2ug4FWmr4ejfwbx8kboz4b5bo8EcaNYLYvwckpx0zpMNfUPQpTi+Vw", + "aG7JpcvKC5B1238ZCInab+6356ItsTuvhWCJ9SKGHJxK6wwkpe2H/f3+tfzx5WH8UhvwHw7sRLsQooGp", + "IQuRGXFHkahpVEfi78+OxN+XzjycAZr422/67NXr3R8H/h/Pdw/f4pl4/eYv3zM4sL7PRNBNkwxBdoi+", + "W+z4Z2mlVlf0zZsT8tb8tHfYZy+0ql2fvSo1VmHGcl0/7R1iQmAIG4GwV+u4Kuhq6Xe/Y2+bgxsmXwmR", + "T1aOITntNJBfJtJWnhn8ws+f/SzyZ3325/r5y8O36KKCJgdOwx3M3EVbcsfGG2Rsuk3Daw/I/uvStl5L", + "67Cp3m3oNtjVKqWG5kV4J3ceq/GcP6TN3aScv6VhvJ2ICLnegm/BQm1Nipv0ViJ3NeHo8qUrHsLQIUhM", + "FaEau2cms+GcOvK662o6VTn8V8s9HMTS6QYfeAddqaAzZZHHlfJMLGVlEK15IC+fHUFIwefd2UfmeZMX", + "9gP25AmM/smTHXbgrWvPeRHpzcvjYmvCVUElVRlyQP8NTNV/86KTQaOZ3cT/MV27qnbsEVz9V872wa63", + "BGzWrJhv8o03fAeO2zOoKx+EDnsEi5xz2CzQ8n7+yy610Cyyb+FnaWteNvKLK3uB4HwwLsVdbXg5iMDL", + "ldHTyhEM2hutVWEEn8L00JuElcFxNM1nuCPh9lYFyxxoB7wtATg2KWWK3pggwPwcD0LnmTrS06hDkPOK", + "wTrQAGFjCjmGM5iGUsJHdifsKOwO2/vp8IWfwyirt7e/yu1xrk0xOH8Gf9KtO+0Hq7ixIFibz1/o/Oe/", + "7LYaKHR+/neO34dnYZHx6XA4xB+25n8J7+NepGM4x936y+48MTCiS+r/p73D9DMMiUi0obmP95BQkgb2", + "dg+g5kDSCMVXN0ygi5wW6CXMjI7/6BdPAW5WQZ2+qLJI5HL/a9RBV8tIkBp8IVBFQTYJgxyOYrhscvdK", + "d/XoNhPjschBny9i0MQAwgDoatlzDrrmroTJVIibbW51ayURiYE7VgpuHXhw4WnlSf1Cs0LkJYQfI2fE", + "+iax60yFvjAMmRGSdqhgmHiywWUF88GgBrx5xydUeAt8vP6YDdCOxBcfPfsGJ3EyY02xtCYyI1kfGT14", + "czEX14zeGLLd2OaEV5VQdq1wYvASQ4xQeNTviHtofIbdl+8vWjEQmfrcIIhWBEymNhUEwZIYiExdMwji", + "LRbUDHEj7NToC8u401N0tmcKwfTmKL3ZelrKZrjdcRVIUzZT83EV37Hp8rgKNE46gyoYxlQAcGAIqsDp", + "gE+6nb9GUeog9Jt8agBc61MqXePqaPmEE2cHEtQoZBUQRl1THhluDD3l0Qt9wlpRVmSqqeG7e7Bvh+za", + "nnCM9wA/eD9ToFqmofvJumAIEe0TjAAfyDmmlh6CTOHFT0wqwILhNtkcbdp3PX4koY3gae8ywLz1Cvx0", + "44n9vqe79m3jGJZbf/uBxPlDkP5D+MVDkP5DkP4tBul75rRoc3uDb6vbA/i4bdN7UbgibkEMbA4V+ioj", + "Bkb4hgT1F/TuiCFmRCnOucoB2pRTTTu0/lWXs3wRRA0v7xkxWxn84XjrLViOP09r63awYB9lKFDPjmxE", + "Agum6SOW8CGMHdz+TkwrqLAKpfmOhCo6ZoeQyhCJMaKZjuLteyGsPFXsTIjKJri41nEnSmEtyviy9HpX", + "TgEJTrO8tl4J+4dXz3xfmOVwKvNF90aDvbaeg6N1w7BntLUDVBpMclsAwL9R2ifRICwGH+Pc/ZYbuAi4", + "4v4hgOrv6RLcvt6W1xVg0jfhAsvvOmZs8f7CE0m8wQDPNI7Ysr/Hi6GQEELNxx+GWqlLvPbYrZ2e8hBe", + "UkEBg7SJeLckk8uxjvAKCsiAiQ8wmOJQ8NLP+C8XQg2sm5WCdWmXoRSLtwbGGH/Rxq0i74cxXuuDKl10", + "eoN1PfA0ci6KJD0WKOUV2ElI3QQc2PQK6w7nIE1FjdEdSI4DKwuRqVEpT7bipyNW8fzMj+NiIvMJm3BV", + "lKDYBGIl1dCvKRBLulTdeqJvmtw0m1cVfWd3ryziKJari7Qc4RQ96Iw3ozPesQb2a4pMQBpuBPysEe2L", + "ysNxw/vW0SMSThkF9FIVAqQN6BEdkW98Hh2I1n7gzdjoDAv1fkGSp7a5StjqMFMxeb0Jp/TC7CRC5csp", + "mOtOeBFAufeCm1YEfpL/45UBkCgtZH7Sg9rCY+GSv2HmnDKcgiwvuSq8DFjSPy/haoeZWoWu/q88pbWo", + "jKiM9pwmAAGcGgmqT8VxUYaZuk4WVMhUiZgC0TMHLvxF5KHOFKROv91SWfImks+tSpWm2/svX17EY3U/", + "Jcy81wl2/sFV8eCq+PJAgW6TXt90mRhRuBAWzT/o0j/IC6hM4pc/homLgs2E+7X6QkBv6VYxlioxENS1", + "SnWBFzD6CULqkxSRuZCxRx0xYcLlw8efb+C3Qs/muoXwSNpyMIH7IZdjIJV09Ki5N8CX2FiWAm7u9+YA", + "FNYLYXtnBdvjVthwd/6X8EWTkgPFzEKca3P3C/PD6+wBUMSlYxWXxsaGzGxZK1R5ymuF3m6ed/XAHTQ3", + "vJoY7rfIf0tbGKCrkziAC20KG2I5uLJl7C/8SV+dCHchhIoFJm2nhgLd3AaaPHV198oIDeMKRHkkXwel", + "Ph9s3Qdb917Zuglv6GLqqbBoImCXy4smtNbSNa/TrdYXAm0hcKUjlFYbiqb9vOi3dtju0hC4Sn5CAByN", + "2jPMCMRLz6JMBHFcSs83S6gwh37Zpnh/IRxWwaWItLACvtVXPBcnWp/5dvExezbcjm2/VKeltJPBWOe1", + "FQU1gKs29zkFJocvrSjHA1tXwpxL/BKWeBd2DHBeIAAF/sRa1hNdl95cBW/FH74m/3oxDA5qLDo65c4y", + "TC5jWmEcj+9yB+Kld3+GQD49nQqICBgwwy/Ywd4bGPmbg6+gfsrr3T348+vdrV3/T4qhh6ru/q/RaOQP", + "WaY+ZIqxrIfX/b0dlmHlO7l1gXswcFLNsl4fXwNqxNfenZXfH8rhcJj1MvURWvQN/+Klf9yXiVRu54Z7", + "ox9DH/i7UOkwFuRqc6JgOzYtW5vu7lq8piNZLmGRRptT/uBQfhCy90zINoQ8JxIftXNEwt001hzdmopV", + "CW6QauF5rQM8sZD8TjF15SyNixcFQGsNOxJ33R5+8M5C9dXPOtXtysFT4bgXHfBdUUiMaD5I3/nY76m6", + "BNGKfKWjtnElDIV3QauxVP+qDT+I33TV8PdLgaVtP/QEipXeTu9veqIKLdapKrxIP0cNQFxc4Js++UuJ", + "953iVKH35rNHrj4xCNDDwotNkWEoPp2UBv7r+4/9dr1hfJIWBP7r+4/v20miLlB0Bz1Die9T29v5aw/I", + "933r+NgatuvqLNFayb/XgoX3Wbh2BlXS94MBDXvcnkjFGnrsM6MvBli/v083CrquvJYLDkcp7LAzecvP", + "9SiM7jPP3FrnIelw8UCsT82/IsLCdJ/aTeK2J8R0qEuxhJi2PtC/Pm41m381iRl9wfDdSBqskEbkwKid", + "w0p/cOlHZEbdxBATDliJ3igCIusmLNrjQ33xigZ2G9QVu3upnJk9EFhDYM22o4DmqkVzLZKjNey99zKP", + "Gz4VSFlLKvg3r2zRrh9wNzkIT3sf369Pvlsf4LL2I9JvKdAfvOjAPhdIh21annVTMlwCVyLHhL5AzhBf", + "7jsbdvgPfQ/zJLxIwV93DK4ZkoFWuoyArzdPWsk4NqbM3jFd4y6lRNBB2N7057jPS2l8Jb8E4NUN0dn3", + "wl1NZDe3X/Pc8TrK4wPJ3pQS+fn0ehM8ub/gLeUIZDYV7JE27PdPfo/DK0scjX3s6Vf6VyvuJr1+Dw2X", + "Xhhr2y3TTzaksWwAT8h2GDbv+72q7qohIhxEYRpRlTxfwvQRscUrJX1UScBjqwEarK3DLB7BoyVH8NPc", + "WsuNyyvtNsozx1jtiEeKE/V6fLxZ692mB+xqlpGcWivcRj1eK2EWyEUYsOkl+b5oaH5BH//KmMnRDTCT", + "VDMD+3INPw/Hqwg9BuaAVqlUBHpnAV3vMAZcF9OWodptLLyzN2kgtH0/n+1i+XQb4k5cLjcZCLW001fa", + "nMiiEIoNkuD6ub3+VdrpgeJXO3zgla0P/n8/8ulqc+YFPMfClKA75ks8pPhit3e0wxLxLzLstLg3bDmc", + "RgxiRSn2+NY0TFiSX6tuieTB+BJ/ZP867vsrafF7AUz7+cyT9yZtFhj9tXjuA2H/Co2mGpkZUGgXbV/P", + "JHpHfHneT9VfEkmCtRE9i1biAscS6xudQnXDuP3e2q+4tRfaFFBal6hCYQSo42eYwpmLAgAdlhTPjVz+", + "5q/Xmw5at+odBJUDorJWycJfZXk83fjBb0Z2j8RasDagMHTDDLYCJbCptFPu8gmxhG83P8o9rcalzFsM", + "KqalQJap/bUZRFTCtDml11PStnglB2didvUditcCdw/2mX+Z6QvVhP437j+Qm+xI5EZQqXAlsIQWgGYW", + "S+7jYMK3dBUHfX2BBs6DQL0BSybSL+p7c8eFaOO2RSsNqqmfMHecEHtAcIPZ8XC4UOOiMgiQNwjOCMyS", + "wzo/S4QsTXKTYha7WCFow4zvVNbiKH+RboL86sqB3lfR+/gOPCA0DqIhxIQLYUIhcQkWTZvf2/THB/Pg", + "5qU+UWkXJ1tL9G99OBOz/WKlw+ZAmClXGNlWkPOmzaxoFEN2VJ9YTxfKNanBGHwODMqfJ6j4cSIIE6xL", + "L0DDPmFWV3l+wlFd7vy5BZoLg/i1u1vUKpq7IenZv/LDHzzVrgq7WKD4YJqAx/ymRHzXFd67quDhiER7", + "aC0nE37puzsIg92MrMaOQicrpPWP4qKZw/qS+uaInjTxN1jGthPUJAyvrop7JahVunZQ8DXxmj24zG6U", + "OSFBo81fNWfnOsbwXLDz1cFY8TA3XzJTl4JyTHEwCb4IwZRGr5k3QpeFYjVR1K+MnpJzbI5rLUK4Q9OE", + "2Rv7C6ZFMkqnUQBDuFaMcoCb+CbMITRwrUiH43j3On/L2DXegL58o+N963teNeaV1/9pIxClcbX2cZBs", + "/9IIuLthQTAudUpBFg39NFZEumr3iD0xbdihLkXyJNAF+p8jdf4qQ/wS2o+R8Ak7S3Is1rjn4mXZMtGu", + "c991kDDF23DRrcoeebgTe7gTMyklLz8Rm3Xh7RZF8N+1hdSVp2q3KJqRvtUbvPRKD1KHztwMO9yC++EX", + "xa164dbQ7ZtxcoBEv+f3Xw+n/mZP/W5RzB2xVZJwpXpvdCnWVOwx1r2d9MOmYnoijJ3IKtXvl+nvXndZ", + "V3M/bPUTIhmdJm1yqb6r145C9q/uOKE4FHzgBS7hYkTyOtkepbi3Wq6f5i0fPG2QTH71KSfBrJ07f5Su", + "d0VKiVQTYaQTBTYU0zWbvM/mTm55wOwhnOCb0kLnyP9zlM0HLn8D17Og3Bna4gXyuhWF7kq+v4JId4vC", + "D3ajOp3vYNdaeaqmvq3O3IQlooTDZ/dMu4PR3k+97g5Eya9Zh4Mjtai9dSR7d+ht66d687LsTPdeEl3R", + "KWm+2ATuB0rdTN74ItF+emriMsG07gm4TrZ4dx5v91G4Ojf8y0wKb3T032pyOG7vZnLCr09L34uHxO/f", + "Fll2JYCvTZM3FUvyRSZ+f+KZoxqcqNWe6GIWarou5Ft35oY/JIX/FpLCH3TFTeSir8nXvLJXnHhVj5/C", + "lIDaBye1LAthlsP7vrPCMq7Y69dvAHwlwoRTXeoGPpWAyiUC3kuFV1NNYff0lA2xwF2oYlvQ2aMKNLUV", + "RYP9QknvAJJOlDb0NH7sxKU7pl7jeT3GlrAw3rCzwFlthZfAY20ApPa5X4IQrxAh1ikVLi4FoKYHiHZ1", + "yojpNh/BXsAO/N4ypBeoy3YKEHbM4M04L/18Dne/Z5WsRClVJ7I6zPU5bg00sCHfTtrPHYG/todg67KT", + "QyNCvifWO2Jz8zVdiNVBgF0geW3ipQDQwe0xPVIu7o7rpSiqYW08x8i9foNVLx6vja4KFP9WTCttuJHl", + "7J2KeNw3zlGB8jyfaqoszJABzPO3hK/CW8fxyNou9hrP+1Wslb14tUtFDXlVGc3zCV7DYSRNo3zsZCov", + "uZHjGfvX//P/MitKkbtj6zwDPcVnRoylEsQC/QOsiA0g2k+e/CBm7JXw0xJ258kTxA6HMiuD0MqTJzvs", + "SEy5Z1l99vzNs2/6zBkBiD+8mvRZwHgF7J/J7MTIIq1hcUBMzbezN+FSRf4ImbxQlVIor2gSkfjGw+ID", + "TLs/PbD+lobBDHAFS1jne7AERMeIw45aH5ZBGUDZdc+McREKafn0RJ7W8D42cSi41cp3BEPEihloclon", + "Kgvpw4oVNRUbo43ERTzClZJxCUdhoKMd9rPInTbMyqn0w3SzMDXMRmjKWsKHJ9Nn34x22Ku6LAeUaQUv", + "w1L5tYfCpFKdwtth5Uc77KgRqlj4khYZ3vMrOtph+w6o81zgEit+Lk8x9QmaxxMg/4GPHh3wU7GvCnGJ", + "JdpgrUew5SO/OFgmxE5kRZTqDD8XxvpFGbAR0sFoh+3p6YlUgtm4SNjb4eErtDIMV2e0jkcv2ctzzzbf", + "zqpkMZ2ojoEQhG8w0BNsDDsRp1yxR1YIdnT08siJ6gjfJHn7uGmiMvrUCGt9G/RPGIpUoJSIqtVMfGWh", + "Hc+gIL1jYTBjqaSdiKLV0F54vdVSXnJrI9GOdlCXYe3HROfY3F7rp7ehFA/8hVISWzaBlJEqBJ9iBX+i", + "b6CqfFKrM/YIbUH8bKzLUl/U1aipM1MwfDioq6YUTuujiXSerlQhz2VRJ8WGWiOHqf1Z0vic1uXxVBee", + "It9qXcZSqv4ZsbBkCf0rb3RB9OcP3WiHvTznZU31ePxZzC2+75+na1FoJYBag6IV9g5fj89BvCx8GAxW", + "yoJ1eHbG0nOWEy9URTEA1s6EOhelrjBm+ES7CZqM3hj2RJ1rZeupwNo+I5CIfhIgGXUO2MbNhOHx487a", + "OuloN3W1N7ckd6IAdu0LAkSJS7clzqF6L5B2u815R4ZXQAZHfoOArXhe7j8asiNa6nPcvHrqNy5TXjOB", + "1iEgHBGf3URIw3JtcLZgFdA4sUTjvF+lo+xXoD4kFhMLGdy6ojo/kKQ6wddXK2E/avdqroTd6g/g4B/w", + "Wal58Vbr19yc4qyfPVvz43eK6nWipuc//XbNTw+5E6/lVDpxx8pvUmhujWE/58X33IkLhKn/fOXYt/H1", + "mn1Tx2/lVOjabeBmINAfjJg1lnAiNsIrCPOSyId1le0Tnp/V1XIVu8l7xzcDnB66XL1O0rg08dmQvQQN", + "nAx5+ozgJCw4DNDaCvoYe4SU0mfSK1DC9pmdcFPEEoo8lvrZLUv6DT/0m+PVDlHgCvzDOizMs8uUVgM0", + "LmkEVX1Sen3DzyUva+uEGZTiXJQsFJvHrHw+dp4Wz4WZZarJlob5xNlYr9w2jtvYAslAlHq5nk6lyxRV", + "H1YFFF00hW0Eq7isQIBvRTWJOsp1rZwdsl02qrxSz8tRprRhI7QFR4w756dHkAKQsD3ATwvJT5W2zsv5", + "k9qxQgsLpi2tAON+XtZpmEBSzDIW3We7YYVwWZVuNA3aeOo2U6Ovt7dH8JquHbswEgyQSC5+7GOeO3Ak", + "Pcdn+y8QaEROpzV5vw9FTUWG2f4LrGAcMGAmAOUedq/wNhwOLS56WILR19vfjmCdS8HDXSpAyEApeuy8", + "VvmEq1NRDIM5og0/Fey1RuYWFGn/d4kVDwEscoeN/B87W1tbFXeTLafp7EB9xd0p/4dW7OirHTayX+1s", + "bZ3U+ZlwA0LXmXvf90trES0R6BbKGX0oaSgftzL1r3/+f//65//865//wz7g58ey+DgI9BuO0NBzZMbY", + "o/m1eexb+Ce10NRlHNhcV5HWwsuWUKFAAtllxZVw4JtCo8DxYx93pE/NjWF5HSXawnAS4xEesn3luaIL", + "GXLQgreHuKu9QTVqjKJMScvAE5e45frsYiJL0Rx9lp58IyptXHLiM+V/qo2woI8xfxRPJ9Q5etuks6Ic", + "swtuGakIc3f7w0z95LlfMrQWx0wYxjyNecWuUc5WC87nvElbjgBTqz8JAFGJWrL6g6BWoEb78lp1iXBL", + "39Cx2qjfjIinLUvbXDYR4rTox1Ou+CnU5e0Q48tDrV5Lf7y1EsEUY5Vneno8JwqpHSYVlaOPuzw4EzMv", + "gQphGHdzl5mBYw2jHygKd37i5YLwKgE2ninUBRBdQoTZB7ypPnNyKqzj0worxNCtx7kwAMzLDrgFzu6F", + "GKHpjJS4dMd5baw2o4bDt7yAviuFZWNPRVe5eL9Ez2kZrwj9DzIjTNt3RK4nMGmxleGmxUivn9xcw+vT", + "GX6wxWHRBjSSrSW5CGH8Ky/Hr0wJ/hF8JiNxicfuWOqRV90UlmhkDSoWLI4X6hFYhGRyJJ8l42xa+7yR", + "vuGXclpPI5VHApIKzgaQxrK1giLe/Zb9PObe1N55ur3d702xbfjL/ykV/RltXqmcOIUb+svBqR74pwN7", + "JquBJgfsABRFYXo7Y15a0TGBPa2cVOTLQXpvJkHQbgjaCCWVVkwHP27NZ8ovXwt16pnV02d/ginEv/2B", + "cH5/ezu9//7r7uD/8sE/tgffvm/+OTwevH/ybwsW/vqTfb9BkY7nGuJEV8jz1wEuPBYvDWz108Tbp0ur", + "jUBCL8zqGtLF5ZPlNuKBMFDmHF5jUllhnO0HzCHk4rEYumU8N9pavG2oyiAAvFhgnHkrwBslTk9ljl9x", + "OPmgrnv7rzFkQ0uJDPVmBZpdXopiK9wrV2heFNKT5UntvB3xbFBNuA3vs0fPDvYeD9lLCfUYeLsnUJW8", + "cNLGW5aCFZrsh3e+BW6FffIEbVuYKERQoq0XCoHTCB/h+hDLCasUCpxDMQgs7cqlclyqcOXj9UC4Jz2F", + "axGaeli7AdvFFaM7HNhnRFbpuh6HO6vnfrt+gVr2m1Hmm16upck/3cgAlh/7Pb+Sg2DfexKOi5WoyP5A", + "P0O/1G2ODWmzELm0VJqUHA/fsXNp5YksPTVow8BUyGXFwS8EtZ0hhsw6CUAC4Iv9ZD39uv7OG0SOfdvw", + "AFyNlcXjm5e95glXWyci57UVjHvRWEAxxD5zutKlPp2xsVC56M+voDfVBDigvOnR8omuXoUj/2EprE1G", + "8gqts8QISRYGnPRVyeXcklzpKAeMSa1NIRV32oBtV6uEy4uxNn7WeZuGvgOdGGmDj8UGcO6WHqdU5Hjj", + "YJlTksTRlYkjwRKdCF66CavK2oJjcKB0IfrMeCugD6GZMudJvclk97VhBbeTE81Nwc6luLDdJV9pRJt3", + "O7yloXVyA5pvHH6wbrri9G8Z8pYN0nKjeB1LfP4+qUJQHnVuGddXhBpDZLmp/VLVU7gQtuhAyVmwjBKr", + "yEbntqCq79IyT7XMW+cmU05XZJKPkq9GO0yGysysMvpcFgALdCFOggHaPM1U7Hj/pxi/R/rY3ou9cDBQ", + "0AEsT4j3E4yMm8Z7JcOQB6HYq2+Gn2tZWGZLfRF7hisI3xIUhye7/j+wjH4fPQGLk/DtYxBGwUq4ONaG", + "SWezhKNRJf6LiTCCTB36WFxW2saSP17T2z3YH7K9ZuUyBU7lMZclugXgehL0RfSnhSsEdNKxrAe3UVnP", + "67NO8ILpMfrZgt/iYoJJ9YTdy3bDp3rMsl6zvVmPTQVX5LNIDGMENwcXvefdSjuY+aAy+gRt0jY68BKf", + "RTLJq/wWXn7ygRX+JQcLjQZPMqgzqQpAFaF9yxTEGbGsF/esn1j6/bzIs97jIXuB9IKAJGWZKWhoyCB2", + "NCRmkYUaaSrr+beyHgbh4AS7jFW4X+5dy9RfNlVxWXEFMDRDdlRXtPPnvKyF3WEjJLERczpTsBkYBNXQ", + "K7wQqAwd5sGv66febGBy+pIDPGSvPXFfaHPmKV5XbuDNHlWQ4TMYl/J04ljleQBeiy5fFtqi6y3MkYDM", + "4qznle+sByBts4p8asxOtHF0HzuwsvC2TT4R5LmhuzBcmtZSvm49ACw/ssasMJKXICGc9hrYuVAOzkGo", + "Um3E2Ag7YXxalTFihygn3l2NtckBPNfzSDxccB8W/YxGlJqvoCHqZeVibdIFkRzT1Qp/FAuptHgQ9J/k", + "80iEbNdp/CTBv/Wh+eNYFh+3InPY+hC588fl3pJ9da7PMCWgYSuNxekmJt7exOuApsdhpmLRulHT85MR", + "XL2mEHTDGGxPWwQ3fF7GZWqEoT07zDOBEbkQWyL195YtjQj6Djk8Fq6Mu0qylJyR3Au2sR9egVlLHdIL", + "V2I/TGIv9fLOybGOtK7WLnyeXziuqAObKgrDnFecLGySDMOs3t7+Ko/zgD/FaEnmWZOiuDL7TNVTT4IQ", + "3Nrr90JOB3xkuDrr9XsQggj+fmd4juFsF9F1wwE01RucuZEnovd+CUzQTeaF4Sw6MsNu9qL2Op1fN+IN", + "3vDqXHMOBdL5xUSoBCmf9JbmkNId+XCdgLb9RV23I6atOwYtoUVtOmuafCqfpuIDy4Y6z0y8skKpGeAr", + "mTsoUUwXIi85aeehu+QYJcFdaywSXl3jtffaN7ixJXDASKHcHq94Dp3fsIhB/gXQ6njd5gfEnfS2SkNS", + "rcurNcXN6rSHw1oxgTG92lhvPtKiUVhUCMbB3AUMxwlJY+9i6hZzAk2lpql++Ld/nGO9cEoegF1Hd46I", + "4cTkAm8i1Si8+MkT9oh2Hxwuqjh2pnaToRGlOOfKHXujEQOiSK/xTx6DA90Ibz/24R4LHFZ9por8tM+m", + "xvTZlFfY6evXbwbcDv5WF6eiq1/8ARWA0DB0nou+pyk3GdelEtb2Y+BG+GtCdg74x9yszyairJK3jRG5", + "oz8gzlmr47/X3BN4h5Sj6OtNOdkxiPtOAmWa+PEuvpdEnTfBbXeaetYQ7nwWWoSiCywLsx8f3588MCTn", + "mAh202mxjquCl1qJdJGCy2jdWFK0uJayrZeYSmUDN0qv7wJjiPd43iKecLg/C1DsXqZQlivuBbCf3/2O", + "kkFeYiiERfYwnxMUwuv8fmXqQ6aYN4Eheqq3w7LehTyTlSgkz3p9/HE+Q9a/9yHDFcBvTnQx2/G7XDth", + "st5H+hAiBfwbT7cz9ZHC6CDUMCRkrTckuI7wEwojCnlSyYDwrbHMw+1gWcpTEDsJkTYtUICt//KvWc9J", + "V4pjP4vjmFiV9d7PT+PZ3DT+DMlKwd/46PDw1eOrplIZXdS5s9dY3JJXTlfsFDS0ZnE712AiTydeWYEM", + "Hz95/IqFvVm2ADSsrtmnudCLg/v3yshc7KAx8Gx7e5v9u1TH1un8bAf9Kh9jO56RUH/eLMh6fVgQmdM/", + "k0PSsfhPv2kvPvhUI7DCVesOoETNouOcKiPG8hJfQHTRHZ5Pxc41NofnTp6LuamKS6/VoDDsXDX0k+1Q", + "uZ+OA/PNHKX9+CJgSUy5g7m+8lpIXZ6FhL0+s0IVTSxBSJbkltHH4NQd7SHHGLydVWInPRtblwNV+OUb", + "DTMFgeuQoDatrfP8jz7PMjXaCeG5zQoj2/Dr1kGXWW93H35qaO6vmTfyYJfjpL/5ONdkoXPYsq5tSNYz", + "67naaSN5CQsZ23u6/XFZ5O73pT7h5V/IL7YJZQR7gDvQcV2mgAMgsrrWvd04bnVvp3ciFYdhdthXt1jG", + "pj2T0MvS/PokXKDfmwBkL4zphfDaLNpui+r8qz327R+//iaFeiiaD7xOL4YsDCXEQ4op3I7AFQH2hFYr", + "TwYR1RzAg3BwrYPtiiJTmJ5KxCXsCJWe71jOlVYy52XjNfLiOD4eaFXOgne6hKTP2owB/EZPpWPSobOn", + "C1/nP5/+8U9//OP2M6/3tWLK/vOv24Nv3//7v3Xt+G3riahkmGvGKQMFpMHKX2rS1xofLg2x3mDeFnz3", + "BeRthWg4zk6BG7JwEbGWDo1x9mKF8Y8vhNCvgGoe7rQxtm/I9riimH0xF2PuGcSJFS4a7/jSG100SeTe", + "zjiW42O8ohztsN0TbRyTY8bVjDlPpKSqz5VqZo/o4pgywM9k1Wrn6ExWTUpOouyHsdL9oIPP9bkw4ImC", + "5Gx+ihjUcEi5EwFzC1LhmtwW2/eNqDT4LyQ+uYmYMg32VXsI8do7DEMCsHBNGVJ2pvKJ0UrXlv1Nn7TB", + "ukL2EeaNwSXv3/QJhf/5hiphrAQ/oh895Ds3bftXaZdU0eopUxfanAkDfvVI6kN2oP12JmPF3pIyTCF1", + "f5ipPe54qU8pEgHlSYwhm7F8IvIzDMQFxP6wpwoGrrTLlBEVFlLF+flOuXFDtj9mJQq4iawYRtzbEHQE", + "d+1N433fDIWk8UJXkIpQzhhXTFzy3PUxSm2AamWzCYgKE7HTvPasQl6ZarIOvmO1MqKEUXoCB7gKv1fz", + "hAa3/3mpLWR/7dLKcS/92IhCPEUxSvMAoQpPpDKKmLAUMxDSEHC4ManPjy3eS0IZYNvaMJwXnNFclGXY", + "lUzlmrjCuWiHjbb2DzI4uFdzh+xQTLUTFF2LgHKZso6f+llDsnyzJSLJEUT/LsS+DdlB0rh1urJMiYtM", + "8SIW8VAFKwyXyrLm0hb2Hn6CxkzcfHvBq4qiJDKFKFSDirtJckiHnRn7yPquTL6AGZyJGak1Gvx05Yzi", + "2mKFYqhQhgEU4ZwM2Q9iRvfSlAGnI2gBecr9STBS5bLiJSo+xFojiwL2NwygbTb6W1NdLE+rvfqeww3R", + "JBRyoDui/cKLNCdUPhtg4dTOiPxn3/xhPiL/swLtN5a6R7t4LZfksxvEQoDe/7c+6cYWaDhmYIE8z0Xl", + "4OItUdhDNmiX/C3xgL47fB2K4AVm3sWRh6vDMyDNezbYHVO84zzc6ekpFWvWCPtRiJIDopsVuVaFbTe/", + "IuukpUdvNEmPVNcNA3al1+MkE3yPwKTCfhRww0Eh/CiTuFLasRPKg8PjL60/rgXF2MO63Uauof9G5mKT", + "SYaB4OdyLYLSeBJSeVddS3nGQulkzSoNlHaeul7QSu+92KPwWl5N2useLmRgKl4+7x7sDzz3bt9ODnuL", + "WvDW3/TJGvDxnscK6wZjaazrx4QzzHRGp1eS8GiEJxJRpKeVVsSf5Si4UbtZgjYf2YxdEquwfvbYN3PJ", + "Y/3Vp3gheOGniv+9FpvMBnu2vSoZ7Ong2/dgvH/Y7j/99uO/dVcR7eoV0m96XaEQf69FDSzZ1EqhdIvq", + "Wa/foyvhfi9HDUoUXSEPS3oFwd/Za0D/DOHn7/u3GiHWEJUnsU6Y3eZyPyXeT05Pux9MCiK1OkToNcKz", + "Un6x9eFv+uRYFleg7ZPuduIZhxiPvTafJ/r4kP0FaDCGakZSY3I6FYXkTpSzTIHtikSKr6ICzSHlHhOO", + "vXYalPYZGE7SGAH5zMmMM5Xq+MGWiEp13rYVmD4B/1sBxi4pHFMv/DKVWjLovsaPj2NAC8V9dSnhaJMk", + "atStEHxnzCHVBU+VKrxB+ISUpBuXq2j7pwZbNAOTnbqKgq/A7w8B+s0qDMIqQMB5lFWIPAeYDDMSAZlC", + "k9OZWQttx0vuAco2Nvpm+6sRq5WTJUrRcpZkT/quJtxmCnaYsvzLcvYdy0uJ4YATXZdFyCeKcfLMgQkG", + "FiBvUggyxcF9wviJBcOz64pCuPtAe4c3QXP3h8N+LzoZbJjalUR6dTAmstyVYY5pevm3aysUf+pUKN43", + "rN+K3Ai3GvkCvDhJLDB+A7UEUtg6yt94hD8PcJ3GshR9JtQ5O+cGo6C0mzweZupHz8Oj+4/axEwC9q//", + "+Sd6kZo+WmhW1NeynI4jmtQmr7agi2WaBv4aVqSUwRR8iDVfqsEQIa55qBYIeOvDmZitUSEIoJZhc6LJ", + "klD2hZHoHlug4SEDfJ8mmwohXkLYz1QXAmLLkZq/2f7Kv4DZ5JBzQS8skOsLGC3Sy1pFhoi0KArgvucz", + "fCrP3+z4SWa08n0HgS4aOoOQ3NaOz23ojZ8EpIZIo9cWLZ2kciZmVPcF4+F0JRSXQ17J4zMxe9wde38m", + "ZivFUSpx+OAfu4P/SyAmnXfQS8u/4NVNWHqqtHCnB/KgdslpvHk/LzYOuBF3FHeKI1heIyrILm2WR5xe", + "zwN6z8Teb5nDtE/cNcQsiuWrnInge+PlPLQAmflOTkVQ3bpsl6Mg+zeNEkAdrcAIINXtIXHwcxACrqvL", + "EWxfQ2SLqv3bgOy3UvJh1aHgrj8h+YeRm0EMYvzqcdZ73AKCg4dL/LzYwPI7xm/6VyfLLQzNiFNxyUie", + "NoP7bxjI8MnxOYV2zQ/0v2srzPHwybKxkoBeMthvtrdv3j27Vu1X2MGj6BW4qvDrbsw9J+r4VYCJxZCi", + "5HDAg9VHgyqhQoXUtq0zZ1UYXb2NdfuuMikQT7owuqpuRuLfm8X269BRl6xjpfvdLOd74Zas483JpNaB", + "WFZbqRCOy/Ie+G6/j6FrYUhXrOtKRu0pOYQBhF1aVoPSv7uui6yTFy+thN+gtCtxQbMD138IAqEiDKwQ", + "Y6kgoTZBW1/wToXEmndWsD1uQ07NkZzGO2SIyqpVGrLMHe/MR1D19BhzeSChoR3i/7YZ63idlJ12Y1+F", + "rBCsFgcZB4ox/yiA5B8TlSY/sia5pvW0s6Xwg98LTGPAfGPKlmheqGJ+8sLXkHlSdDxuN4z7PNcwvhSv", + "4AEKhdJJzsTsQpsi671vf/CxP985ZPrceP9+rzCHZf2RnOhitqGBzPed/vlxbrNiM4ShMpBqwMsyTYzC", + "meF4k6Zjs/SP0HIWbvSPm3lEIvOvfFxMQYrEigR/LIvLOTJt2orZINga9f9x6Wk6X1JJa41T9Q2NtJWZ", + "5MdwgPlSLKeoUgRMame1Ndlq1z6TlI51e2cy5H/d6dnEpLB7cDTntvsWTmh/cUMgLe6KzlU9PRFm/c4B", + "kE3mN8khmky+dvrep/GJSPhX8wnKMlvFKZIiec0SUSYivtEac/IKfumXdu4EBlAGOmxlyae8tf5ZD/BR", + "iO2V5WAqlSync+/Uht6YOFftbG3BhfJEW7fz9OnXX30dOFuzZgss7ne/Y8+FdezA8NzJHDWTAXsBZRuD", + "kuNVkxzdrKIkNLOpn4JIk0T9d1D9jrQPceENfsw59i3YqTdyvFZjhbPs0dPBV5DU4JWiqeBKqtNxTVYQ", + "XfO1DHPKd02yMFs5vdDWnlYWwC44bBgmBzR6EGQ7IB8naoCRTdIE3K4YDtAG3yZlz2881rfp4Y58wG/p", + "onyJtYHh121bEPd57iqY1nVtOxFqPL6pEfVhzk18G3GvL7pCW68V1frVNSd5C/EJe2mw/NW27l1EpC66", + "L7ZCaaOrncrwEcF5NhW7w/dMKCPzCZSoTUkTCmJnCj7us6ZoRvo64e14Ooa85ZNZoGgC0JOEOxk78+Rp", + "dDmoSq4EIN5+x2qL+SGxsFh822hAbAA8SChpk6kpdw2kX1M6CMt3U0mQnPBEY4vLAh12qaOXzZw27qlY", + "7HNpvOXiDiWxENe+GL5PDjxK9OogweToASndnTtkRdTPquO49SH8E39o5nZ1dIVtRr78vDZOlCHVd6Ey", + "TEbkWuWylJS4bSCXSg9KrU6FGRTCArhMQk1GnEqo9QuMCRK4pYOaUBA16jQrNLN6eeDFIjH3PhNQvwPS", + "7epTQdEct4cv/83VH7wRbqKLH7XbheDI4teS1BFiLFTX8b2/p7e/JJGv44D5fpZAKqZH+3rMY0kExyGc", + "QEhlvM65DwULm8TU1uhDbvJcDcIpZkYO2R6UlPYi01rRYr9ewM6YfzhqAEBgKwNap9O+nWSAYCtIRamM", + "VHgzUwHCu7EnoJklMSNLGMkGAMtiB4h1u4miIJ/EwwxRwgMbu6XcNFxurGsPp+WTeNqnKQVGEGoGxoLd", + "G6aY9haB9ud4xCZY4zKgSaj8SyUDI88AaJiKSwNA36j6qxmNE30fwoDJmSkCeSN7wOgLKC7orS2oJTgE", + "fwoW8DsTs1ET1RpR4rnFXAZIpiGO9HvLRv5N+gYd3VUpAow6ZcPlSb20YaZeNOWfwgTIHCq9AKB0BsJr", + "YCNwBlFZQduMDDL+aWhQLxWx07GsXACqSwdLBQfqqcDqB9BupjwtV0J5+7Wcoe5nvSXvBUXJzanAF5vC", + "yp0580TGL8jQCoztkKvTTXl/5vsiPxCNJHEIbTLT+6pBrCjbSuQcWQBmR1oLZRhiPviDDLgNGUBbwPgc", + "twvYkGSoGiLndaoHfZIYGIRU4wdZcFWIQcgkqq0wgzHPETgVuOnf9Ak5gsJ4gwOo0Y8RPQfywAqA47Vt", + "zHs/6VikVpgB1nvHDjJVGTmVkG8H4B8nMzZqNnGEsQwEkBRToVMpkCni58xqNtGW0PWjDMgRN8nz6g4e", + "TMOwmQqYH7kuS14hCyHYHVTFYylDQoHyEg+77uLjcBGxwMPDvEIu3OZZedoljOmOuHlr6p0JeoFxeELy", + "fJs8/g9s+zb99fOnPHEcpxu0ccYNOd/7mPK9LAzuytO1IZ/z55F2mn26eaq+V0HYaxLXYpD2Io3177tY", + "X5jjTcv1jpon6RpKuEUbSwQ86U7w3S9uw1vffbC3SFf4AjS0X+NWrnQQaNUobKlbAIxjtKUj/l1dgh+0", + "QTFY0IR2caO/TG4NBiQBYjphplLxGy8c/HkjpHP0oCfdhp5EtHwPFaUtRCl5YKf3ip0i+A4VyKivJhcA", + "E+LJM+C70mYqMCGCJuq3PaoBswJylQOgJ4EYAVv+LlPzgEMBBAaxUR121UCRenPaN7d7sM8UAWNU2kAB", + "1wRTRqpY4RJERIKrtByN6MuUBQ1glDYLQmF411KhhT0bgaG+6yQJ2KoJlaidCUeYuQXjbViqiHH1mzNX", + "cDVvg88TZOO1+TbUeQ1wjzcXkbOMkz1HRM79Fwi2K6fT2iF8dbg45yoeCoLnFgXbfxF50+jr7W9HEVoX", + "8KXDfXu8Tiew8wW2gb1vMgAWe7hW7Ov1brCrpNrhh16z6036bKvWxCKu7JVX4LRFgSohsoG7m0JMuLaa", + "uAbILGwoDjvFm924pohdviEAtU0qjLQniwmf1+YSLp98ApMYxyJ/aenSTbCL+QPr8skvsZ7pJs6ryycb", + "PK5r9L3iQrK94mkVmRvWEq4cyh5oaqyg0otNrQDxHTuXVlI1XO1NfONkLisOkTME7i8tIvmzSqgC9vxW", + "2UeyRlAJtiq57I4MWl6u9e1EUOkIOgk5sZqAtWm93YhXXn4X/JwBR78IhVFDaRyWQzgBlkVFxU+wi4mA", + "ssnhemhwanRdsUM+dqAmA4B/1JeHbFcl9Qx07XI9FVSbIYaIQvTMjJ2UUhUUSRDQHwNMJQFjK60GMgDO", + "OyoboM0UYgvWLR3zQigrDsHR4/lVZYS1tVkfMxsqGZXCWqjQyqE06isuy9jGVze1l8BRIIYdbKColKFN", + "52ZJKVttCqk4ZNpY5poqMy0AHMJiDQZNPCYbqyGDFCg9QVEVJgy0haKvny0lkqSIQQs0+xMkx8VEWxEw", + "l1rZFhRVcyKw9AfgNt9+hvyhiEA0Ng2TbJTJvRd7Id2hyRsxwurSn+V0SplqYj9zXpZQLT0YTU3N5JiC", + "0Ry4hsdLi2VAClFkCnLpq1PDi6RYSUhhIqLrwLRvI52zC24zJZXD+KsC78J90+LScyrpylksFGKwlkDT", + "lL5QVMqF6p5kKskekpYZ7Y9tgbf3OGumhCgs48VUqrRWNIVAUcgXxcbHQiFzbwoQHFB3u+Zla0xoMXRG", + "PcXNBJ3wRbI5n+sQaCveDWpUAAlv8Iw68MH7hCHeWemhoeoGapzaf7+Gvv5i1cY35SzojH2q4L0FaLE9", + "JJ6S52c2zMTrFC2qoSCSOWLYnG1x6wUsPj+R7574PnY9xQXOjye8NSfIukzdSs1kPh07aCvI8juO2Voq", + "b45yriw7EzMId+UJJov/UzXALGdihtFtoR568MxOIfBViYtSKjEoRImF+hhUNX2E1U0fD1lTszRqeRzf", + "QVYCfaI6ODqWxWheCxpLLBsVR1TOWGX037CySnwZU6mbpMI61pHPlHTAnEEiYiAXL8uFuSel5/x2wM9h", + "PXAFOgOjcq5+EDO7KTBLan5lmNMqAdJdxnSOAjv3sI/BwLB1smvTPJlkPQCnwOXv9derj9ruHSpejYGd", + "PLLOCD6FMEKqj/v4N+er9Vse6brzdDZn8jN069BBg+vcDQmo9VldYeGvlWzMW+Z8YIV/yZNQAJVLMA4Q", + "qCKA0IZVHWZqf4z2IibnetUwvQEa12XZJOqyo7qCi5sdAEjAqHrsZCcgCfVRamc9/8qPVKCKu4kNNaeH", + "vCigAmEu3Qxf+0WWRc4NtnKcT2p1ZodP8LeXoXI0/DiIvw7TytwwHD8DXibjad6w/WPcZClsn5rwX7Uw", + "B9Ph90NJhCHscTcKIfbUu+YFIoegUqj6qPJZrGBYwmajCh2ys0ZG8AITvEZNujUy7MpIT7TwvecT3Kt8", + "GIbKPTMeWcdLMfLsVl94FjLRDjGET2bQCDgikq5DXRLLx2LgG2Gvj35EtttZFqiZQ3ftol4zeL/MpCK3", + "HmKpieNSUK0fP+JNVNbhRSFRgB0kGjzK7yt160Ms1glcsl0I7v8MsN7y4GdhbGdBOPqBOX0mFG21tMmB", + "egfZ9dJibQB/PH/a20NLDJ0fmYKtAJgRp8FZhIn6QWOa6kKOQzH/Yaay3nbWY1IVUBwET7HnWYUW6BkC", + "o3a+CvOKusq/Dc5/SFi43rIIes3JjCFS+BcSPPiDmMUIiTAHcAHi1D4NC/16oXtzou06OBir4joSLAmi", + "+xY6OmiafAFXAhXS0h+dUJWmkBbc0e184lBJKVOEZAGXkCFIP/QNDkEjT0+FIR9vTBFaBmExf/P+Jkzj", + "Koke/PCEQ8kg79gzdSTYUSOkZ1UCtREkvlblbJipkeEXXgLYqABYprEkXPgCsK8F6pdoZAAh2OiamXfg", + "YCH5NlaztEwo/36xXF7gRDprq4VT2O8ZfnHbldWWbdGVuB9xzT8Z9eM3Hf+Gld2uPvBfDvs9ECYXyg2E", + "ynUhinl25GXJ8E558Fz439V494HCIVq3yYdEltGdyxV3sJ8pW+cTb02mAmlQK+nYRArDTT6ZMXHpDAd3", + "AtUdPXjxqs/+/PbNayhK2WeetQNAAbA2Kq/uVWG4UJxwUwz4BTfiu5CxVYiq1DM04PBFysuN+bYwHu44", + "w/s1KESmL0gGde1adzGyZZzjgbffd96+Hl8fN3z6gbGvz9j3UYVaGfA2bY7KA3O/l5HPNyNi7i2Uxm9u", + "O5fdReyrc17KIroKglshHlWUIUF8iJCv0yB5aC9bT2pZFh3J6MG7tWCa7elq1g84Rdr00c8hTB+dH7mR", + "J/4PUg+0wRvrqEXQcy+gwkggmjRTvCyZRTdld9z4UliM3l3kMK8KuDpMQSgW8ScehNINgEl8uhc/oMOu", + "KuizT+/cRukZ6OtTSs+k8MEN9/3yYSw9J0jnthgT9UVhWNJUtj7AP9aqkrNPfvau8NV5MVCIy5usknN7", + "7GmtEI4WavHNBIhvpJqPQooNxvBaBLsM1GDJ7t+cedXiOIschmjqk2v73DPIAdmazpcDpZn2Fi7eOnqL", + "XOVmovYRBqQhwU3hzEMPdxS8jyMocJJLDwAvik9lqasA5b8Q/nor8XLfL4/HvU7Y3FoR7kdOG34qDgWa", + "NC8vJ7y27pY05dsG398tikYmAXDV1RJpJQL/qp26Iej9Dj1pCwK2B1PhjMzt1gf8B/y08wHNyI/3ON0d", + "w81vnHev7DOpNYDd45p199+s5+cN4KcERJSse090VVXOgt8+HQxryg2sSEfDhlYOrAlOGRthJ/AueDR6", + "/aBfe5FeWyqVWE+XhaksSzCAdvuMmu1TqkafQaN9hLkFYLkph9B/gIZAo6XZiUyls8erEFqJVok+ZsQU", + "gj31hUIcvDkqgtrhMCRy7KD7JsSJCwUQAwG4pY9jg5v2dHiQjoR3FdABPz014pQ7wsULVYZDmu4Fl3Cv", + "7meF59FCmYgLbc6GmRrhkoxYXgpuLGuXksBZNr6oMNRCWsyI4LXTU+5knqlkjEO2y0rfEDMLW9CsOZGa", + "EQO86aCyGNQpVEDkKmQiUyUS8lwhduyCj+nlpchrJ4DbvYFmdhsq3JA2vtDZKtfSbtyr1oGiLcM0D9jT", + "3LVS/WZ3hYF6GwpE2PAc5RE37RLnMXyLin1vILkwpONXutSnkEoJQfMDyMBJaA5z3imtEFA3rJ6KUKEq", + "bFGyg99BFNDMnz9p0/jDASQKIheoLeQ3zTBPc+C/PBcMGWGjHm12J7BmhkUlCxQFr2XhZqSpd+Jm9K4b", + "1pfo3DPePlZ6Qa59Bmz5VJhTcV8zG2Yqnxit5D+Qps6EqBgGSTKpmJ2pHMPXxSVuBt7Dh00u5ZlgRxNd", + "yfGsn6kDbd2pEbbPjr4CSgfaBMc+pBWbcG8zZLul1exM6QvFuN2hVuNgYO37mfKPT7gV8BOUTTKw5J7i", + "4IkY5Ho6FSbHV6DW7XPtJuH+ArMsm8SjaW3BtKBBkaQFmNozMRsyytUEFFshBhd8xmD7IH56n1I3vTjB", + "6Vj0+2BnUE+uwpsZOu7hNSxrh8metLqhAX5iISWjaQfiKKhIb8iuZVm9vf3sD2w/ph8+ebLDftS4Qyhm", + "T4S7EELB53bIjmJCsXXcuEyhi0rN4AUmx5CzaExd4b2L7y/WCPate9qgyxqsgEObYVvLvnuwb9mjQAPs", + "F6336KfHtIGs5GeCiUsIRAeyuOBGTDQUqiLxrcOyQHS0vhh4JUDls3Z5PBrlL7uHP+7/+D2uAGVOY5nA", + "JsjWbxqlK+lzYUqs1YGJAHaYqSOClocwQ1zF3YP9EKvRERuoBDdv4CRf1ztCAet48fAPCcgfSQaDpz2I", + "+DjRBZW128MGBy9VriGLFD7r9yDm2H9ecuuOYY7FsfTjg0EBQVHao9+0nW30nRDreM0rp30zUJizt/Pt", + "t98Ov/32Yz95/Vny+huN6jO9/QxeBgHQCMs4i3fKs3AjrA1JTTC6cAV5X8b+sb+muEv2+45qIrZGYOuy", + "MwYGfmb+a6jYdDf3AE/X8IMc8FmpefFW69fcnyH4bg3/1jsVr4bfiELyt/ctYTIVoMDsgMtGaRkEZcMg", + "Uxn5mBF8EiWE4KH59BtOjPL6JDipECC2eXiYv/ieNgnnBEJzXJfQUXN6+2uk+q2VkXd7PGBuJqGXDkYA", + "b7AWilSS9/JCVEbkEQ1iztfxao99+8evv2Fp+f/mA0/VYsjCUIKcFlPpoJSLtAx7omDEZBARsgTQ/THW", + "BNsVRaZGoHEfo2QXdoThkt95010rmfOyiV8hix4fD7yMDrAsoBvZ2oyhPJeeSsfkQq5MkzH2n0//+Kc/", + "/nH7meeLfU/m/pj2dnr//Z9/3R58+/7f/61rx+8Tahbsc8tdvw73ha+6WDD6+tf4+J2icIwYavBs3dEe", + "cideY6rs2pwbPlzqd1930M958T134oLP1vbXI3NqzNaWr9638fWabVDHb+VU6NrduNjB856UaJ138wM/", + "v4boQKznLWltLe5rGv5rAJ8O2CmETl2IE5dGi1ASEc5cx5omQhWVlsoFB6hl4hLKACYBhLbfRNHhn3iP", + "AhlHffK5YOUS4amRnrmJMAjr0o7XZuuEa0N50BCvzYh/sLEsHQXIQWwcgLVSKnC0FWimRkA6ZhY0JctG", + "zh9w9x9hMlhTK6xbnCL6jIVyAPkKzuHwJWWwjkOt8lhgi6sCYw+XJVCFyk1cmn2kpc0I+aQL3+0np/x/", + "Qsnhua6X1y2nRYMzBUb3rxine3NljdNVtJ+uGxODu/vKUNcoytRagFCOCSsZgZtEnQ4CrC+9RHczoSgT", + "FU+KxYw8u+J5Xk/rkmOxpFoho8EKXQH5n4DlM5XzsrRJ1nNgo0wq6wQv/CoFt52uLRuZWiWHZET4dTi6", + "phZeiPLTpDWGbE8lwGfmzAxmHsa9tMhS0tXmyiq1O7m6kNL2hnruYjT/W58k5n9RmwBlhgiwZdjJG4eo", + "vnpohw3ZxvsNL6apZgE7qR2kt7dQtB+441pOB0+DK/jEjXDJdcowdZy/OzsKCb1RNZLfYp0lfj1iuFeB", + "j8kO3noJow7Cv6dlija8SqvLBzs21dYtrRIUMhKgnMVecx0Ti0PCnQ8pJ2C2JEBpyytYLC0sdE/5T1NS", + "IQg3Vk24vWMpHIKMqByJNmGRHyTvJxQB2qjMvZ8Vfe6K8wQ3cKs+CXdJMXPahlizhL0Ch5AfSzEAYyNT", + "f9MnDea3MJiJ7ww/F8amMMJMqqZkLbdw302xVJmC8D+IdcsncHMTothoCOgR4vbMQk0MDMjBYCCn2QwQ", + "ERrzLJxI5XkBMYpaOVkixAENjpdRxbd9ZjXjmRrFgjSjphLQVHBlIeEBuI8RuVbo1AfnkyxLULhjQEAI", + "9OMqogq3I+ygJFEoBBTy8ehzS9YGgpvFofbD8qaBh9ioZdI1C8bLcvZdA/OjM4j6BgPQ6SAwMOavEn4x", + "AKl4sc7N8kpD91Q43G4poauHNlc8CClSqsRp4E8aiYpBWikK4t0SmzOYI8mxw3jZhjx+w9WEblxgmFrd", + "86gyXVuoL+bEtEI4PJo+qSDEb8GzN+xwW0cwAmGzJhAgcdJLVdjgly/5zMZUYSx85hmObLxaJ7NMjQC1", + "ddTC5wUEXwBiSYuLU4FvctorjZUeqOvg0qJYKttgmzXzCDCQWGUcdGY4GeRS96OBUE3CLpvwQl/gRHII", + "cuuzPIgZ6VhdhYj6oCLTXGEoJZ8x64+LygXJICh7AQjA9oKin/xyKKex6kUuvmMTwUs3mWUqOOTgwuFM", + "QlJpreB2dTTWJhf/4YlhBBA+dB9B3agZq7SFKu7N6sKdBwAPcBdQ2dK545oQNlBTyI5qzNWK8k0KvN0J", + "OKnlLF64NI7IrjuXTF0TI2cOcKDlw9zoBcZhrW7flwidrqhgQ8elxbCA70dP44PBsDaOQa0SPXVhST9H", + "CIBz/9Oifehj4Aa3EvRziB2GsJ+VIz7CYZ6JGQWOaAjxhJhZZ2ZYSwMKTowpvJ9mA14HQMgGNpbrShSB", + "aSYMwOv1RqpcVrxEfhKK2ZGdEfTiNLglD1XylbiAjsKy4e/NwsUI3Xw2QCToJQVP/tDvTaUKfz+dDzzp", + "9y4Hp3rgHw48Rx4ESPcB8D1hejtjXlpBgFybYFCwqtcKf3x2070v0VlfxBiAuPNNnksr9Om1Xhr3JLzO", + "ey7Yu8PXEd6no92Q0jBcjRsNgLSzwe4YtmaBpuvTU4LW1mWJ9Vq8zJaKWW+mFbbd/FQqOa2nKV34TT8V", + "5nMiku5j6QyvAmBFxKbmQdFVTgN0iVupqXFPc3vpSERTAiJNYxnQpWJkZXpvOEh7L/aSQkmbyPJNivLc", + "qcESiinOpdeBTgumwOGrPfbHr779AwbaYIT1AZTumtO/cRewtyH7CfDIMzUV05MQuoNR7sx4IrDy3Isw", + "0JmZqsuSEMSNmOpzQeSNHw8zRVGWDdSX/+4cwcnn6+8HrYyGkmj+XcotTOUIl+hKQWy0Og3NvnzLT8lk", + "wswZziojzqWubXhlSoehT5WqIOySjTLaksF21hsN2S6bSgsetGiBfb39bRpRoM+FuTCSUJ2TtI4a0muW", + "y97x4A3sb3f0ZzqOdvhn/AXDQLNeVyDo+w2GF+CGwN4shCoDFQ2AcP/9s5u+1fjlt8QHF5g+jonBnObT", + "Flri2xNdhzRFwgSKHMeqf6EsOREjnZbhVcTwFIjhXkf93kv8I+SJPNbes4GnrKgC1e9VdaeHH5weNmwk", + "srM2f20KQUCYdwQgTkpjZoqbwFALDFo62H2792esRiYtlDWhWFGoeMrLRSZPLKaDc2Ju3wPrvJes876x", + "NlzuB9b2RbI24kfXZG6J+tuUg1kNN5kUzT0SWGd0o/dUC91h7PQK2Am8wUwmxGwY6P2LF+4a5ZoOtmTH", + "tk7EKea3Lqlv7n9OlnJTYadND9DjSj/x0w12u5w63i4uOIPL4U/2EN8bgoK5ewawOMVPIam8FFzV1XKi", + "2sMXuhnCnKrRWd2rdno8Plarq5u9v1XmQnNaWaIeXyFr9ksnGpgNqysmLiuvhtwYR0INIKWezkr/HO89", + "4drup7099uinysmptE7mTTRePgN8AaPLx+n4AraCNmfjUl88ebKTqadDBuXvmgKLmK9sxGldchPq0IW7", + "ONtnOa9cDAjIFGNsNF90LQlYId86aLwcdFpeZOrZkO3paVU7gWA6lpUaLzJPuBVFqC0MZdbAh2Iz9dWQ", + "HdUnfg3QHYjrEfJPY9XaUJqNPYKKpf8eXSqPYazhchEL38J7WLS92SNWCYNawWO/YG+beKIUlp0jmDEO", + "MXhtAKQHQ4eiDwnDG6GoI1ez8C6bcNsgBlF8TSQkaRk/wZtwrCwKNkBQpDIVl5f8qehtBhRU/2Ht2MVE", + "5hO4YMnpK0IA8cMoy2bIMNh+2AZvXQkEzGndL59Lzp4d7EELDWJJCO1BHcnPQhqmL1TcBIzSwtUb4FsF", + "osoPoEbebID7cEEkafsxfLTjXDVU2BkRBARxq0Ibu7wj9IiOcawnxhNbYw5L4kajkj5/eAHeDI7AubTy", + "RJZQedygbZ/LikN4AJSPm7FKqILKut+i8XO3qxXYRFHDrW/gLvl1ba54qJPGX3FZ1kYktyU3hamWa20K", + "qbgDpCDbgnTDsn/g2QAOX4hc2gYuDeJT+Pjm716CgF0o9/kp0vxD8texvCLDZkGrWtQHuzaveSUln/2i", + "d8s64AsUPNe1Ilog5A+uitU5P27FAt4AeW4BG1luuuz6n79cOgUEzVX0SfMJ3PSBJq9MkvALdXO28wI9", + "LlpDVyh7N0mRt6szfrxdg/1K7SKchaiHbVAz/PRB/UaVw09csOvrg7fiSwkujU1xkaYOTzcXOXL8VCSr", + "+6LBFb/XPATGjYO9e+PzaumKI2XWj/pBtq6T+n8q5k5FRLz//ENhBC/WPxKH/u0v40D4od6L4xCGsqqK", + "IC/u5jzcmUD6ec4zAfA2kB6efymHMPX13shRtPxcoENzhaYLUenpioePvhTrK4x3pboSXqIo/AcxcbX2", + "FNIVWtpTQhw3SZ9bH+K/UYjosjzh+dlyuj2kNzZBuf0PXQHD6QivGzN8bw6CXzZRQMz3wxG4MohHlyUs", + "VbcNgdWxbvhIeLmwvvr05bol7q01cfRgRnyyBtO+NL+RAwEXuOsfiF/g9S/DoICxfglH4hcMZng4GJ98", + "MC6ILFefB3HphIKwCX8upLKOe2l9Rblxeutl/PiWSo/P97tOBfL4FWvmimlfc4GYsvPFZgFjQ6uXbuuD", + "CkWzl5dMXphId7hcWxtUV2WOrUxlDF3tRxCkDeuJXdu11vbM7w4gBKqu/Vm2PcsSuumd+7HsN8/t56d3", + "R4z+kzcekHnMbGBqRbGV86RAH0EqTH4GTM8iEtQVFHH1gd3C4pYg9+vO+zkqkHnPTu8GSl6HmX4ZhHQo", + "kiqyS5lIyI7o4CLtsqafQUFUJnS56vgCX3jg/8lNCi7Jip2jN5aIgIh7URiNeEFYjjZCcH7qXhq9IvD9", + "hdHVr54P+El2sYBNHvm5PlfcwBldLRES8NP1FIY1CAKhl5aTxEv1cLTbW4QrsuJk4wuMh/rKxQ1vmQZ0", + "A3tNY+Yn+uoL3be1TKnY9RvAbljHjoqfEN4Do9XttKXot6Y0+I0fR0xVXX4cMe/5V8+jcZq3zaXXO//v", + "KJv4Gvo9fvLZ1ELGweqTH8d+EN6+jZNHnb3hSo7JUXXVyduNgeTB6JnS192Hj8+/33JjxNmuWrg1nBjz", + "y/drlHMLm7W4OQdhS4TjBXd8mLgebwaDKsx6KRhVGILSjo11rYpOB8oiEcURfxpxbIWEq60P9K9r0cvP", + "MergTsim+8K1CYW4H3et1yDAkCeTEOIiETSl9+R0WrsWPST4DEvIYVp6MpjqQpQpa503xxHiw00Ee2t4", + "Iaki/ZvXrDKikLnThuXc8VIjWMSI2qVfR8NMHYQXMX2v1LwIyCmjrN7e/iqflseFNPBvsYWP/BbSA0TK", + "FZeVtt4UjEisVE4cwMH3GUwkDGVZabhmKL2NCtSxMELloululfGTrI8Rzkhx3llG+enNj28pE9pdhlWL", + "IGC8lEWCd2fZhTDIsQIyb3KhcktDDjczIRdWzAHp3dIw3k4Eq5XnS0WoU5uC/Zt4nqQFr0Ztr0AHhiJV", + "0lmIIxCq8D8hhD4vZp0aw7JzanvJuaeny/O5ETXW8RNIs24O+yNnhGBCWTE9KUWfykZDTf2jn988ZhrA", + "ZjFleZypseCuNp4fwRCG7A0wnM/gBJmKlR/AGPG/I9LSqMXTRhG8pgQjZtxIzUxVCU/CjGtpWBgr7XYn", + "dB0t26Zc/m3GcWee2vlhXMm8AhrL17d85pEVhST7R1NejrWZesLQxawfdnQARQYjcNXjL5Sh3qQievWQ", + "o1hqVNGk2vMtDeI5oV5qqtr4CDkBe7q9vb3NjL6wj39rPJ4Y8xImj+O9gXAEmBDq8LUpezu9LYhrIWXy", + "w0Lxd5mfMamc0QUVe3GaoDpAQcNl2j3YB4iG3/2O7QE2A3stTww3Ulj/mN6qjD6XhbBQqWVwKpTnwKJg", + "Ry9+sA3i3k+VULsH+1EJhkXd8c0M2JMn3+snT3bY6FS6SX0yzPV0CwFoixP6x9ap3qrOTrdscTbCT97O", + "KnEEc4JP/5Pe8y+wR6qaPsbXDmZuohW8Qpi2+MbB7GCfXjmsrVt8ITdQtVhqgO3YLUuckJeECLHNx1iL", + "7ExUDmCZZypvgEP8XN3E6Pp0wnJdCEYLE4BToN6QySfSidzVhpcAxHcuxUXkgwBsyI20WlHh6NoKlnMr", + "2GktCw6VG6xAgIu/TrlUEXcF+nn/aOJcZXe2aA2HUm8VOrePQVKS1XUqnJPq9DigMPV7l4NC2qrksx/x", + "je/xDXaUvqFzy8cD4OOIWfbX3i+AEXL04gdmJ7ouC7YP4/VifQpU8jeRu//V6/f+rC9Yodl+C2K9tXL+", + "rV8mnqnmXLF9NpaqwFUkzDr7v3rvF8tNvdFKeg7ol4pARGa6NoGc87K2TpjfW6piAe/hiW/IHF5hf4YX", + "Al4LfRg+i5UkIuUbwcuBk1MRGIhGWBSqpgv9TLgp7A5SHFXR8ES3CzWWitZbDXoML/GDQpwar335L460", + "7wdfbCEd1C6OdFyrPP2+VkmXe0ZCOSU2RkQGy/h4LHJEgMTmIPkRPwXNHD7DtgcXshDxU1o3Wnmo7uQ/", + "w/Vj+UTkZ3G9dtjo+5dv2RYO5R9wjg+Mngo3EbX1xrORuQ1v0Z8jr6JW2jj29bPtbd86lhYWtinpAYVi", + "AO3cREqaxhExO7NOTKmWcSkMTFSqseFY66o2hN4ZkLFwoilW3+KpCJTyZu6ljoMR6R2XQ479eQg7JQM1", + "zpDoufPvArS8HINoc4HukLaEhbJdrYPUzJ1IHZagWdvu00IwO4C87RlYx1kZsrf4MwKJwxsNthNQLBbZ", + "YdO6dLIqY6XmcKIQ3RKAflBis0KMpUKmFdcf1fpZJeDoFNzxOCAaHwqnJ08AfhWbevIE55nX1ulpGHkS", + "QJapXyayFCwUQuhj31Q+hxCFwxm24lx4NnwilBhLZ6Ns8oKGHfGxcHB6XioLZwYGmWtlpYXaDaEidlwe", + "/BoBtf4hCgbo71Kd+kZe1WXJ3opLh08ZbKufv6dLbaaocgDwbFUZXRnpN/d5Kc4Fm2JtHmr/uXCejI6E", + "Fya+aT/cAb/ghrYCiH3qpwmT9K+x/5+3L11u5EYTfBVEuSdMqUnqqrJ76HDMqEoqW2NdLcntmW06mGAm", + "RKaVBNhAUip2hfrnPsA+4j7JBr4DQCZTh9vt/VNFZSbuD999WFUpKkuoC7FUFsaE+t2+zz+vlF0LmjlM", + "BReOJ4HUj3VZkFUuSXdFx54ck7jmU06YBliwq6UupC0aX6dHGs8STiPLMs8cjvXnsRZi/Ia3ekLM4PjN", + "SMAr/xKitiuVPPNP8cvGQ//Yr8k/HL9B69n4TT99vQyphFstoW1Zt4bZ7BUVlo1e8ZOQaR+A33/81zEk", + "wxm/6Yvxmzu1fjC2GL/5OW342G/OwEtP/9IJzOtF9cKYnrN8YUwwqnQMCbYabNsxw8fXTfF1O8MK2Rdm", + "2nHoLx48Qdmqnhv79GKaLRp/Jn88NqBNFiwrXDaGh+otjS/DzpQ6r1aFGpR6IKuKwQgBsx8gJGwVDQ3/", + "PY71I1wtTOd3tNZyUeYJthK9rGtG2RYwM9kL883EQFzoak11vG5LVRVMAwqP5LO4x5noWZWbxUJB7SOP", + "/hLctPXCcFhvbEAY1bNfPBguCTLP9W4r9alkdslV5kFZ6PjkFgV4xGuq6HOzBIu64CyBe3WOBVrIaQES", + "IA5QtdUFXBmiNo2NGsv2Kw20sADS4rui/tmgXxj9pZc05sqWtcieOPwMZa6l9KyD7+XQX0Ogq9TRaHOO", + "dFVpitlTdzTcEzEcDh8zwvWE0qNHOj7/Ipki3NxsrGEufirUBkgeXhZHe7/2Wz03D8jTwVHiQQ7FGfMY", + "yClQhaXAecBXgTwG+n2N1eR8G5DynkW6P3sIwljIhD75d9g9dtmcyMu9NlF5OkQGvXqeMhM9//EWkOT4", + "eEjtSOr9yZ8PUvGBdIO1WQ1etzA/BWw3kW6yNqsJHusLs+mnU5nsz6xcZM1nB/SsNe0JnN1kadVt+Ylw", + "C3BSH8xiSqVixNWqUnxvMj8eGo8yT38yLIiyqldwcdUnz5KW90r08rkxTkF5ROnE0pYLaRGUtlCkiA+c", + "6GG/fep0CyTKKcjWU0BBmPFF2lJ6lraX8X73Rba5XxkioY4X4o8iNPXIxzxQ32ZVN+YoeqAnkUXhaM2g", + "YGjflVBYNaliGcsdyJw47gGB+w5Oaazd3Nh6LnUxFMef/Gcul5W0oHcHLh9zjcI9RHlgquC60S0TpRtr", + "7MwjQY8lCwOo0Syxpi59pqm4jZ8uJHwdYHkaLynImRpr1oCLKUj/rGjKjC2UnUzX2XATT0DfGcp4aZHh", + "xtUGdEm4HVYTMvTUUItZ4bqNrYdjfa1qkfnfHm8zkQCrB51WX+jVQtky39GrxVTZHSqb1R/rqTGVknrH", + "/9/3WFl5GX/H/9jxv1wtF0uwpVSlvuPpDRm5YblNh/WOSzZ4hf2iVLTfiKww+QT/gHqZUAYZKVy1hq2X", + "YrmaVmXuTwaUMVgIk/cDwGqxcjWVNINzIfqy/tI12GsYZ6x9Ew8htL/8ccCeopZ3vg+Vq0JBXeV7ZQNJ", + "rNViWYF6zItYSkcczCw53b5C5RVozLywJxf+PRbigPNcynreR2vOWGtP5iuQlJbztQMFRZgNLG6qqIp4", + "LqthlzxAwf4TWTckgSe54PEblLXwpT9WUOIk75uAuUEPfRNCrQRhzIgga9XHebUlhGen1DWk3mRZCavH", + "V3ghNiSbQHZaUw/Pn5z9s4zitZf5oawqKeSEnJp70r1k8SAIocMOBFLWx5TDCkoFVGt/YYTUslr7wyc6", + "wMhIaAWQV1qV19Va8FSHY31YQCXcjdtdG8QdpQNND9cM9reoBVhjjTmTh+LCw3jM5AxVxKHWbpoMbCqt", + "LZVNi6tTzuKxXsg11iVOEFyYdUAJMeVzbqoE0CGrmJx5ojbWt6V1NehO/d7dQhrmvAJ3CjBMYI3iUvM0", + "vmGto1XSs7uArsf6QdqFH2a10FgFHjZqUpW3Kl/nlZqAm34m/MyR2IiM7muRJVXlx5pmDbHwqzxXqnAd", + "uBurF4/1e8Sa/hiIXY4sNJbSgbMdNe5v141oCjiF+hSEigiH7Sk0OGHWOQ0qda8qInrmlmALKSJwmdAK", + "iaNCNlRWIpsgNw0f+4mDfgevJbFgo00c9CtFMv8rUcuBjMarA7HJ34YBpjHvZGj64gWOpS8yT52yLYEq", + "IZgV1WI2MAIQC2RQcHE9T3OyYdJvwvpFjs9Y/1eDzwuM1NawKRzcAENKBwaKL3oSGL8BKMUGwGrjMgB0", + "0Cxg7pQmVRQVkZbV2pUOPQkAqlDd4/oi2U4P97ErsjcOYVBkMZ8b9Pubs1NRyxnIJ0TZeTR41+gv8H4D", + "5LsGmLadZRvYUsZxW7GjkyPX94NAJvxazYwt/Rpuy6pW1g2f5DYHpPFLhYBILmFDVrVhDDHs4PB/Azcf", + "BsL5AXiJgfjx6nQHGCGCItoSXrbdCkjjHDkuGIXRRYQH4sd8l+crrI3YI6YMdc63lZEexMIugsOA63sw", + "zv32udxYRbMjJs73dmNXagcQCPFeYUJHnqW7KRcqmQbze9CSuT3iO5OhkdTtcPkmwO59oe4VJpbH/r9T", + "xi0l1A+LA8yUAfOFH+BU1mW9KtROZfQMfgl4FweqqC6thxTOd81rnCnj5hLh4gOc+ScxUwZqc5a5gHex", + "JyiEKhHQrJqRqhwnek3IL85SLaYKCK3v/C/gFyTCMwJs1PmDOgd01MnRuHJRVtJ68olwTKdSmanv8H2p", + "Pd4BZXraVdJDuZAzuhP+vym2iJcP0AyIlsEXgCoTbMhUZKhPlDyIUxEz3ipr2cEpZZkzuHKs7dokVJn4", + "v//7/4gs0SFsfIqihf/UI80xl+ZN2xLUdzYnGE4/D49+DmqYVW0Wsi5z8RFZwahtl/wKJAoQPdGCxlKU", + "AX1ZYN5Jn07WymwShJ1se3sElwWMnZE0Rrafv3TcFl0hSuWgLerKGEMs5NLTY1TzBHpMzhMC1SG++8OT", + "xK8gdEjrhtWKc7nwDMYHo+9ReILXTeEkHnYCAWzwIgQZtYstfRISBbe6vS0/Kd6ay5RugqoUiZk/ZdQ3", + "gLLNCWPLWRk7BS856OEHQpJEOEGR85Ty57pb7/OvUtAcIyv/q5RZz6iVaEdJueMZH1IqRYIQX4WVdpCN", + "5CtaoOjtD4AnEG6O2r3Ojw/o44PXfNykcD1VzJTQA9+DwIdpMwC898rV4tLKvPZUB9WNoFklCQZqlbS5", + "j4Rf48KNeY1lU0AgGZCCjKEVdB2sswZjW6lnowTbtHWLisx0U1PPxW3gbtApGxQjpCkaiMOi6OYu2vxD", + "4k9AfgGwTuShgPVoM0Vwjz3fBMyNKLDED8sBvodr1PY+p8cG5L/00p1usGe3Ab998YW4jBYCf/tdWaSm", + "yC69IiG9ubxXYl7O5spGASU3rubqFxFWXemaTB514VHJrQQ7ftBWo8ZMf1kH0zhXDoVDJqrateuAl1O4", + "C7Py0uEcSh01CTIsCklpMqMmtgfFaXDchTWlx9DSqS+VDQo+f4EdCW/ClX9HeTRMClQleAZHjP5vbk5F", + "zzNLgxszOC3vseYRe93hfrg4QSx7Fdl6yEsLwWq3qZvBrd9gGZE2ZOsmT6qj+NnG0qmwK9ZPncr8bmah", + "BjzVVsNi/SsNDgFUI+pK3taigipTgSfCOuYf0kQF/lVI1QCrBskcUB66F6DWAb0WPAtRV5lYmqrM152m", + "bCAIyFVQ1S9S7ozf1HVF2hteNn64/3bO1tIXTOHU5e9vCl85ZSdl8UrbLujP5JNGptQ++grD5ffmQZQ1", + "1H5yo+1tD9sRNrC8GkFSEPIbsBRcFUsbuRhCdU6kPBBdkOlaFOpWrqoaVfoNBgvqXYlSO79XRgOleb8J", + "gXalnZcV7Foc7AqncqML1wGLvvkxlYgrGhCPkiJfb08gSuVEuVioooS4G9GjEjg0JkzliC4a9jGlqtq9", + "vd3d3bR/yKZeLhQapWYmeDTC1HKjndJuFeT6D+i14a/DlSJWHNkz9tuS7NlRNwWqtCBxutUv3BWUsjZv", + "CsF98758XaSKX1auNrXGqdb22SsFI//+F6qt0X5CTfaEGrt93WDSE+7iJYeFplMCW3OIFUwn9hq3gits", + "DnBFtzOBFuK0Scm/tMp5YlLqhlTikFyRIUCLq48fDg4O/l3g4kVPDWfDvsj2d/ffDXb3Brt7N3v7o93d", + "0e7u/0KzXUQHbJeL5qyHskJ/SBGVnUE5wBjiI26zHms/aYYuB6papHFfovVmpuwg+KonGAbaD8U1m/TG", + "eqVLusiZdp5xX8G/C/gXf4Kqj/n3guTQg13nxeeINODpu4V/+E4sSr2qidPYfzv3D/ffirlZWXz2Neip", + "vhaFXDsIEZJkX9z76k9z5Ir9L//RnnhQ6o63wt/hD9Iht+sFEsyMGp0g4WzbF/YpCrY3J/CNl7GSrp5I", + "CKbzsJZAEVc3ZVy+B+shlJ412mVDQcHsYKAMqMZorBqJ33mWDRzMsIQjEUmqanmjPOPpRbsPEjjzX7Uw", + "JM1x7r4PJZSuATnTAvhEaAmBVtAMTs1MXBn05P5Vgx/sNjfu1My6mCOsAVDQ6Ae7AAyB52lw1XPphRxl", + "oS4oHX30a7xGgy/dan8zohIA/UbTMFDRy0Z1RnK0uFNrhyL+RW9vi8qDAoyeG9TBFMopW8qKtb9aqQKp", + "NrDrREddLjU0+8fe7u7Ak7FPgTGfSw3vgc3zAlHUVkxNUeI1OSt1uZAVW68Dxy16/zjYFdN1TbwxN93C", + "XUBPyY9AgxlIBoI8Tu7Umgue1oZ9oSPD2/vHosytofu7RYsu0bhvNHlWVhK8S6HbXFaoIVmUVQV3IeWT", + "cUZcH/e9msv70lia0lUno9EL2bewPOlAnAK7MQDDQHDqFL2lRVIrihUG6mCRTWjyboDdiZmVufK7VJpC", + "KHKTTSqBQnSssIp6KJAnQtajCAxJYELclqBh4TQe5qpacIAKunQzsDa9zz0EJuVXMXRxNmNXDwYZdjen", + "W3Jy/vFCYIyDH8R3kooJdV1N+Jp9u/82dDMBu/69rL492HXcC5yBKtK6vpGpAl31t2/3RdLdu4U/1lpW", + "E/r+272Dg68bRq4zKHDKU9sQRra3yRoK98+0rJ8EBAlSdImzbVM8AcUARLFhgLORZKZtawldk6bKCuKf", + "worBe0faO3LkCxxpOGpmDBbmnibO10fVNKPaiEyvwJI3gEiJ5PosrfFYHBz8a4NI49Kq+9KsXLXu2Hi0", + "cdItmUs9gzK3tIfNHfKjD/lVhq54D5GOS9qelMvu2pzjOFurclnlqwoILUqF8YgSnSke9mm5KOtUieHv", + "tcnvIMZobk2whTFXtr2NTn3nN5egeVnrXOS+hWO3dI43gFgT7hLhm1AM+Hhv7BqTCQ8ppRb/aGAP6oDv", + "AMVTQcgA2Vwb6f3APf3mNOxfe8/YAou+CNhXlBOAefJdMA/IDKCx/PNcapNGckBHz8dxwGRfE8VxZMQJ", + "UB8wbGMMQ4giIBGLaLiHK15TM+CpYMuxyOUSw2x4I3DVHATypWvEgORKTFX9oJQWQZvHelJQ3HRGeBAR", + "R+tKghJR0eaniZHfOG1HBa4cR4mLhbIzBcDUH+upzO9Wyx2rgKT3WRdU3pfFSlYJsQuhHxgNetEIDTi+", + "vS3zUukaPIL9sJCYCEs6bXgRAURI4ZCeUtww+j7BzMc6xGKh5xOpPWlBwcBwp9bknbaUpXVbUdmknOgF", + "vwDgRnj2R6WXiaZQujpJn90d+QilrclBAEKzyI8kTXYelEFJuVq07I/1/mA5x+I1kFG9t3/5wVM/U5vc", + "VEMBZowQpx/IsltK7eKuYUwYWzTGuq0HhDrcTpQ1JG4VRbLAzbLuG+qUvaE4I2/6GD8LRknlxPenHxI+", + "G+KCVAVOy2ltXpgglWtPKvYih9AsjZODZ15Seo84CXKOwwLuh6gYrCp+yu4p/cawuKmNEcb67VBcpr1L", + "jOBGnZBkPIu0pDbCKmeq+2T4d0NxxcUAK2OWgeHBTshWG1yFwCBMvHY6N4qjwz3/iHGfLCBvbweTnkd7", + "5ya4phxengTULwbilxX5C9V8MUJoHPUD8MnhhsySIWGAOEWMJiToMxYrl4razFQ9R+2Tp9TopTStwOz0", + "YXMVuItzqYtKFVDk3aZbRGQnoADfyz/2dxeOKVDiZYOXqG7du+3tRC5pMNvUADVZbiQwGYV4FzsHlcMT", + "fbfmAfEAaMaAc18Bh8B9JhTwwVhXDyAit7exXBIRvltJK3WtwrEmR0DgCjvO+oceIpOyXm+hNWhjm0Xp", + "eCQgXj2tHkhPCKndFm0oxACHwou0rUWJXmTom4ALVlJ5q2gVQfBPGSWPzqxCngavbIAqgihSyqycsh6d", + "3ZaVEn8EexpSNZwa2CAG5KdHqBsce0EjxJuUQuP7VXVHtiFHNwFwa1UNjB1o43mVmXBqITWLrMBWAVk7", + "A7LWO/KY7Hqtc1jideCsOKpbLdk23AjuVp/IkRcxoVnZXDlRlXdKXM/Nsrxd98f60rh6Zj0pvT4ARymp", + "0XUx7BS1HIrDyhlxp82DFtKNqNcmm9cfa//Y0wqkxmicgUDvip6oAQTO2Bw/ASz+3tNCHAfWRMthPRvN", + "Z7qO7rl3aj1k2d+fgVVq8CDXyAmMxppPx0McnXhfoOuLE+hLWcQXTGEDbExBrweaalr+08QGReuw27QM", + "OIKlnHEEQS8u4k75G7M/FNfK0x98voQQV9PkaCJyPAiemfDGjcRqiStkJeTGYuj7rjUtwe/9racMSyVr", + "QGfas2swi4HQxJmD42VE4MzZ+a9c13UjXwOCqZ2z9fWfT8Hyf318evzhRmyLj1cXZ+xk78TF1dHxlXj/", + "P6IsxOnJ2ckNJOAQFx8/Xh/fiN1srIUYAOinjvlH7/1G0fL81OfrqS2LYC/2ba5WmgR8jqeam5Wt1juF", + "LKv1Flp9ZbgxIQQHL4anWzBvCPCWxaLUO3JZ7uzv7r8d7O7t8AKGvzij/wPcaL8tC49297+qvEj07f67", + "xuwToOesZhQ+ah1U7mU4wTa0tqnKzSIEw2IeNhwZjZth1gcCcMSpvFMYeA8gBWopcEYSvczfy53d3d09", + "mHPWF+HJPj8ZDodbOD4yxIi2EGLgtj9Iq+bG02/1CTAafn2JAi4NVWoBG9IXTkEyBOJ4ALSma4I9nLif", + "9dSW+Z3rgJJCVbWcILINgOK59wgpjT1uTZHCsQOk4JeUTbgyDwOm5STnQNxAbI6hbR7E33OcNTM71/5W", + "VMo5YnbwEJ3HnqA7o1D/gSdfteQd9IgV1+77iEQODkzeKmTeIO1peYuSql0tSfvUYkaua6vkIkm70fdX", + "1kpd+IFQaYz6R1zCT4dX5yfn3+F8ayCX5I6o85WF0H1EKwiVfisqCtmwHlV6SYnYF/QjR/TguTujq3WU", + "o7zkBRj8CqUvMD9j9BuKZXAg99KWZuWCDpPS05AHjgej0c7OzlLW853a7GBD8D00kAzCgxmp1QYicwej", + "nZ3pKr9TNTTxHx4u5N+NFtcHfnyaSipXsss8wQicDw4jnJZLNzfBJVL84CGO1b1jfQSe8omgRrIk7Cfl", + "2SMPEpY4o5CIFKy0YqXLv62QiqVqgHb5nU0tANzzi9YXT6dyeEJ0Aobwzl9VvBsN2d+FywT7wtgcMrEh", + "enxZ9m8JgOgNm5K2xojTCDcktUctQ4e24BgFxGBDJk6ukX/htuUt3adMcZtunv2xjiQEkhhIz5lT3B3o", + "Hjx3AhfjSuXl0gIUXkl9Jz6uwKrUu7r6GMRxZAcabuTgug2+5NdEosBUT1ge1edurWv5CUOJzYOyt6tK", + "JPMv9QxuBxipm0kNSs8NZFNTrEceIaxqZUETyP7KeArG+q8Oz4886r+48v+eX9zAh1f+iiddrZW0I0yI", + "tb+7v4vpT+bWM3Xxo/EbdNJawovxm6D/vSZeNqz1XGLaoErq2UrGkeikNs5lhKpVCOyLbHpwgwheRZzA", + "A5ZAubGYkQaST102VZ3X7NsTtBGxQ0zwxwv5HkECZ4JnPNbk8+ZaDms8LhMTXFo3uHTEYvjOJr6zCfEw", + "aC4DyGDH2uR4ozcNj5u0w4QSnlhIVDdUVTkDX2P2PuB9SyI9Jr7/ifbyXCOyg/xWyWoESz3G+FZUi1+h", + "ehHzdBP1COB7G6xN4SLAE3EJbooUN4JpzRIk+TA3DlAjxDXbmnX/AejRBaAVloODkVsk7oMX5kZgYYS1", + "MIyAyQi6B1mIvtvbP0AbL/z19t1X4zetWcPFHuvD5bKCVPWN64sVhg7PjzxNRcfc7gmGM22e7x8pqGI9", + "qlU+16Yys7X4Y+smjt888mKiOQOEJIzhAERVQ7xlyWY5CEPABcCDYDgN58gLO0b/u+Qk2NlzY7W984ub", + "uNCt1koV9/zUYsNaC7W0Ckxs4uKKgsNGgO7uwaieOmakht5zUytWElEEChqOMakha7ldbgLYWqnvSj3r", + "U4oZ/xjXkiRuYBR+dfid6F0hLZfV4HA18/uhCvFdSI22hX7ILUKkC86epqL3NyGD09MzZHA2MZDlobgn", + "+Ax4PBfulnQCqwMOrj0Ddoxmzt719fFWigEX0t4VXkaP48P9KbUnvyKP5prgwBvHAIIfvdCBoPKehDke", + "ziBfw1ifBORS015aswJdC97Y4DyJL2NeOdiGD5V0zqN0F/fPCYAThz4CnuNgp21/Gwe8/54ZWdZoBkd8", + "28kKpNsSvfJDelPR88Q2cE1bIg0PODmi9xRhjvofPpOc5o46qD5bOFyfJ9JnZsZvK+3gSQA8IilXnNyo", + "S1PPWZ4AHXhmESMAMT9STI/kBxB/W4G7c0Cze8M28WJX7ciJ/DEQ6i30eAPkkbUpEbnutuhMFnRcQ8/p", + "I3xahawdmQ09wn2aW/KLmcoKbjPbhxJ0k1LUwBgE1LXvR6ULHRAy6WOVzk2hLOXtRpkKoqwCkGOMIvst", + "DMKRh1xTQz913z2jlQVwpHm+smiVlnqDaUiytEAIKym5FxAnSxbnQXCB5KmQ1rE2S/EOvEG2OsPFO6n8", + "DONIKrkEozITd1A/+C/2dumJpbU03AIJzixxDCghR69DWBS+W5SfVDH1CxjIcmfxaSrLAXYJNqHB/V63", + "t2IjWDR8kEtdQGDxBNwLcKK7G354/pQPGLbEpV1pOGqixGZVgwRPcB9Pdi0KVVOCQDz0mVyC5ELpCMSi", + "1OVitQCNoZubquhO2NW54wtwsFKiUtLqGAvvN3OlW/u7KPUEZjABlOff7Q7fxf2Vn+j1TC4nS2Vz8tI8", + "2B1u7sZAZK3+spH4Qakl8jK8/EDuwBnX1eK//02YWzRUhzpY2FvX8NlIXPvvMKcsOs3mq7q8V3EvhfoE", + "UdWNrkG3i1fMSxJ8fm+HpMC+rv29mQFd+4ApUebm4SkO2ib4hHOfoGLA2ttM9MiVeWv0lDQ2YMRS9FFR", + "Kx4UWN7Ewrga0pKic6F1t5nvpZKwyGtYQugkpHlAuv6gytmcNlm5mPOwWvN637EqOPDLiFizlPPLOLop", + "4VbBfWRpFeGHFveScCxjDUg64ViIzHCSvCmFEoB0ZOl5nXB0SlpS12xvN6KC0FBG8TyR5HgKyKSWDmYE", + "Su/ryJk3tK+iF4/1j+FQUd2NiJr5tdoILa01D6hDMOjzcoDhlIyzwMM8wK4DnTV2g3cOEfzCk0SzqqtS", + "WZfqU2DLn1eocArBFzUqzys8nhUHo7JDudZ2oVrh6iNAajNVZpvU0yG/rHzx7Cvq4BosW6cuhZRzSSJa", + "kg4xnuv1WhTswrOOC1NAoohG2CvmOgtakXagTf1gQmg/hvGAs0jgaCJfEkLhE30K9j7W78/231EKgvbM", + "iQmHoN9hjEAFPQGmeET/mHZWy2ZoJd2cDy1foQ6X1+CvHiYeqEUaQQOxVWXxKbxcqMXEI/WQQaLpnX6B", + "2QI4ExV/nYkepw7dGomTW8gv0uesvrihJdRkM3aNRKO3cgqEdGNFrcC/a6tp3KdB/Kay9CRkNTO2rOeL", + "p3VRooeBilEZtdWpjRK9TW3U1oY6SvQ21FFbm/oo0dvUR22FkBg82i/dZph1PGEOmSShAXK+uBdC/jbT", + "bARwjXHnSTD6dbw0BLCHy6U1n8qFv4J3A62k9ThZe1IzhS06/OH8fIszEgIbs8EVNwH8vh33PtyUnlsH", + "/L20xYO0aiDzXFUkJRWlq+Fj9gZMHIOuT86OIA7GrvIgS37601c7h2dHI3H4l//eRyPuX/578G5vH+kH", + "GYgRrdAk01ysA3F4dTYS58cX59D4+uxY9K5zWZHXW23LTzFx35YIcwVFlAea92X9Zy8J6Zp9DlEjX6xy", + "VTDo3xpTLy24YUHGGvA0jHGm50Z8d/lj6rFigve47+3D5Y+EXwq1rMw6cWfuiL17ATPEM+pADeFlAzs8", + "y15zo2c5fVNVciG7OX1ZVQPPH1eL5P3K0tt5XS9HOzuV57fmxtWjvb23B28pEqmJpRDZn5mCYMw/Q8fF", + "gskLhKCzzZwzcPY5Tg3uLtEI86BdIm5FLUKIsnEcoItJ2SCmnRN/ZZtf8S6h7ltBmP9C6hVk0kkvLvm+", + "YLkXmdhct7eP2drfvRxI7wSL+B8I/YhFI16xnJj4ISypnywI09k0FnGkNNhkm3MipxTuqSgXeHmgCaIk", + "5LL9OmulY+oqDp5DxozE4+be3HKwXDxZPHVM6cAECqc/otQHtcGVEx75r+uL80tZz9n2wYFRLeDO4FGo", + "WEaMBT//w3DllB1OvYxFSQ/DVo3E9+DiNZWe36SniLoxpGtpzWJZtw4h7u1IHNPPpnswdMGNAtKlZCxh", + "m0fiYskJGsB1VLvAXX3TyuIBCcp4rPJWmIU/kqJJ5EE+42zZLXreAsuus2iB58ZsrxjrbcwW7axLaR2n", + "LkMnbnzEyw81ZQL8NECmNUOEWcrk8FswJV7dCUy4gSx5vZwrjqMleckgbv/pbRN1hfld49r+dRPEzXrN", + "DPFLftic30+2BJA7Q4R1HAamUz57BpH5uw68H7og00m34SI9dDwkPl9I9AY5esH+N0lXPRKfO89iJP66", + "O9zri93hvv/n4GfxmGFShQbogLrM3Ck9QDnbc9wvjZJuKL74GtRJw3d9MX7zdt//sT/cFY80ZMghSU7c", + "wPZj1MavWY9vQD1e+FsKmf2Stn9F70skFsC1/pxhUj9tBmb5jfAzMOj5yFPBEgMQgIY7QlJTQD+XRMSB", + "2WiQUgrlh4x3JExReRiPH5Hcw6+l0rLk6MypKqzJ77KxTnH9cmWXxiU58zPAIzwLZG6yobio5+gmiXOi", + "aIixnloDvp1ZqwXaZkL9D4zZ8hO2tzJXbPGzKw3JenKpx5qcwEGHGvS+xBEk2ZPaV/IZtWaD1Xl/eHiy", + "M52pgVt4rkfpwf3e8J1nZpo6yAvYP9FDzxOsMbf10rANHut5DutV/BVP6dIsoWgeapxHIovd+RMG6+0A", + "AHHAmfdQXYvPoNJVXNlS6cOTF5cCUNO1FEj3F6B+cIA7Gb6Uy3Jyp8je5+4Gw+EwXcoZL6G7m0z09t4d", + "fFVs9Tu+wHWI3sHu1/tdX8hCDnZ390MfAXp+uhbvEe5fWjZdj84jBNeiYV3WUiebHdXg4zeYqoxt0AMl", + "XT3Y61z90535w3vy7b5/m5u5smqIz5WeVaWbD+4PNl6BTqYq9WwlK/+eC+FEVc1JyLIy1lmkKPCYb3DA", + "WJS4V4qMDIx/VzYbio+eSX4wA1eDi1uiCOI0LP1oGpXg2QKuUZHRw0TNMjKcxGKGiCXK5iLRqZAymqC1", + "O8WHcia9bIosO+VjP7lFBJUy4hwGCAw5xtd7uk8UqU9pTZ8gj2MNOmEMnZmrRTfLE3K+Ees5eA+y4yVw", + "nYDKMbVA+XfUiDcTZuRG3ytw5OQUP7QVUXpAPrqDye22ajzLsLT5oteJnDwgfnGJ3qkj8fkz6IAeH8dj", + "/YGt/eLzZ7b8w4uj2J9/l3T/+Ph7yLQdkiqfjLgGh8Lo4gkSyfZ2lwThtzwULJuH1784KAc3W5WF2qEA", + "QwrJh3BCPpmOCPdcQmgxegQngc6YpYhk4lFMl8C9e0noU93s3apclfd4MZox5NKhUvZTnWoscKW08Ozz", + "55B37fExG4kTjHxDHRdl5IbPvigpO+3j43A4/Px5p7yFBh/YKURWojKzMufvMZ+B5x65hX+Cg9SEFoC3", + "5AYrXSnnopcJN8PnNLt7ZSEYLQxKrf8Tbunjo8eGnz//551ah9+QtDj8VUn4YyROjVlibjt2etjefr8q", + "q3pQavG9qpbKso/73lBsb7vcrqbf14tqe1sMxBXaIBCAd1y9BveJmaNsZrWVeY2R2FSiwQt939+cnQLI", + "ZlkW4QiefP4c+hfzelFNSOp9fOQG8D8P7LjOLU4A2U4yNdELPyV+DiVxsP1hUUCQRAV+IBj2NYUIXlVh", + "EhTRW/ZFUd73xXxvMP+qL6qyL1Sdo6+4iC4Wy0qWtDz0A7MKwu8KSPccSrWBKWh7W/0NNu6YbbjRl5k9", + "k/lI3ZN75IGwp/7GWaXHbzACfPxm6/HxEH4SYLa+9/AAchYEqcHnnE4UYDptRajiwM8Zgqhh2t8p/QPw", + "8TWpEOAVaVD89YNighvWkCdXgs1XtvoWkogeyVr+eHUSJh5f1/PSDeGbycpWHR8gI+lREyU8B6wELYa/", + "LGfoVLXRBpyoOdqEvamx0VI/1ajtqL05CG2eEB6lhvIQP16dYsZSihoBKIJwhBFkWpVOffVWoCMH1pgR", + "P16dBG8J/BIG2/llqWbfTKFBfzgcZgyTGXHTmdjB3w7+GIif1NSP79ouSiEb4Hqp2IkAozFEdEnv8kGH", + "2gB8m9AZHZIeH0BeHVlDOkR2dO+hcWlEH7Y3/E6tscYGbFhMk/qBJncZMgnEfdvePoG0s5Dv1DxoLH7d", + "F1Y5UFv3ylsKBdjqN1mKsLGhp8ujjw6pyaeasRYG/WGmdwjhsEoX4MQiHWa8Dc09PvPNr+C6ewzY7Ae5", + "lTPz97KqJH0VkALCCNWXhRVbUwX44JUh9Vxa4w8Io6n45Lg2Lbt6eBYIURzVOPFClXiYl7WqSlfTy0tb", + "3nvac3KJaA+Ie0gicn199VHIupb5nWPQ4qlgokFwt3EJkd7b3T17z99+f3NzKYowdxBvzaqOqWw4PjOm", + "aPtG/F3ZkJCebLxKFh5BU69w4CLwa5tz2N99+6flpz7WuSVAYLi6Vmokusut7jSUnF/whg4GuUHFw1tA", + "3HAvb4zRiMDhz1AJ8ebi4jwkuroBjc6FLdF7klLqnlOuoK0nsWEcAoodDzFlZMBB3a9FpfSsnp9Je6fs", + "t5g823MBuv727UtNCwV7qOy34zfjcd2Jv2BlHym3Q/TT294+2B18tftvqL5CexMaf4ik4WX7r+uLcwbI", + "D1QXh9TeeC2yERoVWxkAGIYh8h9XiMlKwJ3fE/K/fnHw80jIsk+G+EXFvMENFdmn4/C9r3QZEy2EQl48", + "ykXD2HV6eiaW0jqSDIUgfXa4WFm645noTY2ptkaQrPULTkXrRTqYO2Y+T4DUS1MBXeNBZZA+fWsEEinV", + "BhZuKXO4lRG8Q7NwbpnooVf4FtsaKJGTsWi/oa1AFhOM3yKjs874C8d3hJS+/qouV/WowXKhbVuctEpS", + "s4k3hCAKLLg3Ev8ltRJHBi9v94FtwFpISpgQSIC/60ZR6nCPk3LUtTF6gCcOv6n1d0ac+DWFmsudjWW1", + "vFPurtQ7M4ONx/onSkEng3AJlXCCtQ2BtQQdbhFsSGClqjGXWGkTM1LpBOfJUsWwkRn7GCmF31Ggzmwf", + "4urGPKI/2IUCyy2rPNumJExYjN3pGSXRR66bytrFFOLhFvx6mTkRjoN96mXD7FPi69OSa0tqDaslQA0m", + "nlC3A0oBmCXltgkF4TIMbyBrmQiF+yjLOdCLpVVL5WlS9odhJkqsNkhBxlzZ8A/DDnOcSFTs8ejoeyjT", + "99fdn4doF8kCTisT5dMNlOlhdgEBi2PAJYV9DDlzNdn4SFrAUgIQ4hYceDAiM4RxNcvvxUraDcse72iI", + "ZioBEwBHFFRj/zyUhN363bUb56ZWo9TA528emRXRobGsKkxQ2TZEpnpl8qPuHfzpbayK0dwNvKycJJP1", + "Qb9B+bT37uCrTvVS0Cr9NnVRqtN+pVa700H5MlIA1KaLa1Wvlv+ipSdQQxV4ftcVhm8bmvs/fL64PD4/", + "PJkcXp5Mfjj+n8fOrehMML+9Tb6+kVF9KJ2qIH3X95jNPLxywd0R3ezzNTjUh9TeJEDtkE8ZZD+nLC1Q", + "ZT2ovzoKlKaurS0cgDrnhZIeQdyuKnIIoJ5R1Lt34kNlVpC5jMxBaCC/V5VZehKys/RCRL7ui9x/mBjG", + "IJozQMmOyyWnCyMmJSXnENjdcmUBX+yo7w0Zpm7Zn0Lem7IQD3NTqRjJ0EhEGaNPkPr7fg7TlPnXwXXt", + "BgJM8LNGWv2dzUKezUz75OSzNxRHGBGYpu3bTCEPYnZnrtBncxnTVv7+2YzZeP87FMrurEHxVLXlV+Qr", + "3ueSSYnLK9DUzu1NQj+TpOvRC/WJGn6JE+vTMzlg/3c4dC8MDEgI4AKgdT7vnFV3+CvjLt94Ao1b80si", + "Ciu5bNQYbFakCVGhnVUI/8rn/fNTawPms6sQAsAyF2SJRTsKln7capqW/8QoTa6yUOrlqh5rCZyzxXL1", + "aEgjzV2fF8YFGfDv2iwRlCq59FgdtbEhTC8Yyo4og8VPfui5XC6VdohQ1mYVEgRabPAfbFxHYdzxc8xj", + "irYtSDYmlLZlPvd3tN/O3EZpFZLYbE6UGGsaDMX3yqov/UxkjQlAlyqvcWJxUEwkJHpUUoHRn1YPZOaD", + "ZBuUG7Oht9gKTuOH3OpcPZBHVG3EMWezvMHkjcEZvWPlzeU9tQ8R+XnRF2uoJhdwKt18rC8vrm/EDgYm", + "7nyG/8HKskM7tfMZfsCz1zEODUZgOBxuIobjMMVw/o0VkdUn7EjT/ogLxGJZzTPkPKOJX1H8OKYabafp", + "TRKhPpGmF+JCIf0w1fJAxb6n0Zg51/cxs8q5gHLOWi+auw75e1676bR1p8bcCVmLDFKDTXAOVAMnbiiP", + "B8mW4csYWW2tsWg45FKShwAy8TxCyacuT1DRHAnqavbb91P5QRymMoZ8xSNOU1vbdTup8kdZVqpIOu0u", + "QlLbks2Q6hPmhPeIyh+8ub195jhjSsHWgXL6a+wbc83GlXlpqNQhzgi8ipJYpXsFqXCcWSjYBHI3DTmU", + "YVlJGsIOyOllmxDQPv3HDBN7w46Sngpz9JqFqudq5TgFM+yKU+3DWUHkG6Wxp61uZ/NNczjdl7JNrEFN", + "H7waii2OM1ksjOZliutcaWlL07BDRjHs8PJE/KiDm1VQHCUbTsfQOnhAvU7Z+zJXnK/PBW13ax1cQa/H", + "pQkiZtrqjBmijkBq97dgAjqJUQqMHADH3H0Kp1wTBDI+QkVpouGHIsl9SVy2iqkHLTh2u1gqdxjAhew7", + "IaW8pB4D/zyXnGISA8ZrWHhNUVKQGdzYWmThm4lVS1naCQcOZMIZ6jTGu+TSEzj/HSn1bXmvCsyxNgx2", + "0St/dSCtMjgOsP432ILoMlLgX23XQamMFaVInQxuZCyvBDkBzOxoLmCVr41Clgegv61MTWx7RE1BumRD", + "KDgwIZ6DkpdQigIZmn5qF9kKCzhNEpsnQfaYcVvfK1eXM5mY3K4IUl1tlnRMC/lJyNqLenUbOkMNek+q", + "QmFUgM4GyoXaL8w1IEewkaD8xlB+QPavTD/A4Mcja5ZUU5ooJxJ8kR0dnx7fHL+IcOCsrxRyjrgpnrPB", + "oUYi62AWujo5GCa0GwwwbgOnD5gyI0oBtdD2dlJNljm1XhJjt8VMUCNzuNHR2RRq26ZDjTUn1UJ0gvIf", + "7iRF0ctCDVlRHkdN/KW26LjaaW03mchY8QoyR71HJHEmfzE2YYH9V56fBo9B2CGnatfnfFOEWiK/N3oN", + "v4aNA4uGf1ItqfEb7HNAfQ6oxkuMzqfaqCRDHox2dhZrsv5SgrSQXLDtjSq+s3JJsY4UlHxczMBJyH+A", + "b+O+Qk1V2FK4EJD+tDTazcslKvXYma620iP8mH1kKE7LuyQPTF9AX1DxzY01V4vrLI7GPPJSWSyFjQUK", + "0jsUa83G9YiLe09/1MPmSjAfGNoVPA/jJxFCXZM5wjo5rUtJmU4pZ5YuKOmqn4lWlAMN0/KFPqAoCO5F", + "UBcP5mbJFv5Bkggp3U1M6+Y59cStahbX0F0TJXD28GVXcKjJS1lN6HVb98gn/1y1t+SA/NdpSQJl0xmy", + "CTKtHZG9YA15atpW5UrXWG2CEyInZa8mm6WuXqyOdgxH/isrowVM1SiR9lsrm8kcrwGwCDLJJH6nmgVj", + "tvy1QncCLoX/4LkKrGCUVklDgN6skLZxLdui1XOl0ajTX1kWLQDVvWtUSsSTSZ9sb4teHasnbGFGn8C/", + "AosLpoSkXkejaAIRiLSGx4tFFjbr20GRforlglIYqDalUB5ajZ8rYuOn59oEfT9yej0QfaVzxXr0hvYZ", + "JirrmKEsaRwmDmCzCZWiR4EfpRYAP5gdeCCOOMNE2ptCNhIqgoYUFKJR5OuZYmDhfCH0JeRSDsmjX6gH", + "JqSLvNvNzen/jwJfz1bzAkLh/kn8ir6Ck64qlpPN6l8RSd34ey2wNeYL5y6+IXBoYCqq/gXFo6ERJzKC", + "PPV+Jw/pqfioVPFPLua3YN2YSMZDRCVdDZXU/JRheWmn3whTeSGD8EtXna6NumRXCnIfa6xV988eV20V", + "UPFJQsGfWSIUNmsfGvcRfMaSvr4RVt1a5eYCMkR7ml5RziBTFY2T5eNLJbV/FgiX5cTv3q8FPy+qQTsg", + "5f6YngM+MIpVJRlrdWEeEkkQRepXFVML+INpZxOPxFh5TqPagTFaJdmo3BlRekgICSXSEF026KtrEliK", + "1w/dt+uvhTqQvt/e3hbsFPRKdSisAn/CJoPwqrJs0EtMEP3ra6s12bG0rBsnpY7V1Tb4gKcrrBGD/uur", + "q8HxcXGk9qE9XW7t6aJqmHohBNs8V0ANwdYfDrAwW9210gLodRdMS8r2YbryJtB1FU5TbWD+xUyZyE68", + "SD6p6+rbg19bPw2Xg7XT9va/jsXT/vTvG7XT3r092A+s7xnpGIFvVEWbRwdiewO5ttKLXghX+ttKVTWm", + "KvdHSX7w/tw8OucVBryCEKT91O9dY+pB/9ku5paKmk/WcGtLPQPxAdOLgyoC7qMThTWYWhxVWKEUdTCc", + "RJ6fat0koQTITKNqgrrCK35ZSS1knkMRkhkVBkiKBmwk7Oio5wZ6nWRWrAJiZVKTSwyl3kgizc0C4H8h", + "Nambni7f9txIoAraHCpoelT31jxTku0yCOMbVc5Aokl5XS/YlLUT5kHD8ZI3NSWhL2ZcLy3WgqOSbl0c", + "ruhBkDNfxtiGIpjODSgKVODUR8mOQkWtdGpuLiniD/BVyvVSUn34jEGqVXXug9R+MguAaEQluhNynylj", + "h97tBMfnN5cAZCExZFojvrOWHemsNpxlPEeL+KuRfavB4LtX2/MDCE4aWoonjftBw5DWUg4sJfOsLzVP", + "je9PMJzt/mXlAWLCtbF/Q/9786cdE7zsENx+mqJU0xgetzvVlHjRtdw4EPDt8CS3scXDhoUqoN3KzFiV", + "8RN4AEDNknI2TzFuUhyI2gF+KYBVSUFdlLehZy4QIsXU1HWltMrvSLpFY4CbGwuVvFoFECmrTblQA6e0", + "K1mY4eyIyMG7EHIaESkt5C/K+mvUVixNid0IHgto3Y9ZnqBTnk2ayBAwn1XkaIoZGRsEaSCOwcrYoRih", + "vIltEmzRFTlNCAgn/3ydRUpe9HKdxZcTAk5CQsCo5GbgwzR+Zd5lZ/HHtqoKceKPpJkUkJTyckYaYDSa", + "gl7If/hBaqreyDAe9WVUtioooTrzAV6kRTKwYFZt0EEl7uKPTtmOnYPHr+3SY6kS5ebY8WV42NF98vK1", + "g3gxyTPgjQOnsQ6X5Q9q3TGOb+TfvHYQax6IJoYQJMpxGQe7Mg8oH3SMd2UeSHh49ZAfpJuWWlhD7kYz", + "a6BQ8GKqrMdE6cCV6hyzUh2jXZczDaooV8sKk3VAMQGZ145Kf4V6OsPk2GR+5yGy69D4VUfRTywscv3n", + "0wEGzNKwqsBq5xTbdKvydV6pZLgT/q7z9nIfIVmce/P4s/+wlrPv/D75y/s5QAGs5U3/TS1ncKuJZDdx", + "REd51s1SLR3JRjfQDdw4HnpVz5OB6VI1rkCA0Qg/aQ8hEUrsJn2d7EAcJjmruI8/P/78+P8CAAD//w==", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/go/pkg/sdk/requests.go b/go/pkg/sdk/requests.go index 3f22f1282d..9238d64da6 100644 --- a/go/pkg/sdk/requests.go +++ b/go/pkg/sdk/requests.go @@ -191,6 +191,12 @@ type QueryRequest struct { // GraphQueries contains declarative graph matching, traversal, and path queries. GraphQueries map[string]GraphQuery `json:"graph_queries,omitempty"` + // GraphMetric reads one globally ranked, published graph metric generation. + GraphMetric *GraphMetricQuery `json:"graph_metric,omitempty"` + + // GraphMetricRerank blends a published graph metric into search hit scores. + GraphMetricRerank *GraphMetricRerank `json:"graph_metric_rerank,omitempty"` + // Hierarchy controls top-level result shape, bounded child hits, and projected ancestors. // A non-nil empty object selects direct index matches without ancestor hydration. Hierarchy *QueryHierarchy `json:"hierarchy,omitempty"` @@ -242,6 +248,12 @@ func (q QueryRequest) MarshalJSON() ([]byte, error) { Hierarchy: q.Hierarchy, ForeignSources: q.ForeignSources, } + if q.GraphMetric != nil { + oapiReq.GraphMetric = *q.GraphMetric + } + if q.GraphMetricRerank != nil { + oapiReq.GraphMetricRerank = *q.GraphMetricRerank + } // Preserve the distinction between an omitted projection and an explicitly // empty identity-only projection. The generated OpenAPI type uses a pointer // for this optional array so [] remains present on the wire. @@ -322,6 +334,14 @@ func (q *QueryRequest) UnmarshalJSON(data []byte) error { q.Pruner = oapiReq.Pruner q.SemanticSearch = oapiReq.SemanticSearch q.DocumentRenderer = oapiReq.DocumentRenderer + q.GraphMetric = nil + if !reflect.ValueOf(oapiReq.GraphMetric).IsZero() { + q.GraphMetric = &oapiReq.GraphMetric + } + q.GraphMetricRerank = nil + if !reflect.ValueOf(oapiReq.GraphMetricRerank).IsZero() { + q.GraphMetricRerank = &oapiReq.GraphMetricRerank + } q.GraphQueries = oapiReq.GraphQueries if q.GraphQueries != nil { if err := validateNamedGraphQueries(q.GraphQueries); err != nil { diff --git a/go/pkg/sdk/requests_test.go b/go/pkg/sdk/requests_test.go index 2206248fa5..5f91c8a70d 100644 --- a/go/pkg/sdk/requests_test.go +++ b/go/pkg/sdk/requests_test.go @@ -359,3 +359,78 @@ func TestQueryRequestMarshalPreservesJoin(t *testing.T) { t.Fatalf("Marshal encoded unexpected join: %s", body) } } + +func TestGraphIndexStatsRuntimeSummaryRoundTrip(t *testing.T) { + body, err := json.Marshal(GraphIndexStats{ + IndexType: GraphIndexStatsIndexType("graph"), + TotalEdges: 4, + GraphMetricRuntime: GraphMetricRuntimeStats{ + Enabled: true, + Role: GraphMetricRuntimeStatsRole("worker_pool"), + OwnerIdHash: 17, + WorkerCount: 3, + TakeoverCount: 2, + LostLeases: 1, + TotalPagesClaimed: 6, + LastPagesCompleted: 3, + LastBudgetExhausted: true, + }, + }) + if err != nil { + t.Fatalf("Marshal graph stats: %v", err) + } + for _, want := range [][]byte{ + []byte(`"graph_metric_runtime"`), + []byte(`"role":"worker_pool"`), + []byte(`"owner_id_hash":17`), + []byte(`"last_budget_exhausted":true`), + } { + if !bytes.Contains(body, want) { + t.Fatalf("Marshal omitted graph metric runtime field %s: %s", want, body) + } + } + + var stats GraphIndexStats + if err := json.Unmarshal(body, &stats); err != nil { + t.Fatalf("Unmarshal graph stats: %v", err) + } + if stats.GraphMetricRuntime.Role != GraphMetricRuntimeStatsRole("worker_pool") { + t.Fatalf("unexpected runtime role: %q", stats.GraphMetricRuntime.Role) + } + if stats.GraphMetricRuntime.OwnerIdHash != 17 || + stats.GraphMetricRuntime.WorkerCount != 3 || + stats.GraphMetricRuntime.TotalPagesClaimed != 6 || + !stats.GraphMetricRuntime.LastBudgetExhausted { + t.Fatalf("unexpected runtime summary: %+v", stats.GraphMetricRuntime) + } +} + +func TestQueryRequestMarshalPreservesDirectGraphMetricQuery(t *testing.T) { + body, err := json.Marshal(QueryRequest{ + Table: "docs", + GraphMetric: &GraphMetricQuery{ + Name: "central", + Index: "graph_idx", + Metric: "pagerank", + TopK: 25, + MetricFreshness: GraphMetricQueryMetricFreshness("fresh"), + }, + }) + if err != nil { + t.Fatalf("Marshal graph metric query: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("Unmarshal graph metric query: %v", err) + } + metric, ok := decoded["graph_metric"].(map[string]any) + if !ok { + t.Fatalf("graph_metric missing from request: %s", body) + } + if metric["name"] != "central" || metric["index"] != "graph_idx" || + metric["metric"] != "pagerank" || metric["top_k"] != float64(25) || + metric["metric_freshness"] != "fresh" { + t.Fatalf("unexpected graph_metric payload: %#v", metric) + } +} diff --git a/go/pkg/sdk/types.go b/go/pkg/sdk/types.go index c0e99ca95a..4bc76f097c 100644 --- a/go/pkg/sdk/types.go +++ b/go/pkg/sdk/types.go @@ -276,6 +276,7 @@ type ( // Graph index types GraphIndexConfig = oapi.GraphIndexConfig GraphIndexStats = oapi.GraphIndexStats + GraphIndexStatsIndexType = oapi.GraphIndexStatsIndexType GraphArtifactSourceConfig = oapi.GraphArtifactSourceConfig GraphArtifactSourceConfigFormat = oapi.GraphArtifactSourceConfigFormat GraphArtifactProducerConfig = oapi.GraphArtifactProducerConfig @@ -292,11 +293,41 @@ type ( GraphTemplateValue = oapi.GraphTemplateValue GraphTemplateValue0 = oapi.GraphTemplateValue0 GraphTemplateValue1 = oapi.GraphTemplateValue1 - EdgeTypeConfig = oapi.EdgeTypeConfig - EdgeTypeConfigTopology = oapi.EdgeTypeConfigTopology - EdgeDirection = oapi.EdgeDirection - Edge = oapi.Edge - EdgesResponse = oapi.EdgesResponse + GraphMetricActionResponse = oapi.GraphMetricActionResponse + GraphMetricBuildPageStatus = oapi.GraphMetricBuildPageStatus + GraphMetricBuildPageStatusRangeKind = oapi.GraphMetricBuildPageStatusRangeKind + GraphMetricBuildPageStatusState = oapi.GraphMetricBuildPageStatusState + GraphMetricEdgeFilterStatus = oapi.GraphMetricEdgeFilterStatus + GraphMetricEdgeFilterStatusMode = oapi.GraphMetricEdgeFilterStatusMode + GraphMetricEvent = oapi.GraphMetricEvent + GraphMetricEventKind = oapi.GraphMetricEventKind + GraphMetricFilter = oapi.GraphMetricFilter + GraphMetricFilterOp = oapi.GraphMetricFilterOp + GraphMetricOrder = oapi.GraphMetricOrder + GraphMetricOrderDirection = oapi.GraphMetricOrderDirection + GraphMetricOrderNulls = oapi.GraphMetricOrderNulls + GraphMetricProfile = oapi.GraphMetricProfile + GraphMetricQuery = oapi.GraphMetricQuery + GraphMetricQueryMetricFreshness = oapi.GraphMetricQueryMetricFreshness + GraphMetricRerank = oapi.GraphMetricRerank + GraphMetricRerankMetricFreshness = oapi.GraphMetricRerankMetricFreshness + GraphMetricRerankScoreDetails = oapi.GraphMetricRerankScoreDetails + GraphMetricResult = oapi.GraphMetricResult + GraphMetricRuntimeStats = oapi.GraphMetricRuntimeStats + GraphMetricRuntimeStatsRole = oapi.GraphMetricRuntimeStatsRole + GraphMetricScore = oapi.GraphMetricScore + GraphMetricStatus = oapi.GraphMetricStatus + GraphMetricStatusPhase = oapi.GraphMetricStatusPhase + // GraphQueryMetricFreshness is retained for source compatibility; metric + // freshness now belongs to the canonical traversal operation. + GraphQueryMetricFreshness = oapi.GraphTraversalMetricFreshness + GraphTraversalMetricFreshness = oapi.GraphTraversalMetricFreshness + QueryScoreDetails = oapi.QueryScoreDetails + EdgeTypeConfig = oapi.EdgeTypeConfig + EdgeTypeConfigTopology = oapi.EdgeTypeConfigTopology + EdgeDirection = oapi.EdgeDirection + Edge = oapi.Edge + EdgesResponse = oapi.EdgesResponse // Graph query types GraphQuery = oapi.GraphQuery diff --git a/openapi.yaml b/openapi.yaml index 4abde23934..03438d0fcd 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2409,6 +2409,93 @@ paths: - BasicAuth: [] - ApiKeyAuth: [] - BearerAuth: [] + /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action}: + parameters: + - name: tableName + in: path + required: true + description: Name of the table + schema: + type: string + - name: indexName + in: path + required: true + description: Name of the graph index + schema: + type: string + - name: metricName + in: path + required: true + description: Name of the configured graph metric + schema: + type: string + - name: action + in: path + required: true + description: Operational action to apply to the graph metric materialization + schema: + type: string + enum: + - refresh + - rebuild + - delete + - pause + - resume + post: + summary: Execute a graph metric operational action + description: > + Refresh, rebuild, delete, pause, or resume maintenance for a configured + + graph metric. The metric configuration remains owned by the graph index. + + Refresh and rebuild durably enqueue bounded, resumable maintenance and + + return the aggregate shard status without waiting for graph-sized work. + + `delete` clears materialized metric state and durably disables automatic + + maintenance. A later refresh, rebuild, or resume action re-enables the + + metric and can publish a new generation. + tags: + - index_management + operationId: executeGraphMetricAction + responses: + '200': + description: Aggregate graph metric status after the action is durably accepted + content: + application/json: + schema: + $ref: '#/components/schemas/GraphMetricActionResponse' + '400': + $ref: '#/components/responses/BadRequest' + '409': + description: The table topology or write-owner generation changed, or only some shards accepted + the action; retrying is consistency-safe and reuses any still-active build + content: + text/plain: + schema: + type: string + '429': + description: Local storage resources are temporarily exhausted + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + $ref: '#/components/responses/NotFound' + '405': + description: Graph metric actions are unavailable for this runtime + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + $ref: '#/components/responses/InternalServerError' + security: + - BasicAuth: [] + - ApiKeyAuth: [] + - BearerAuth: [] /db/v1/transactions/commit: post: summary: Commit an OCC transaction @@ -11789,6 +11876,13 @@ components: $ref: '#/components/schemas/CreatedIndex' status: $ref: '#/components/schemas/IndexStats' + GraphMetricActionResponse: + type: object + required: + - status + properties: + status: + $ref: '#/components/schemas/GraphMetricStatus' LsmStorageStatus: type: object description: Compact LSM backend operational status. Detailed low-level counters are available through @@ -12782,9 +12876,11 @@ components: participant propagation is still completing. `committed_repair_required` - means the primary write committed, but a terminal enrichment failure needs + means the primary write committed, but a terminal background materialization - operator repair and will not be retried indefinitely. + failure needs operator repair and will not be retried indefinitely. Inspect + + `failure` when present; retrying the document write is unnecessary. inserted: type: integer description: Number of documents successfully inserted @@ -12794,6 +12890,34 @@ components: transformed: type: integer description: Number of documents successfully transformed + failure: + $ref: '#/components/schemas/BatchCommittedFailure' + BatchCommittedFailure: + type: object + description: > + Additive details for a committed batch that needs operator action. The + + open string code is forward-compatible with older SDKs; clients should + + treat unknown codes as non-retryable when `retryable` is false. + required: + - code + - message + - retryable + properties: + code: + type: string + description: Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + message: + type: string + description: Actionable operator guidance. + reason: + type: string + description: Optional stable reason within the failure category, such as `build_budget_exceeded`. + retryable: + type: boolean + description: Whether replaying the document mutation is safe. Committed repair outcomes are + false. DenseRepairBackpressureError: type: object description: A dense-index rebuild is retaining replay history and the node has reached its hard @@ -15315,6 +15439,10 @@ components: - profile - required: - reranker + - required: + - graph_metric + - required: + - graph_metric_rerank - required: - analyses - required: @@ -15906,6 +16034,16 @@ components: } ``` + graph_metric: + $ref: '#/components/schemas/GraphMetricQuery' + description: Direct top-k read from a published graph metric generation. Results are returned + in graph_metric_results under the requested name or the metric name when no explicit name + is supplied. + graph_metric_rerank: + $ref: '#/components/schemas/GraphMetricRerank' + description: Blend a published graph metric feature into ordinary search hit scores. Requests + may require either any published generation or a generation that is fresh with respect to + graph writes. analyses: $ref: '#/components/schemas/Analyses' graph_queries: @@ -16398,10 +16536,43 @@ components: merge: $ref: '#/components/schemas/MergeProfile' description: Result merge statistics (present for hybrid search). + graph_metrics: + type: array + items: + $ref: '#/components/schemas/GraphMetricProfile' + description: Graph metric freshness and generation details for metric-aware query work. sort: $ref: '#/components/schemas/SortProfile' description: Sort execution statistics (present when the query used ordered page options and profiling was enabled). + GraphMetricProfile: + type: object + required: + - query_name + - source + - index_name + - metric_name + - freshness + - status + properties: + query_name: + type: string + description: Name of the graph query or graph metric query that used the metric. + source: + type: string + description: Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + index_name: + type: string + description: Graph index that owns the metric. + metric_name: + type: string + description: Graph metric name within the index. + freshness: + type: string + description: Effective freshness mode requested for this metric use. + status: + $ref: '#/components/schemas/GraphMetricStatus' + description: Published generation and freshness status observed by the query. SortProfile: type: object additionalProperties: false @@ -17008,6 +17179,67 @@ components: description: Deprecated child chunk hits included by the v0.2-compatible implicit source rollup. items: $ref: '#/components/schemas/HierarchyMatchHit' + QueryScoreDetails: + type: object + description: Optional score provenance for ranking features that changed the final hit score. + properties: + graph_metric_rerank: + $ref: '#/components/schemas/GraphMetricRerankScoreDetails' + description: Score contribution from an explicit graph_metric_rerank request. + GraphMetricRerankScoreDetails: + type: object + required: + - index_name + - metric_name + - base_score + - base_weight + - metric_score_used + - metric_weight + - missing_score_used + - final_score + - published_generation + properties: + index_name: + type: string + description: Graph index that provided the metric score. + metric_name: + type: string + description: Graph metric used as a score feature. + base_score: + type: number + format: double + description: Hit score before graph metric rerank composition. + base_weight: + type: number + format: double + description: Weight applied to the base score. + metric_score: + type: number + format: double + nullable: true + description: Published metric score for this hit, or null when the hit was missing from the + metric generation. + metric_score_used: + type: number + format: double + description: Metric feature value used in the formula after applying missing_score fallback + if needed. + metric_weight: + type: number + format: double + description: Weight applied to the metric score feature. + missing_score_used: + type: boolean + description: True when metric_score was missing and the request's missing_score fallback was + used. + final_score: + type: number + format: double + description: Final hit score after graph metric rerank composition. + published_generation: + type: integer + format: int64 + description: Published graph metric score generation used for this hit. QueryHit: type: object description: A single query result hit @@ -17046,6 +17278,10 @@ components: format: double description: Scores partitioned by index when using RRF search. x-go-name: indexScores + _score_details: + $ref: '#/components/schemas/QueryScoreDetails' + description: Optional score provenance for ranking features applied to this hit. + x-go-name: scoreDetails _source: type: object additionalProperties: true @@ -17160,6 +17396,11 @@ components: additionalProperties: $ref: '#/components/schemas/AnalysesResult' description: Analysis results like PCA and t-SNE per index embeddings. + graph_metric_results: + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricResult' + description: Results from direct graph metric reads. profile: x-go-type-skip-optional-pointer: false allOf: @@ -19304,6 +19545,68 @@ components: description: Durable graph edge type. Values must be valid UTF-8 and encode to at most 64 KiB; `maxLength` is the standard-schema code-point ceiling and `x-antfly-max-utf8-bytes` carries the exact wire-byte limit. + GraphMetricEdgeFilter: + type: object + additionalProperties: false + description: Omitting this object selects all edge types. A types list selects only those types; + mode and types cannot both be supplied. + properties: + mode: + type: string + enum: + - all + types: + type: array + minItems: 1 + uniqueItems: true + items: + $ref: '#/components/schemas/GraphEdgeType' + GraphMetricConfig: + type: object + additionalProperties: false + description: Published metric configuration. If kind is omitted, the metric name must be a supported + kind. + properties: + enabled: + type: boolean + default: true + kind: + type: string + enum: + - pagerank + - degree + - eigenvector + - hits_authority + - hits_hub + refresh: + type: string + enum: + - background + - manual + default: background + description: Serverless accepts background only. + damping: + type: number + format: double + minimum: 0 + exclusiveMinimum: true + maximum: 1 + exclusiveMaximum: true + default: 0.85 + tolerance: + type: number + format: double + minimum: 0 + exclusiveMinimum: true + default: 1.0e-06 + max_iterations: + type: integer + format: int32 + minimum: 1 + maximum: 1000 + default: 50 + edge_filter: + $ref: '#/components/schemas/GraphMetricEdgeFilter' GraphTemplateValue: description: A literal string or finite numeric value, or a Handlebars template evaluated for each materialized graph item. @@ -19820,6 +20123,13 @@ components: - - sources - source properties: + metrics: + type: object + description: Named published graph metrics. Serverless supports background refresh only and + limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, + and 128 UTF-8 bytes per metric name. + additionalProperties: + $ref: '#/components/schemas/GraphMetricConfig' sources: type: array minItems: 1 @@ -20629,6 +20939,10 @@ components: type: object description: Credential-free normalized graph configuration returned after creation. properties: + metrics: + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricConfig' summarizer: $ref: '#/components/schemas/CreatedProviderConfig' template: @@ -21837,103 +22151,258 @@ components: type: object additionalProperties: true description: Artifact promotion replay diagnostics. - GraphIndexStats: + GraphMetricRuntimeStats: type: object - description: Statistics for graph index - required: - - index_type + description: Summarized graph metric maintenance runtime state. Identity fields are stable hashes, + not raw process or owner identifiers. properties: - index_type: - x-go-type-skip-optional-pointer: true + enabled: + type: boolean + role: type: string enum: - - graph - description: Discriminator for the index stats variant. - readiness: - $ref: '#/components/schemas/IndexReadinessStatus' - deprecated: true - description: Deprecated compatibility projection. Use milestones and revision fields. - incarnation: - x-go-type-skip-optional-pointer: true - type: string - description: Opaque identity of the desired index incarnation. Clients may compare it for equality - but must not interpret its contents. - target_revision: - x-go-type-skip-optional-pointer: true + - combined + - coordinator + - worker + - worker_pool + runtime_id_hash: type: integer format: uint64 - published_revision: - x-go-type-skip-optional-pointer: true + owner_id_hash: type: integer format: uint64 - milestones: - $ref: '#/components/schemas/IndexMilestones' - error: - x-go-type-skip-optional-pointer: true - type: string - description: Error message if stats could not be retrieved - total_edges: - x-go-type-skip-optional-pointer: true + lease_key_hash: type: integer format: uint64 - description: Total number of edges in the graph - edge_types: - type: object - additionalProperties: - type: integer - format: uint64 - description: Count of edges per edge type - rebuilding: - x-go-type-skip-optional-pointer: true + worker_id_hash: + type: integer + format: uint64 + worker_count: + type: integer + format: uint64 + lease_owned: type: boolean - description: Whether the index is currently rebuilding - repair: - $ref: '#/components/schemas/IndexRepairStatus' - backfill_active: - x-go-type-skip-optional-pointer: true + has_lease: type: boolean - description: Whether the index is actively rebuilding, materializing, or catching up. - backfill_progress: - x-go-type-skip-optional-pointer: true - type: number - format: double - description: Rebuild progress as a ratio from 0.0 to 1.0 - backfill_items_processed: - x-go-type-skip-optional-pointer: true + acquisition_count: type: integer format: uint64 - description: Number of edges indexed during current rebuild - backfill_state: - x-go-type-skip-optional-pointer: true - type: string - description: Operational readiness state such as ready, running, retrying, degraded, or failed. - doc_count: - x-go-type-skip-optional-pointer: true + takeover_count: type: integer format: uint64 - description: Number of documents covered by the graph index. - edge_count: - x-go-type-skip-optional-pointer: true + lease_acquire_failures: type: integer format: uint64 - description: Number of graph edges currently indexed. - node_count: - x-go-type-skip-optional-pointer: true + lost_leases: type: integer format: uint64 - description: Number of graph nodes currently indexed. - replay_applied_sequence: - x-go-type-skip-optional-pointer: true + last_acquired_ms: type: integer format: uint64 - replay_target_sequence: - x-go-type-skip-optional-pointer: true + lease_expires_at_ms: type: integer format: uint64 - replay_catch_up_required: - x-go-type-skip-optional-pointer: true + description: Cached expiry of the currently held maintenance lease, or zero when no lease is + held. + lease_renew_after_ms: + type: integer + format: uint64 + description: Earliest time the runtime will renew its maintenance lease, or zero when no lease + is held. + renewal_count: + type: integer + format: uint64 + description: Number of durable maintenance lease renewals completed by this runtime. + started: type: boolean - runtime_present: + shutdown: + type: boolean + notified: + type: boolean + ticks_started: + type: integer + format: uint64 + ticks_completed: + type: integer + format: uint64 + durable_progress_ticks: + type: integer + format: uint64 + idle_ticks: + type: integer + format: uint64 + error_ticks: + type: integer + format: uint64 + last_error_name: + type: string + total_metrics_scanned: + type: integer + format: uint64 + total_active_builds: + type: integer + format: uint64 + total_builds_started: + type: integer + format: uint64 + total_worker_steps: + type: integer + format: uint64 + total_coordinator_steps: + type: integer + format: uint64 + total_retired_input_records: + description: Consumed intermediate records retired at completed reduction barriers. + type: integer + format: uint64 + total_pages_claimed: + type: integer + format: uint64 + total_pages_completed: + type: integer + format: uint64 + total_phases_advanced: + type: integer + format: uint64 + total_published: + type: integer + format: uint64 + total_failed_builds: + type: integer + format: uint64 + last_metrics_scanned: + type: integer + format: uint64 + last_active_builds: + type: integer + format: uint64 + last_builds_started: + type: integer + format: uint64 + last_worker_steps: + type: integer + format: uint64 + last_coordinator_steps: + type: integer + format: uint64 + last_retired_input_records: + description: Consumed intermediate records retired in the latest maintenance tick. + type: integer + format: uint64 + last_pages_claimed: + type: integer + format: uint64 + last_pages_completed: + type: integer + format: uint64 + last_phases_advanced: + type: integer + format: uint64 + last_published: + type: integer + format: uint64 + last_failed_builds: + type: integer + format: uint64 + last_budget_exhausted: + type: boolean + GraphIndexStats: + type: object + description: Statistics for graph index + required: + - index_type + properties: + index_type: + x-go-type-skip-optional-pointer: true + type: string + enum: + - graph + description: Discriminator for the index stats variant. + readiness: + $ref: '#/components/schemas/IndexReadinessStatus' + deprecated: true + description: Deprecated compatibility projection. Use milestones and revision fields. + incarnation: + x-go-type-skip-optional-pointer: true + type: string + description: Opaque identity of the desired index incarnation. Clients may compare it for equality + but must not interpret its contents. + target_revision: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + published_revision: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + milestones: + $ref: '#/components/schemas/IndexMilestones' + error: + x-go-type-skip-optional-pointer: true + type: string + description: Error message if stats could not be retrieved + total_edges: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + description: Total number of edges in the graph + edge_types: + type: object + additionalProperties: + type: integer + format: uint64 + description: Count of edges per edge type + rebuilding: + x-go-type-skip-optional-pointer: true + type: boolean + description: Whether the index is currently rebuilding + repair: + $ref: '#/components/schemas/IndexRepairStatus' + backfill_active: + x-go-type-skip-optional-pointer: true + type: boolean + description: Whether the index is actively rebuilding, materializing, or catching up. + backfill_progress: + x-go-type-skip-optional-pointer: true + type: number + format: double + description: Rebuild progress as a ratio from 0.0 to 1.0 + backfill_items_processed: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + description: Number of edges indexed during current rebuild + backfill_state: + x-go-type-skip-optional-pointer: true + type: string + description: Operational readiness state such as ready, running, retrying, degraded, or failed. + doc_count: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + description: Number of documents covered by the graph index. + edge_count: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + description: Number of graph edges currently indexed. + node_count: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + description: Number of graph nodes currently indexed. + replay_applied_sequence: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + replay_target_sequence: + x-go-type-skip-optional-pointer: true + type: integer + format: uint64 + replay_catch_up_required: + x-go-type-skip-optional-pointer: true + type: boolean + runtime_present: x-go-type-skip-optional-pointer: true type: boolean runtime_fresh: @@ -22076,6 +22545,9 @@ components: result_nodes: type: integer format: uint64 + graph_metric_runtime: + x-go-type-skip-optional-pointer: true + $ref: '#/components/schemas/GraphMetricRuntimeStats' AlgebraicIndexStats: type: object description: Compact public statistics for an algebraic sidecar index. Detailed runtime, adaptive, @@ -22393,6 +22865,251 @@ components: - $ref: '#/components/schemas/EmbeddingsIndexStats' - $ref: '#/components/schemas/GraphIndexStats' - $ref: '#/components/schemas/AlgebraicIndexStats' + GraphMetricEdgeFilterStatus: + type: object + required: + - mode + properties: + mode: + type: string + enum: + - all + - types + types: + type: array + items: + type: string + GraphMetricBuildPageStatus: + type: object + required: + - phase + - iteration + - page_id + - state + - range_kind + properties: + phase: + type: string + iteration: + type: integer + format: int64 + page_id: + type: integer + format: int64 + state: + type: string + enum: + - pending + - leased + - complete + - failed + range_kind: + type: string + enum: + - full + - reverse_edges + - nodes + - scores + - contributions + - job_control + - summary + worker_id: + type: string + description: Worker id that owns or last failed this page. + lease_expires_at_ms: + type: integer + format: int64 + description: Unix epoch milliseconds when the page lease expires, or 0 when not leased. + attempt: + type: integer + format: int64 + description: Current attempt number for this page. + cursor: + type: string + description: Opaque resumable cursor for this page. + completed_units: + type: integer + format: int64 + description: Completed work units for this page. + total_units: + type: integer + format: int64 + description: Estimated total work units for this page. + last_error: + type: string + description: Last page-level error. + GraphMetricEvent: + type: object + required: + - sequence + - kind + - at_ms + - target_edge_generation + - published_generation + - score_count + properties: + sequence: + type: integer + format: int64 + kind: + type: string + enum: + - publish + - delete + - pause + - resume + - failed + at_ms: + type: integer + format: int64 + target_edge_generation: + type: integer + format: int64 + published_generation: + type: integer + format: int64 + score_count: + type: integer + format: int64 + GraphMetricStatus: + type: object + required: + - state + - phase + - published_generation + - edge_generation + - target_edge_generation + - build_queued + - progress + - converged + - iterations_completed + - delta + - computed_at_ms + properties: + state: + type: string + phase: + type: string + enum: + - idle + - computing + - publishing + - complete + - prepare_generation + - scan_edges_and_out_degree + - initialize_ranks + - iterate_contributions + - reduce_ranks + - hits_hub_contributions + - hits_hub_reduce_ranks + - check_convergence + - publish_generation + - cleanup_old_generations + edge_filter: + $ref: '#/components/schemas/GraphMetricEdgeFilterStatus' + metadata_version: + type: integer + format: int64 + description: Version of the published graph metric metadata schema. + config_fingerprint: + type: string + pattern: ^[0-9a-f]{16}$ + description: Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every + SDK preserves all 64 bits. + maintenance_paused: + type: boolean + build_queued: + type: boolean + description: Whether a local or distributed build is queued after the currently published or + building generation. + published_generation: + type: integer + format: int64 + edge_generation: + type: integer + format: int64 + target_edge_generation: + type: integer + format: int64 + queued_generation: + type: integer + format: int64 + description: Pending edge generation waiting to build, or 0 when no build is queued. + building_generation: + type: integer + format: int64 + description: Edge generation currently held by an active build lease, or 0 when idle. + build_job_id: + type: integer + format: int64 + description: Durable identifier for the active graph metric build job, or 0 when idle. + build_started_at_ms: + type: integer + format: int64 + description: Unix epoch milliseconds when the active graph metric build started, or 0 when idle. + build_iteration: + type: integer + format: int64 + description: Iteration number reported by the active build lease, or 0 when idle or not iterative. + build_lease_expires_at_ms: + type: integer + format: int64 + description: Unix epoch milliseconds when the active build lease expires, or 0 when idle. + build_worker_id: + type: string + description: Worker id that owns the active build lease. Local builds use `local`. + build_cursor: + type: string + description: Opaque resumable cursor for the active build phase. Empty or omitted when idle + or when the phase has no cursor. + build_completed_units: + type: integer + format: int64 + description: Completed work units for the active graph metric build, or 0 when idle or unknown. + build_total_units: + type: integer + format: int64 + description: Estimated total work units for the active graph metric build, or 0 when idle or + unknown. + build_pages: + type: array + description: Active leased or failed build pages for the current build phase, capped and ordered + by durable page key. + items: + $ref: '#/components/schemas/GraphMetricBuildPageStatus' + build_pages_truncated: + type: boolean + description: Whether build_pages was capped before every active page could be included. + retry_count: + type: integer + format: int64 + description: Number of consecutive failed build attempts for the current target generation, + or 0 when no failure applies. + last_error: + type: string + description: Last build error for the current failed target generation. + progress: + type: number + format: double + description: Build progress for the target edge generation, from 0.0 to 1.0 + converged: + type: boolean + iterations_completed: + type: integer + format: int64 + delta: + type: number + format: double + computed_at_ms: + type: integer + format: int64 + last_event: + $ref: '#/components/schemas/GraphMetricEvent' + recent_events: + type: array + description: Recent graph metric events, newest first. + items: + $ref: '#/components/schemas/GraphMetricEvent' ChatToolName: type: string description: > @@ -24318,6 +25035,90 @@ components: provider: cohere model: rerank-v4.0-pro field: content + GraphMetricQuery: + type: object + description: Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables + require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required + instead of merging mathematically incompatible shard-local scores. + required: + - index + - metric + properties: + name: + type: string + minLength: 1 + description: Optional result key. Defaults to the metric name. + index: + type: string + minLength: 1 + description: Graph index that owns the published metric. + metric: + type: string + minLength: 1 + description: Graph metric to read. + top_k: + type: integer + format: int32 + minimum: 1 + maximum: 10000 + default: 10 + description: Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated + metric snapshot. + metric_freshness: + type: string + enum: + - published + - fresh + default: published + description: Whether the latest published generation may be stale or must match the graph edge + generation. + GraphMetricRerank: + type: object + description: Blends a published graph metric into hit scores. Multi-shard tables require a globally + coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. + required: + - index + - metric + properties: + index: + type: string + description: Graph index that owns the published metric. + metric: + type: string + description: Graph metric name to blend into the search hit score. + candidate_count: + type: integer + format: int32 + minimum: 1 + maximum: 10000 + description: Bounded retrieval window scored by the graph metric before offset and limit are + applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. + An explicit value must cover offset plus limit. Larger windows improve promotion recall at + predictable linear score-read cost. + base_weight: + type: number + format: double + default: 1.0 + description: Multiplier applied to the existing hit score before adding the graph metric feature. + weight: + type: number + format: double + default: 1.0 + description: Multiplier applied to the graph metric score before it is added to the existing + hit score. + missing_score: + type: number + format: double + default: 0.0 + description: Metric feature value to use for hits that do not have a score in the published + metric generation. + metric_freshness: + type: string + enum: + - published + - fresh + default: published + description: Whether stale published generations are acceptable or the metric must be fresh. GraphIdentifier: type: string minLength: 1 @@ -25124,6 +25925,50 @@ components: - $ref: '#/components/schemas/GraphIdentityNodeSelector' - $ref: '#/components/schemas/GraphResultRefNodeSelector' description: Select graph nodes using exactly one explicit, exact selector form. + GraphMetricOrder: + type: object + required: + - metric + properties: + metric: + type: string + minLength: 1 + direction: + type: string + enum: + - asc + - desc + nulls: + type: string + enum: + - first + - last + - nulls_first + - nulls_last + GraphMetricFilter: + type: object + required: + - metric + - op + - value + properties: + metric: + type: string + minLength: 1 + op: + type: string + description: Semantic comparison operator. Named values keep generated SDK enums portable and + readable. + enum: + - gt + - gte + - lt + - lte + - eq + - neq + value: + type: number + format: double GraphTraversal: type: object additionalProperties: false @@ -25172,6 +26017,38 @@ components: items: type: string description: Requires include_documents=true. Omit to include all document fields. + metrics: + type: array + maxItems: 16 + uniqueItems: true + items: + type: string + minLength: 1 + description: Graph metric names to project onto returned traversal nodes. + order_by: + type: array + maxItems: 8 + uniqueItems: true + items: + $ref: '#/components/schemas/GraphMetricOrder' + description: Sort traversal candidates by graph metric score before applying limit. + where_metric: + type: array + maxItems: 32 + items: + $ref: '#/components/schemas/GraphMetricFilter' + description: Filter traversal candidates by graph metric score before applying limit. + metric_freshness: + type: string + enum: + - published + - fresh + default: published + description: Freshness required for projected, ordered, and filtered graph metrics. + include_metric_status: + type: boolean + default: false + description: Include graph metric status metadata in the traversal profile. filter: $ref: '#/components/schemas/GraphDocumentFilter' description: Non-scoring structured stored-document predicate for reached nodes. @@ -25625,6 +26502,70 @@ components: type: array items: type: string + metrics: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 16 + uniqueItems: true + items: + type: string + minLength: 1 + description: Graph metric names to project onto legacy graph_searches result nodes. + order_by: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 8 + uniqueItems: true + items: + $ref: '#/components/schemas/GraphMetricOrder' + description: Sort legacy graph_searches result nodes by graph metric score. + where_metric: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 32 + items: + $ref: '#/components/schemas/GraphMetricFilter' + description: Filter legacy graph_searches result nodes by graph metric score. + metric_freshness: + x-go-type-skip-optional-pointer: true + type: string + enum: + - published + - fresh + description: Freshness required for projected, ordered, and filtered graph metrics. + include_metric_status: + x-go-type-skip-optional-pointer: true + type: boolean + description: Include graph metric status metadata in the legacy graph_searches result. + GraphMetricScore: + type: object + required: + - node + - score + properties: + node: + type: string + score: + type: number + format: double + GraphMetricResult: + type: object + required: + - index_name + - metric + - scores + - status + properties: + index_name: + type: string + metric: + type: string + scores: + type: array + items: + $ref: '#/components/schemas/GraphMetricScore' + status: + $ref: '#/components/schemas/GraphMetricStatus' GraphBindingNode: type: object additionalProperties: false @@ -25856,6 +26797,12 @@ components: type: string description: Algebraic provenance labels folded into this result, when requested by an algebraic graph executor + metrics: + x-go-type-skip-optional-pointer: true + type: object + additionalProperties: true + description: Projected graph metric scores keyed by metric name. Values are numbers or null + when a requested metric has no score for the node. evidence: x-go-type-skip-optional-pointer: true type: object @@ -25882,6 +26829,11 @@ components: items: $ref: '#/components/schemas/GraphResultNode' description: Traversal result nodes; requested paths are stored on each node. + metric_status: + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricStatus' + description: Graph metric status metadata keyed by metric name when requested. stats: $ref: '#/components/schemas/GraphResultStats' GraphPath: @@ -26118,6 +27070,12 @@ components: deprecated: true description: Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field. + metric_status: + x-go-type-skip-optional-pointer: true + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricStatus' + description: Graph metric status metadata keyed by metric name. StatefulGraphResult: description: Graph result emitted by the stateful compatibility transport. Canonical graph_queries produce GraphResult; deprecated graph_searches may produce LegacyGraphSearchResult during the diff --git a/py/packages/sdk/examples/advanced_search.py b/py/packages/sdk/examples/advanced_search.py index 23e31b2940..dae53c0e88 100644 --- a/py/packages/sdk/examples/advanced_search.py +++ b/py/packages/sdk/examples/advanced_search.py @@ -2,6 +2,7 @@ """Advanced search examples for Antfly SDK.""" from typing import cast + from antfly import AntflyClient from antfly.client_generated.models.query_hits_total import QueryHitsTotal from antfly.client_generated.models.query_hits_total_relation import QueryHitsTotalRelation diff --git a/py/packages/sdk/examples/basic_usage.py b/py/packages/sdk/examples/basic_usage.py index 5d822f217d..ee8712efdd 100644 --- a/py/packages/sdk/examples/basic_usage.py +++ b/py/packages/sdk/examples/basic_usage.py @@ -2,6 +2,7 @@ """Basic usage example for Antfly SDK.""" from typing import cast + from antfly import AntflyClient from antfly.client_generated.types import Unset diff --git a/py/packages/sdk/src/antfly/client_generated/api/index_management/execute_graph_metric_action.py b/py/packages/sdk/src/antfly/client_generated/api/index_management/execute_graph_metric_action.py new file mode 100644 index 0000000000..e5ca96249f --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/api/index_management/execute_graph_metric_action.py @@ -0,0 +1,260 @@ +from http import HTTPStatus +from typing import Any +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.error import Error +from ...models.execute_graph_metric_action_action import ExecuteGraphMetricActionAction +from ...models.graph_metric_action_response import GraphMetricActionResponse +from ...types import Response + + +def _get_kwargs( + table_name: str, + index_name: str, + metric_name: str, + action: ExecuteGraphMetricActionAction, +) -> dict[str, Any]: + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/db/v1/tables/{table_name}/indexes/{index_name}/graph-metrics/{metric_name}:{action}".format( + table_name=quote(str(table_name), safe=""), + index_name=quote(str(index_name), safe=""), + metric_name=quote(str(metric_name), safe=""), + action=quote(str(action), safe=""), + ), + } + + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Error | GraphMetricActionResponse | str | None: + if response.status_code == 200: + response_200 = GraphMetricActionResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 + + if response.status_code == 404: + response_404 = Error.from_dict(response.json()) + + return response_404 + + if response.status_code == 405: + response_405 = Error.from_dict(response.json()) + + return response_405 + + if response.status_code == 409: + response_409 = response.text + return response_409 + + if response.status_code == 429: + response_429 = Error.from_dict(response.json()) + + return response_429 + + if response.status_code == 500: + response_500 = Error.from_dict(response.json()) + + return response_500 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[Error | GraphMetricActionResponse | str]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + table_name: str, + index_name: str, + metric_name: str, + action: ExecuteGraphMetricActionAction, + *, + client: AuthenticatedClient, +) -> Response[Error | GraphMetricActionResponse | str]: + """Execute a graph metric operational action + + Refresh, rebuild, delete, pause, or resume maintenance for a configured + graph metric. The metric configuration remains owned by the graph index. + Refresh and rebuild durably enqueue bounded, resumable maintenance and + return the aggregate shard status without waiting for graph-sized work. + `delete` clears materialized metric state and durably disables automatic + maintenance. A later refresh, rebuild, or resume action re-enables the + metric and can publish a new generation. + + Args: + table_name (str): + index_name (str): + metric_name (str): + action (ExecuteGraphMetricActionAction): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Error | GraphMetricActionResponse | str] + """ + + kwargs = _get_kwargs( + table_name=table_name, + index_name=index_name, + metric_name=metric_name, + action=action, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + table_name: str, + index_name: str, + metric_name: str, + action: ExecuteGraphMetricActionAction, + *, + client: AuthenticatedClient, +) -> Error | GraphMetricActionResponse | str | None: + """Execute a graph metric operational action + + Refresh, rebuild, delete, pause, or resume maintenance for a configured + graph metric. The metric configuration remains owned by the graph index. + Refresh and rebuild durably enqueue bounded, resumable maintenance and + return the aggregate shard status without waiting for graph-sized work. + `delete` clears materialized metric state and durably disables automatic + maintenance. A later refresh, rebuild, or resume action re-enables the + metric and can publish a new generation. + + Args: + table_name (str): + index_name (str): + metric_name (str): + action (ExecuteGraphMetricActionAction): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Error | GraphMetricActionResponse | str + """ + + return sync_detailed( + table_name=table_name, + index_name=index_name, + metric_name=metric_name, + action=action, + client=client, + ).parsed + + +async def asyncio_detailed( + table_name: str, + index_name: str, + metric_name: str, + action: ExecuteGraphMetricActionAction, + *, + client: AuthenticatedClient, +) -> Response[Error | GraphMetricActionResponse | str]: + """Execute a graph metric operational action + + Refresh, rebuild, delete, pause, or resume maintenance for a configured + graph metric. The metric configuration remains owned by the graph index. + Refresh and rebuild durably enqueue bounded, resumable maintenance and + return the aggregate shard status without waiting for graph-sized work. + `delete` clears materialized metric state and durably disables automatic + maintenance. A later refresh, rebuild, or resume action re-enables the + metric and can publish a new generation. + + Args: + table_name (str): + index_name (str): + metric_name (str): + action (ExecuteGraphMetricActionAction): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Error | GraphMetricActionResponse | str] + """ + + kwargs = _get_kwargs( + table_name=table_name, + index_name=index_name, + metric_name=metric_name, + action=action, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + table_name: str, + index_name: str, + metric_name: str, + action: ExecuteGraphMetricActionAction, + *, + client: AuthenticatedClient, +) -> Error | GraphMetricActionResponse | str | None: + """Execute a graph metric operational action + + Refresh, rebuild, delete, pause, or resume maintenance for a configured + graph metric. The metric configuration remains owned by the graph index. + Refresh and rebuild durably enqueue bounded, resumable maintenance and + return the aggregate shard status without waiting for graph-sized work. + `delete` clears materialized metric state and durably disables automatic + maintenance. A later refresh, rebuild, or resume action re-enables the + metric and can publish a new generation. + + Args: + table_name (str): + index_name (str): + metric_name (str): + action (ExecuteGraphMetricActionAction): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Error | GraphMetricActionResponse | str + """ + + return ( + await asyncio_detailed( + table_name=table_name, + index_name=index_name, + metric_name=metric_name, + action=action, + client=client, + ) + ).parsed diff --git a/py/packages/sdk/src/antfly/client_generated/models/__init__.py b/py/packages/sdk/src/antfly/client_generated/models/__init__.py index 054a28ddda..8333441232 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/__init__.py +++ b/py/packages/sdk/src/antfly/client_generated/models/__init__.py @@ -60,6 +60,7 @@ from .backup_request import BackupRequest from .backup_request_format import BackupRequestFormat from .backup_table_response_201 import BackupTableResponse201 +from .batch_committed_failure import BatchCommittedFailure from .batch_request import BatchRequest from .batch_request_inserts import BatchRequestInserts from .batch_request_inserts_additional_property import BatchRequestInsertsAdditionalProperty @@ -151,6 +152,7 @@ from .created_graph_artifact_source_config_format import CreatedGraphArtifactSourceConfigFormat from .created_graph_index import CreatedGraphIndex from .created_graph_index_config import CreatedGraphIndexConfig +from .created_graph_index_config_metrics import CreatedGraphIndexConfigMetrics from .created_graph_index_type import CreatedGraphIndexType from .created_index_common import CreatedIndexCommon from .created_provider_config import CreatedProviderConfig @@ -246,6 +248,7 @@ from .exact_sort_error import ExactSortError from .exact_sort_error_error import ExactSortErrorError from .exact_sort_error_status import ExactSortErrorStatus +from .execute_graph_metric_action_action import ExecuteGraphMetricActionAction from .execution_policy import ExecutionPolicy from .extension_error import ExtensionError from .extension_member import ExtensionMember @@ -383,6 +386,7 @@ from .graph_exact_result_stats import GraphExactResultStats from .graph_identity_node_selector import GraphIdentityNodeSelector from .graph_index_config import GraphIndexConfig +from .graph_index_config_metrics import GraphIndexConfigMetrics from .graph_index_stats import GraphIndexStats from .graph_index_stats_algebraic_graph import GraphIndexStatsAlgebraicGraph from .graph_index_stats_algebraic_graph_traversal import GraphIndexStatsAlgebraicGraphTraversal @@ -404,8 +408,39 @@ from .graph_match_operation_limit_exceeded_error_error import GraphMatchOperationLimitExceededErrorError from .graph_match_operation_limit_exceeded_error_status import GraphMatchOperationLimitExceededErrorStatus from .graph_match_query import GraphMatchQuery +from .graph_metric_action_response import GraphMetricActionResponse +from .graph_metric_build_page_status import GraphMetricBuildPageStatus +from .graph_metric_build_page_status_range_kind import GraphMetricBuildPageStatusRangeKind +from .graph_metric_build_page_status_state import GraphMetricBuildPageStatusState +from .graph_metric_config import GraphMetricConfig +from .graph_metric_config_kind import GraphMetricConfigKind +from .graph_metric_config_refresh import GraphMetricConfigRefresh +from .graph_metric_edge_filter import GraphMetricEdgeFilter +from .graph_metric_edge_filter_mode import GraphMetricEdgeFilterMode +from .graph_metric_edge_filter_status import GraphMetricEdgeFilterStatus +from .graph_metric_edge_filter_status_mode import GraphMetricEdgeFilterStatusMode +from .graph_metric_event import GraphMetricEvent +from .graph_metric_event_kind import GraphMetricEventKind +from .graph_metric_filter import GraphMetricFilter +from .graph_metric_filter_op import GraphMetricFilterOp +from .graph_metric_order import GraphMetricOrder +from .graph_metric_order_direction import GraphMetricOrderDirection +from .graph_metric_order_nulls import GraphMetricOrderNulls +from .graph_metric_profile import GraphMetricProfile +from .graph_metric_query import GraphMetricQuery +from .graph_metric_query_metric_freshness import GraphMetricQueryMetricFreshness +from .graph_metric_rerank import GraphMetricRerank +from .graph_metric_rerank_metric_freshness import GraphMetricRerankMetricFreshness +from .graph_metric_rerank_score_details import GraphMetricRerankScoreDetails +from .graph_metric_result import GraphMetricResult +from .graph_metric_runtime_stats import GraphMetricRuntimeStats +from .graph_metric_runtime_stats_role import GraphMetricRuntimeStatsRole +from .graph_metric_score import GraphMetricScore +from .graph_metric_status import GraphMetricStatus +from .graph_metric_status_phase import GraphMetricStatusPhase from .graph_nodes_result import GraphNodesResult from .graph_nodes_result_kind import GraphNodesResultKind +from .graph_nodes_result_metric_status import GraphNodesResultMetricStatus from .graph_not_equal_predicate import GraphNotEqualPredicate from .graph_not_exists_pattern import GraphNotExistsPattern from .graph_optional_match import GraphOptionalMatch @@ -440,6 +475,7 @@ from .graph_result_node import GraphResultNode from .graph_result_node_document import GraphResultNodeDocument from .graph_result_node_evidence import GraphResultNodeEvidence +from .graph_result_node_metrics import GraphResultNodeMetrics from .graph_result_ref_node_selector import GraphResultRefNodeSelector from .graph_result_row import GraphResultRow from .graph_result_stats import GraphResultStats @@ -448,6 +484,7 @@ from .graph_shortest_path import GraphShortestPath from .graph_shortest_path_query import GraphShortestPathQuery from .graph_traversal import GraphTraversal +from .graph_traversal_metric_freshness import GraphTraversalMetricFreshness from .graph_traverse_query import GraphTraverseQuery from .graph_where_and import GraphWhereAnd from .graph_where_not_equal import GraphWhereNotEqual @@ -680,11 +717,13 @@ from .legacy_graph_document_query import LegacyGraphDocumentQuery from .legacy_graph_node_selector import LegacyGraphNodeSelector from .legacy_graph_query import LegacyGraphQuery +from .legacy_graph_query_metric_freshness import LegacyGraphQueryMetricFreshness from .legacy_graph_result_node import LegacyGraphResultNode from .legacy_graph_result_node_document import LegacyGraphResultNodeDocument from .legacy_graph_result_node_evidence import LegacyGraphResultNodeEvidence from .legacy_graph_search_result import LegacyGraphSearchResult from .legacy_graph_search_result_kind import LegacyGraphSearchResultKind +from .legacy_graph_search_result_metric_status import LegacyGraphSearchResultMetricStatus from .linear_merge_page_status import LinearMergePageStatus from .linear_merge_request import LinearMergeRequest from .linear_merge_request_records import LinearMergeRequestRecords @@ -802,6 +841,8 @@ from .query_result_base import QueryResultBase from .query_result_base_aggregations import QueryResultBaseAggregations from .query_result_base_analyses import QueryResultBaseAnalyses +from .query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults +from .query_score_details import QueryScoreDetails from .query_strategy import QueryStrategy from .query_string_query import QueryStringQuery from .query_temporarily_unavailable_error import QueryTemporarilyUnavailableError @@ -1074,6 +1115,7 @@ "BackupRequest", "BackupRequestFormat", "BackupTableResponse201", + "BatchCommittedFailure", "BatchRequest", "BatchRequestInserts", "BatchRequestInsertsAdditionalProperty", @@ -1154,6 +1196,7 @@ "CreatedGraphArtifactSourceConfigFormat", "CreatedGraphIndex", "CreatedGraphIndexConfig", + "CreatedGraphIndexConfigMetrics", "CreatedGraphIndexType", "CreatedIndexCommon", "CreatedProviderConfig", @@ -1258,6 +1301,7 @@ "ExactSortErrorStatus", "ExaSearchConfig", "ExaSearchConfigSearchType", + "ExecuteGraphMetricActionAction", "ExecutionPolicy", "ExtensionError", "ExtensionMember", @@ -1395,6 +1439,7 @@ "GraphExactResultStats", "GraphIdentityNodeSelector", "GraphIndexConfig", + "GraphIndexConfigMetrics", "GraphIndexStats", "GraphIndexStatsAlgebraicGraph", "GraphIndexStatsAlgebraicGraphTraversal", @@ -1416,8 +1461,39 @@ "GraphMatchOperationLimitExceededErrorError", "GraphMatchOperationLimitExceededErrorStatus", "GraphMatchQuery", + "GraphMetricActionResponse", + "GraphMetricBuildPageStatus", + "GraphMetricBuildPageStatusRangeKind", + "GraphMetricBuildPageStatusState", + "GraphMetricConfig", + "GraphMetricConfigKind", + "GraphMetricConfigRefresh", + "GraphMetricEdgeFilter", + "GraphMetricEdgeFilterMode", + "GraphMetricEdgeFilterStatus", + "GraphMetricEdgeFilterStatusMode", + "GraphMetricEvent", + "GraphMetricEventKind", + "GraphMetricFilter", + "GraphMetricFilterOp", + "GraphMetricOrder", + "GraphMetricOrderDirection", + "GraphMetricOrderNulls", + "GraphMetricProfile", + "GraphMetricQuery", + "GraphMetricQueryMetricFreshness", + "GraphMetricRerank", + "GraphMetricRerankMetricFreshness", + "GraphMetricRerankScoreDetails", + "GraphMetricResult", + "GraphMetricRuntimeStats", + "GraphMetricRuntimeStatsRole", + "GraphMetricScore", + "GraphMetricStatus", + "GraphMetricStatusPhase", "GraphNodesResult", "GraphNodesResultKind", + "GraphNodesResultMetricStatus", "GraphNotEqualPredicate", "GraphNotExistsPattern", "GraphOptionalMatch", @@ -1452,6 +1528,7 @@ "GraphResultNode", "GraphResultNodeDocument", "GraphResultNodeEvidence", + "GraphResultNodeMetrics", "GraphResultRefNodeSelector", "GraphResultRow", "GraphResultStats", @@ -1460,6 +1537,7 @@ "GraphShortestPath", "GraphShortestPathQuery", "GraphTraversal", + "GraphTraversalMetricFreshness", "GraphTraverseQuery", "GraphWhereAnd", "GraphWhereNotEqual", @@ -1690,11 +1768,13 @@ "LegacyGraphDocumentQuery", "LegacyGraphNodeSelector", "LegacyGraphQuery", + "LegacyGraphQueryMetricFreshness", "LegacyGraphResultNode", "LegacyGraphResultNodeDocument", "LegacyGraphResultNodeEvidence", "LegacyGraphSearchResult", "LegacyGraphSearchResultKind", + "LegacyGraphSearchResultMetricStatus", "LinearMergePageStatus", "LinearMergeRequest", "LinearMergeRequestRecords", @@ -1810,6 +1890,8 @@ "QueryResultBase", "QueryResultBaseAggregations", "QueryResultBaseAnalyses", + "QueryResultBaseGraphMetricResults", + "QueryScoreDetails", "QueryStrategy", "QueryStringQuery", "QueryTemporarilyUnavailableError", diff --git a/py/packages/sdk/src/antfly/client_generated/models/batch_committed_failure.py b/py/packages/sdk/src/antfly/client_generated/models/batch_committed_failure.py new file mode 100644 index 0000000000..2d26adcd65 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/batch_committed_failure.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="BatchCommittedFailure") + + +@_attrs_define +class BatchCommittedFailure: + """Additive details for a committed batch that needs operator action. The + open string code is forward-compatible with older SDKs; clients should + treat unknown codes as non-retryable when `retryable` is false. + + Attributes: + code (str): Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + message (str): Actionable operator guidance. + retryable (bool): Whether replaying the document mutation is safe. Committed repair outcomes are false. + reason (str | Unset): Optional stable reason within the failure category, such as `build_budget_exceeded`. + """ + + code: str + message: str + retryable: bool + reason: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + code = self.code + + message = self.message + + retryable = self.retryable + + reason = self.reason + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "code": code, + "message": message, + "retryable": retryable, + } + ) + if reason is not UNSET: + field_dict["reason"] = reason + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + code = d.pop("code") + + message = d.pop("message") + + retryable = d.pop("retryable") + + reason = d.pop("reason", UNSET) + + batch_committed_failure = cls( + code=code, + message=message, + retryable=retryable, + reason=reason, + ) + + batch_committed_failure.additional_properties = d + return batch_committed_failure + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/batch_response.py b/py/packages/sdk/src/antfly/client_generated/models/batch_response.py index f9989419dc..6696ce3310 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/batch_response.py +++ b/py/packages/sdk/src/antfly/client_generated/models/batch_response.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from attrs import define as _attrs_define from attrs import field as _attrs_field @@ -9,6 +9,10 @@ from ..models.batch_response_status import BatchResponseStatus from ..types import UNSET, Unset +if TYPE_CHECKING: + from ..models.batch_committed_failure import BatchCommittedFailure + + T = TypeVar("T", bound="BatchResponse") @@ -18,17 +22,22 @@ class BatchResponse: Attributes: status (BatchResponseStatus | Unset): Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` - means the primary write committed, but a terminal enrichment failure needs - operator repair and will not be retried indefinitely. + means the primary write committed, but a terminal background materialization + failure needs operator repair and will not be retried indefinitely. Inspect + `failure` when present; retrying the document write is unnecessary. inserted (int | Unset): Number of documents successfully inserted deleted (int | Unset): Number of documents successfully deleted transformed (int | Unset): Number of documents successfully transformed + failure (BatchCommittedFailure | Unset): Additive details for a committed batch that needs operator action. The + open string code is forward-compatible with older SDKs; clients should + treat unknown codes as non-retryable when `retryable` is false. """ status: BatchResponseStatus | Unset = UNSET inserted: int | Unset = UNSET deleted: int | Unset = UNSET transformed: int | Unset = UNSET + failure: BatchCommittedFailure | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -42,6 +51,10 @@ def to_dict(self) -> dict[str, Any]: transformed = self.transformed + failure: dict[str, Any] | Unset = UNSET + if not isinstance(self.failure, Unset): + failure = self.failure.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -53,11 +66,15 @@ def to_dict(self) -> dict[str, Any]: field_dict["deleted"] = deleted if transformed is not UNSET: field_dict["transformed"] = transformed + if failure is not UNSET: + field_dict["failure"] = failure return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.batch_committed_failure import BatchCommittedFailure + d = dict(src_dict) _status = d.pop("status", UNSET) status: BatchResponseStatus | Unset @@ -72,11 +89,19 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: transformed = d.pop("transformed", UNSET) + _failure = d.pop("failure", UNSET) + failure: BatchCommittedFailure | Unset + if isinstance(_failure, Unset): + failure = UNSET + else: + failure = BatchCommittedFailure.from_dict(_failure) + batch_response = cls( status=status, inserted=inserted, deleted=deleted, transformed=transformed, + failure=failure, ) batch_response.additional_properties = d diff --git a/py/packages/sdk/src/antfly/client_generated/models/create_graph_index_request.py b/py/packages/sdk/src/antfly/client_generated/models/create_graph_index_request.py index 2e2d21fd70..6d66078771 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/create_graph_index_request.py +++ b/py/packages/sdk/src/antfly/client_generated/models/create_graph_index_request.py @@ -16,6 +16,7 @@ from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig from ..models.graph_artifact_producer_config import GraphArtifactProducerConfig from ..models.graph_artifact_source_config import GraphArtifactSourceConfig + from ..models.graph_index_config_metrics import GraphIndexConfigMetrics from ..models.graph_resolver_config import GraphResolverConfig @@ -31,6 +32,9 @@ class CreateGraphIndexRequest: description (str | Unset): Optional description of the index and its purpose version (int | Unset): Version of the index implementation. Defaults to 0. Default: 0. enrichments (list[EnrichmentConfig] | Unset): Inline managed enrichment definitions required by this index. + metrics (GraphIndexConfigMetrics | Unset): Named published graph metrics. Serverless supports background refresh + only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 + UTF-8 bytes per metric name. sources (list[GraphArtifactSourceConfig] | Unset): Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires @@ -60,6 +64,7 @@ class CreateGraphIndexRequest: description: str | Unset = UNSET version: int | Unset = 0 enrichments: list[EnrichmentConfig] | Unset = UNSET + metrics: GraphIndexConfigMetrics | Unset = UNSET sources: list[GraphArtifactSourceConfig] | Unset = UNSET summarizer: GeneratorConfig | Unset = UNSET template: str | Unset = UNSET @@ -85,6 +90,10 @@ def to_dict(self) -> dict[str, Any]: enrichments_item = enrichments_item_data.to_dict() enrichments.append(enrichments_item) + metrics: dict[str, Any] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics.to_dict() + sources: list[dict[str, Any]] | Unset = UNSET if not isinstance(self.sources, Unset): sources = [] @@ -139,6 +148,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["version"] = version if enrichments is not UNSET: field_dict["enrichments"] = enrichments + if metrics is not UNSET: + field_dict["metrics"] = metrics if sources is not UNSET: field_dict["sources"] = sources if summarizer is not UNSET: @@ -168,6 +179,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig from ..models.graph_artifact_producer_config import GraphArtifactProducerConfig from ..models.graph_artifact_source_config import GraphArtifactSourceConfig + from ..models.graph_index_config_metrics import GraphIndexConfigMetrics from ..models.graph_resolver_config import GraphResolverConfig d = dict(src_dict) @@ -186,6 +198,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: enrichments.append(enrichments_item) + _metrics = d.pop("metrics", UNSET) + metrics: GraphIndexConfigMetrics | Unset + if isinstance(_metrics, Unset): + metrics = UNSET + else: + metrics = GraphIndexConfigMetrics.from_dict(_metrics) + _sources = d.pop("sources", UNSET) sources: list[GraphArtifactSourceConfig] | Unset = UNSET if _sources is not UNSET: @@ -250,6 +269,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: description=description, version=version, enrichments=enrichments, + metrics=metrics, sources=sources, summarizer=summarizer, template=template, diff --git a/py/packages/sdk/src/antfly/client_generated/models/created_graph_index.py b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index.py index 38f335f424..8d75db87fc 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/created_graph_index.py +++ b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index.py @@ -13,6 +13,7 @@ from ..models.created_enrichment_config import CreatedEnrichmentConfig from ..models.created_graph_artifact_producer_config import CreatedGraphArtifactProducerConfig from ..models.created_graph_artifact_source_config import CreatedGraphArtifactSourceConfig + from ..models.created_graph_index_config_metrics import CreatedGraphIndexConfigMetrics from ..models.created_provider_config import CreatedProviderConfig from ..models.edge_type_config import EdgeTypeConfig from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig @@ -33,6 +34,7 @@ class CreatedGraphIndex: version (int | Unset): Version of the index implementation. Defaults to 0. Default: 0. enrichments (list[CreatedEnrichmentConfig] | Unset): Normalized inline managed enrichment definitions required by this index. + metrics (CreatedGraphIndexConfigMetrics | Unset): summarizer (CreatedProviderConfig | Unset): Credential-free provider configuration returned after index creation. Only non-secret provider settings are represented. template (str | Unset): @@ -53,6 +55,7 @@ class CreatedGraphIndex: description: str | Unset = UNSET version: int | Unset = 0 enrichments: list[CreatedEnrichmentConfig] | Unset = UNSET + metrics: CreatedGraphIndexConfigMetrics | Unset = UNSET summarizer: CreatedProviderConfig | Unset = UNSET template: str | Unset = UNSET edge_types: list[EdgeTypeConfig] | Unset = UNSET @@ -79,6 +82,10 @@ def to_dict(self) -> dict[str, Any]: enrichments_item = enrichments_item_data.to_dict() enrichments.append(enrichments_item) + metrics: dict[str, Any] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics.to_dict() + summarizer: dict[str, Any] | Unset = UNSET if not isinstance(self.summarizer, Unset): summarizer = self.summarizer.to_dict() @@ -130,6 +137,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["version"] = version if enrichments is not UNSET: field_dict["enrichments"] = enrichments + if metrics is not UNSET: + field_dict["metrics"] = metrics if summarizer is not UNSET: field_dict["summarizer"] = summarizer if template is not UNSET: @@ -154,6 +163,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.created_enrichment_config import CreatedEnrichmentConfig from ..models.created_graph_artifact_producer_config import CreatedGraphArtifactProducerConfig from ..models.created_graph_artifact_source_config import CreatedGraphArtifactSourceConfig + from ..models.created_graph_index_config_metrics import CreatedGraphIndexConfigMetrics from ..models.created_provider_config import CreatedProviderConfig from ..models.edge_type_config import EdgeTypeConfig from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig @@ -177,6 +187,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: enrichments.append(enrichments_item) + _metrics = d.pop("metrics", UNSET) + metrics: CreatedGraphIndexConfigMetrics | Unset + if isinstance(_metrics, Unset): + metrics = UNSET + else: + metrics = CreatedGraphIndexConfigMetrics.from_dict(_metrics) + _summarizer = d.pop("summarizer", UNSET) summarizer: CreatedProviderConfig | Unset if isinstance(_summarizer, Unset): @@ -235,6 +252,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: description=description, version=version, enrichments=enrichments, + metrics=metrics, summarizer=summarizer, template=template, edge_types=edge_types, diff --git a/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config.py b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config.py index c67121cdd1..5804705332 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config.py +++ b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from ..models.created_graph_artifact_producer_config import CreatedGraphArtifactProducerConfig from ..models.created_graph_artifact_source_config import CreatedGraphArtifactSourceConfig + from ..models.created_graph_index_config_metrics import CreatedGraphIndexConfigMetrics from ..models.created_provider_config import CreatedProviderConfig from ..models.edge_type_config import EdgeTypeConfig from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig @@ -25,6 +26,7 @@ class CreatedGraphIndexConfig: """Credential-free normalized graph configuration returned after creation. Attributes: + metrics (CreatedGraphIndexConfigMetrics | Unset): summarizer (CreatedProviderConfig | Unset): Credential-free provider configuration returned after index creation. Only non-secret provider settings are represented. template (str | Unset): @@ -40,6 +42,7 @@ class CreatedGraphIndexConfig: resolvers (list[GraphResolverConfig] | Unset): """ + metrics: CreatedGraphIndexConfigMetrics | Unset = UNSET summarizer: CreatedProviderConfig | Unset = UNSET template: str | Unset = UNSET edge_types: list[EdgeTypeConfig] | Unset = UNSET @@ -51,6 +54,10 @@ class CreatedGraphIndexConfig: additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: + metrics: dict[str, Any] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics.to_dict() + summarizer: dict[str, Any] | Unset = UNSET if not isinstance(self.summarizer, Unset): summarizer = self.summarizer.to_dict() @@ -91,6 +98,8 @@ def to_dict(self) -> dict[str, Any]: field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) + if metrics is not UNSET: + field_dict["metrics"] = metrics if summarizer is not UNSET: field_dict["summarizer"] = summarizer if template is not UNSET: @@ -114,12 +123,20 @@ def to_dict(self) -> dict[str, Any]: def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.created_graph_artifact_producer_config import CreatedGraphArtifactProducerConfig from ..models.created_graph_artifact_source_config import CreatedGraphArtifactSourceConfig + from ..models.created_graph_index_config_metrics import CreatedGraphIndexConfigMetrics from ..models.created_provider_config import CreatedProviderConfig from ..models.edge_type_config import EdgeTypeConfig from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig from ..models.graph_resolver_config import GraphResolverConfig d = dict(src_dict) + _metrics = d.pop("metrics", UNSET) + metrics: CreatedGraphIndexConfigMetrics | Unset + if isinstance(_metrics, Unset): + metrics = UNSET + else: + metrics = CreatedGraphIndexConfigMetrics.from_dict(_metrics) + _summarizer = d.pop("summarizer", UNSET) summarizer: CreatedProviderConfig | Unset if isinstance(_summarizer, Unset): @@ -173,6 +190,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: resolvers.append(resolvers_item) created_graph_index_config = cls( + metrics=metrics, summarizer=summarizer, template=template, edge_types=edge_types, diff --git a/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config_metrics.py b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config_metrics.py new file mode 100644 index 0000000000..28f6617c6d --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/created_graph_index_config_metrics.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_config import GraphMetricConfig + + +T = TypeVar("T", bound="CreatedGraphIndexConfigMetrics") + + +@_attrs_define +class CreatedGraphIndexConfigMetrics: + """ """ + + additional_properties: dict[str, GraphMetricConfig] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_config import GraphMetricConfig + + d = dict(src_dict) + created_graph_index_config_metrics = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = GraphMetricConfig.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + created_graph_index_config_metrics.additional_properties = additional_properties + return created_graph_index_config_metrics + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> GraphMetricConfig: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: GraphMetricConfig) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/execute_graph_metric_action_action.py b/py/packages/sdk/src/antfly/client_generated/models/execute_graph_metric_action_action.py new file mode 100644 index 0000000000..8806e2ffcf --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/execute_graph_metric_action_action.py @@ -0,0 +1,12 @@ +from enum import StrEnum + + +class ExecuteGraphMetricActionAction(StrEnum): + DELETE = "delete" + PAUSE = "pause" + REBUILD = "rebuild" + REFRESH = "refresh" + RESUME = "resume" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/global_stateful_query_request.py b/py/packages/sdk/src/antfly/client_generated/models/global_stateful_query_request.py index 7bc7e144e4..c3c7dc8128 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/global_stateful_query_request.py +++ b/py/packages/sdk/src/antfly/client_generated/models/global_stateful_query_request.py @@ -22,6 +22,8 @@ from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -298,6 +300,12 @@ class GlobalStatefulQueryRequest: Has minor performance overhead — not recommended for production traffic. reranker (RerankerConfig | Unset): A unified configuration for a reranking provider. Example: {'provider': 'cohere', 'model': 'rerank-v4.0-pro', 'field': 'content'}. + graph_metric (GraphMetricQuery | Unset): Reads a published graph metric. Score-bearing graph metric queries on + multi-shard tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + graph_metric_rerank (GraphMetricRerank | Unset): Blends a published graph metric into hit scores. Multi-shard + tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required. analyses (Analyses | Unset): graph_queries (GraphQueries | Unset): Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH @@ -502,6 +510,8 @@ class GlobalStatefulQueryRequest: count: bool | Unset = UNSET profile: bool | Unset = UNSET reranker: RerankerConfig | Unset = UNSET + graph_metric: GraphMetricQuery | Unset = UNSET + graph_metric_rerank: GraphMetricRerank | Unset = UNSET analyses: Analyses | Unset = UNSET graph_queries: GraphQueries | Unset = UNSET document_renderer: str | Unset = UNSET @@ -780,6 +790,14 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.reranker, Unset): reranker = self.reranker.to_dict() + graph_metric: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric, Unset): + graph_metric = self.graph_metric.to_dict() + + graph_metric_rerank: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_rerank, Unset): + graph_metric_rerank = self.graph_metric_rerank.to_dict() + analyses: dict[str, Any] | Unset = UNSET if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() @@ -869,6 +887,10 @@ def to_dict(self) -> dict[str, Any]: field_dict["profile"] = profile if reranker is not UNSET: field_dict["reranker"] = reranker + if graph_metric is not UNSET: + field_dict["graph_metric"] = graph_metric + if graph_metric_rerank is not UNSET: + field_dict["graph_metric_rerank"] = graph_metric_rerank if analyses is not UNSET: field_dict["analyses"] = analyses if graph_queries is not UNSET: @@ -902,6 +924,8 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -1739,6 +1763,20 @@ def _parse_exclusion_query( else: reranker = RerankerConfig.from_dict(_reranker) + _graph_metric = d.pop("graph_metric", UNSET) + graph_metric: GraphMetricQuery | Unset + if isinstance(_graph_metric, Unset): + graph_metric = UNSET + else: + graph_metric = GraphMetricQuery.from_dict(_graph_metric) + + _graph_metric_rerank = d.pop("graph_metric_rerank", UNSET) + graph_metric_rerank: GraphMetricRerank | Unset + if isinstance(_graph_metric_rerank, Unset): + graph_metric_rerank = UNSET + else: + graph_metric_rerank = GraphMetricRerank.from_dict(_graph_metric_rerank) + _analyses = d.pop("analyses", UNSET) analyses: Analyses | Unset if isinstance(_analyses, Unset): @@ -1818,6 +1856,8 @@ def _parse_exclusion_query( count=count, profile=profile, reranker=reranker, + graph_metric=graph_metric, + graph_metric_rerank=graph_metric_rerank, analyses=analyses, graph_queries=graph_queries, document_renderer=document_renderer, diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_index_config.py b/py/packages/sdk/src/antfly/client_generated/models/graph_index_config.py index a4cd7db7a4..f590ac14b6 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/graph_index_config.py +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_index_config.py @@ -14,6 +14,7 @@ from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig from ..models.graph_artifact_producer_config import GraphArtifactProducerConfig from ..models.graph_artifact_source_config import GraphArtifactSourceConfig + from ..models.graph_index_config_metrics import GraphIndexConfigMetrics from ..models.graph_resolver_config import GraphResolverConfig @@ -25,6 +26,9 @@ class GraphIndexConfig: """Configuration for graph index type Attributes: + metrics (GraphIndexConfigMetrics | Unset): Named published graph metrics. Serverless supports background refresh + only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 + UTF-8 bytes per metric name. sources (list[GraphArtifactSourceConfig] | Unset): Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires @@ -50,6 +54,7 @@ class GraphIndexConfig: resolvers (list[GraphResolverConfig] | Unset): """ + metrics: GraphIndexConfigMetrics | Unset = UNSET sources: list[GraphArtifactSourceConfig] | Unset = UNSET summarizer: GeneratorConfig | Unset = UNSET template: str | Unset = UNSET @@ -62,6 +67,10 @@ class GraphIndexConfig: additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: + metrics: dict[str, Any] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics.to_dict() + sources: list[dict[str, Any]] | Unset = UNSET if not isinstance(self.sources, Unset): sources = [] @@ -106,6 +115,8 @@ def to_dict(self) -> dict[str, Any]: field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) + if metrics is not UNSET: + field_dict["metrics"] = metrics if sources is not UNSET: field_dict["sources"] = sources if summarizer is not UNSET: @@ -134,9 +145,17 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.graph_algebraic_planning_config import GraphAlgebraicPlanningConfig from ..models.graph_artifact_producer_config import GraphArtifactProducerConfig from ..models.graph_artifact_source_config import GraphArtifactSourceConfig + from ..models.graph_index_config_metrics import GraphIndexConfigMetrics from ..models.graph_resolver_config import GraphResolverConfig d = dict(src_dict) + _metrics = d.pop("metrics", UNSET) + metrics: GraphIndexConfigMetrics | Unset + if isinstance(_metrics, Unset): + metrics = UNSET + else: + metrics = GraphIndexConfigMetrics.from_dict(_metrics) + _sources = d.pop("sources", UNSET) sources: list[GraphArtifactSourceConfig] | Unset = UNSET if _sources is not UNSET: @@ -197,6 +216,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: resolvers.append(resolvers_item) graph_index_config = cls( + metrics=metrics, sources=sources, summarizer=summarizer, template=template, diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_index_config_metrics.py b/py/packages/sdk/src/antfly/client_generated/models/graph_index_config_metrics.py new file mode 100644 index 0000000000..bb7bdbb1ea --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_index_config_metrics.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_config import GraphMetricConfig + + +T = TypeVar("T", bound="GraphIndexConfigMetrics") + + +@_attrs_define +class GraphIndexConfigMetrics: + """Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics + per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + + """ + + additional_properties: dict[str, GraphMetricConfig] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_config import GraphMetricConfig + + d = dict(src_dict) + graph_index_config_metrics = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = GraphMetricConfig.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + graph_index_config_metrics.additional_properties = additional_properties + return graph_index_config_metrics + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> GraphMetricConfig: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: GraphMetricConfig) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_index_stats.py b/py/packages/sdk/src/antfly/client_generated/models/graph_index_stats.py index dc6e406d4b..a7b234fc43 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/graph_index_stats.py +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_index_stats.py @@ -17,6 +17,7 @@ from ..models.graph_index_stats_resolution import GraphIndexStatsResolution from ..models.graph_index_stats_resolver_replay import GraphIndexStatsResolverReplay from ..models.graph_index_stats_source_artifact import GraphIndexStatsSourceArtifact + from ..models.graph_metric_runtime_stats import GraphMetricRuntimeStats from ..models.index_milestones import IndexMilestones from ..models.index_readiness_status import IndexReadinessStatus from ..models.index_repair_status import IndexRepairStatus @@ -90,6 +91,8 @@ class GraphIndexStats: promotion (GraphIndexStatsPromotion | Unset): Artifact promotion replay diagnostics. algebraic_graph (GraphIndexStatsAlgebraicGraph | Unset): Algebraic graph execution health for bounded semiring traversal. + graph_metric_runtime (GraphMetricRuntimeStats | Unset): Summarized graph metric maintenance runtime state. + Identity fields are stable hashes, not raw process or owner identifiers. """ index_type: GraphIndexStatsIndexType @@ -144,6 +147,7 @@ class GraphIndexStats: resolution: GraphIndexStatsResolution | Unset = UNSET promotion: GraphIndexStatsPromotion | Unset = UNSET algebraic_graph: GraphIndexStatsAlgebraicGraph | Unset = UNSET + graph_metric_runtime: GraphMetricRuntimeStats | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -271,6 +275,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.algebraic_graph, Unset): algebraic_graph = self.algebraic_graph.to_dict() + graph_metric_runtime: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_runtime, Unset): + graph_metric_runtime = self.graph_metric_runtime.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -380,6 +388,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["promotion"] = promotion if algebraic_graph is not UNSET: field_dict["algebraic_graph"] = algebraic_graph + if graph_metric_runtime is not UNSET: + field_dict["graph_metric_runtime"] = graph_metric_runtime return field_dict @@ -392,6 +402,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.graph_index_stats_resolution import GraphIndexStatsResolution from ..models.graph_index_stats_resolver_replay import GraphIndexStatsResolverReplay from ..models.graph_index_stats_source_artifact import GraphIndexStatsSourceArtifact + from ..models.graph_metric_runtime_stats import GraphMetricRuntimeStats from ..models.index_milestones import IndexMilestones from ..models.index_readiness_status import IndexReadinessStatus from ..models.index_repair_status import IndexRepairStatus @@ -551,6 +562,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: algebraic_graph = GraphIndexStatsAlgebraicGraph.from_dict(_algebraic_graph) + _graph_metric_runtime = d.pop("graph_metric_runtime", UNSET) + graph_metric_runtime: GraphMetricRuntimeStats | Unset + if isinstance(_graph_metric_runtime, Unset): + graph_metric_runtime = UNSET + else: + graph_metric_runtime = GraphMetricRuntimeStats.from_dict(_graph_metric_runtime) + graph_index_stats = cls( index_type=index_type, readiness=readiness, @@ -604,6 +622,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: resolution=resolution, promotion=promotion, algebraic_graph=algebraic_graph, + graph_metric_runtime=graph_metric_runtime, ) graph_index_stats.additional_properties = d diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_action_response.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_action_response.py new file mode 100644 index 0000000000..bf89d575c2 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_action_response.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_status import GraphMetricStatus + + +T = TypeVar("T", bound="GraphMetricActionResponse") + + +@_attrs_define +class GraphMetricActionResponse: + """ + Attributes: + status (GraphMetricStatus): + """ + + status: GraphMetricStatus + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + status = self.status.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "status": status, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_status import GraphMetricStatus + + d = dict(src_dict) + status = GraphMetricStatus.from_dict(d.pop("status")) + + graph_metric_action_response = cls( + status=status, + ) + + graph_metric_action_response.additional_properties = d + return graph_metric_action_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status.py new file mode 100644 index 0000000000..d9b4fb52fd --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_build_page_status_range_kind import GraphMetricBuildPageStatusRangeKind +from ..models.graph_metric_build_page_status_state import GraphMetricBuildPageStatusState +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricBuildPageStatus") + + +@_attrs_define +class GraphMetricBuildPageStatus: + """ + Attributes: + phase (str): + iteration (int): + page_id (int): + state (GraphMetricBuildPageStatusState): + range_kind (GraphMetricBuildPageStatusRangeKind): + worker_id (str | Unset): Worker id that owns or last failed this page. + lease_expires_at_ms (int | Unset): Unix epoch milliseconds when the page lease expires, or 0 when not leased. + attempt (int | Unset): Current attempt number for this page. + cursor (str | Unset): Opaque resumable cursor for this page. + completed_units (int | Unset): Completed work units for this page. + total_units (int | Unset): Estimated total work units for this page. + last_error (str | Unset): Last page-level error. + """ + + phase: str + iteration: int + page_id: int + state: GraphMetricBuildPageStatusState + range_kind: GraphMetricBuildPageStatusRangeKind + worker_id: str | Unset = UNSET + lease_expires_at_ms: int | Unset = UNSET + attempt: int | Unset = UNSET + cursor: str | Unset = UNSET + completed_units: int | Unset = UNSET + total_units: int | Unset = UNSET + last_error: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + phase = self.phase + + iteration = self.iteration + + page_id = self.page_id + + state = self.state.value + + range_kind = self.range_kind.value + + worker_id = self.worker_id + + lease_expires_at_ms = self.lease_expires_at_ms + + attempt = self.attempt + + cursor = self.cursor + + completed_units = self.completed_units + + total_units = self.total_units + + last_error = self.last_error + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "phase": phase, + "iteration": iteration, + "page_id": page_id, + "state": state, + "range_kind": range_kind, + } + ) + if worker_id is not UNSET: + field_dict["worker_id"] = worker_id + if lease_expires_at_ms is not UNSET: + field_dict["lease_expires_at_ms"] = lease_expires_at_ms + if attempt is not UNSET: + field_dict["attempt"] = attempt + if cursor is not UNSET: + field_dict["cursor"] = cursor + if completed_units is not UNSET: + field_dict["completed_units"] = completed_units + if total_units is not UNSET: + field_dict["total_units"] = total_units + if last_error is not UNSET: + field_dict["last_error"] = last_error + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + phase = d.pop("phase") + + iteration = d.pop("iteration") + + page_id = d.pop("page_id") + + state = GraphMetricBuildPageStatusState(d.pop("state")) + + range_kind = GraphMetricBuildPageStatusRangeKind(d.pop("range_kind")) + + worker_id = d.pop("worker_id", UNSET) + + lease_expires_at_ms = d.pop("lease_expires_at_ms", UNSET) + + attempt = d.pop("attempt", UNSET) + + cursor = d.pop("cursor", UNSET) + + completed_units = d.pop("completed_units", UNSET) + + total_units = d.pop("total_units", UNSET) + + last_error = d.pop("last_error", UNSET) + + graph_metric_build_page_status = cls( + phase=phase, + iteration=iteration, + page_id=page_id, + state=state, + range_kind=range_kind, + worker_id=worker_id, + lease_expires_at_ms=lease_expires_at_ms, + attempt=attempt, + cursor=cursor, + completed_units=completed_units, + total_units=total_units, + last_error=last_error, + ) + + graph_metric_build_page_status.additional_properties = d + return graph_metric_build_page_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_range_kind.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_range_kind.py new file mode 100644 index 0000000000..7ee3a84b11 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_range_kind.py @@ -0,0 +1,14 @@ +from enum import StrEnum + + +class GraphMetricBuildPageStatusRangeKind(StrEnum): + CONTRIBUTIONS = "contributions" + FULL = "full" + JOB_CONTROL = "job_control" + NODES = "nodes" + REVERSE_EDGES = "reverse_edges" + SCORES = "scores" + SUMMARY = "summary" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_state.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_state.py new file mode 100644 index 0000000000..d0a27f2def --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_build_page_status_state.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class GraphMetricBuildPageStatusState(StrEnum): + COMPLETE = "complete" + FAILED = "failed" + LEASED = "leased" + PENDING = "pending" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config.py new file mode 100644 index 0000000000..13839c1c21 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define + +from ..models.graph_metric_config_kind import GraphMetricConfigKind +from ..models.graph_metric_config_refresh import GraphMetricConfigRefresh +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.graph_metric_edge_filter import GraphMetricEdgeFilter + + +T = TypeVar("T", bound="GraphMetricConfig") + + +@_attrs_define +class GraphMetricConfig: + """Published metric configuration. If kind is omitted, the metric name must be a supported kind. + + Attributes: + enabled (bool | Unset): Default: True. + kind (GraphMetricConfigKind | Unset): + refresh (GraphMetricConfigRefresh | Unset): Serverless accepts background only. Default: + GraphMetricConfigRefresh.BACKGROUND. + damping (float | Unset): Default: 0.85. + tolerance (float | Unset): Default: 1e-06. + max_iterations (int | Unset): Default: 50. + edge_filter (GraphMetricEdgeFilter | Unset): Omitting this object selects all edge types. A types list selects + only those types; mode and types cannot both be supplied. + """ + + enabled: bool | Unset = True + kind: GraphMetricConfigKind | Unset = UNSET + refresh: GraphMetricConfigRefresh | Unset = GraphMetricConfigRefresh.BACKGROUND + damping: float | Unset = 0.85 + tolerance: float | Unset = 1e-06 + max_iterations: int | Unset = 50 + edge_filter: GraphMetricEdgeFilter | Unset = UNSET + + def to_dict(self) -> dict[str, Any]: + enabled = self.enabled + + kind: str | Unset = UNSET + if not isinstance(self.kind, Unset): + kind = self.kind.value + + refresh: str | Unset = UNSET + if not isinstance(self.refresh, Unset): + refresh = self.refresh.value + + damping = self.damping + + tolerance = self.tolerance + + max_iterations = self.max_iterations + + edge_filter: dict[str, Any] | Unset = UNSET + if not isinstance(self.edge_filter, Unset): + edge_filter = self.edge_filter.to_dict() + + field_dict: dict[str, Any] = {} + + field_dict.update({}) + if enabled is not UNSET: + field_dict["enabled"] = enabled + if kind is not UNSET: + field_dict["kind"] = kind + if refresh is not UNSET: + field_dict["refresh"] = refresh + if damping is not UNSET: + field_dict["damping"] = damping + if tolerance is not UNSET: + field_dict["tolerance"] = tolerance + if max_iterations is not UNSET: + field_dict["max_iterations"] = max_iterations + if edge_filter is not UNSET: + field_dict["edge_filter"] = edge_filter + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_edge_filter import GraphMetricEdgeFilter + + d = dict(src_dict) + enabled = d.pop("enabled", UNSET) + + _kind = d.pop("kind", UNSET) + kind: GraphMetricConfigKind | Unset + if isinstance(_kind, Unset): + kind = UNSET + else: + kind = GraphMetricConfigKind(_kind) + + _refresh = d.pop("refresh", UNSET) + refresh: GraphMetricConfigRefresh | Unset + if isinstance(_refresh, Unset): + refresh = UNSET + else: + refresh = GraphMetricConfigRefresh(_refresh) + + damping = d.pop("damping", UNSET) + + tolerance = d.pop("tolerance", UNSET) + + max_iterations = d.pop("max_iterations", UNSET) + + _edge_filter = d.pop("edge_filter", UNSET) + edge_filter: GraphMetricEdgeFilter | Unset + if isinstance(_edge_filter, Unset): + edge_filter = UNSET + else: + edge_filter = GraphMetricEdgeFilter.from_dict(_edge_filter) + + graph_metric_config = cls( + enabled=enabled, + kind=kind, + refresh=refresh, + damping=damping, + tolerance=tolerance, + max_iterations=max_iterations, + edge_filter=edge_filter, + ) + + return graph_metric_config diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_kind.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_kind.py new file mode 100644 index 0000000000..5826d5485b --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_kind.py @@ -0,0 +1,12 @@ +from enum import StrEnum + + +class GraphMetricConfigKind(StrEnum): + DEGREE = "degree" + EIGENVECTOR = "eigenvector" + HITS_AUTHORITY = "hits_authority" + HITS_HUB = "hits_hub" + PAGERANK = "pagerank" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_refresh.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_refresh.py new file mode 100644 index 0000000000..9b093d1892 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_config_refresh.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphMetricConfigRefresh(StrEnum): + BACKGROUND = "background" + MANUAL = "manual" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter.py new file mode 100644 index 0000000000..8658d6ec5a --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define + +from ..models.graph_metric_edge_filter_mode import GraphMetricEdgeFilterMode +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricEdgeFilter") + + +@_attrs_define +class GraphMetricEdgeFilter: + """Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be + supplied. + + Attributes: + mode (GraphMetricEdgeFilterMode | Unset): + types (list[str] | Unset): + """ + + mode: GraphMetricEdgeFilterMode | Unset = UNSET + types: list[str] | Unset = UNSET + + def to_dict(self) -> dict[str, Any]: + mode: str | Unset = UNSET + if not isinstance(self.mode, Unset): + mode = self.mode.value + + types: list[str] | Unset = UNSET + if not isinstance(self.types, Unset): + types = self.types + + field_dict: dict[str, Any] = {} + + field_dict.update({}) + if mode is not UNSET: + field_dict["mode"] = mode + if types is not UNSET: + field_dict["types"] = types + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + _mode = d.pop("mode", UNSET) + mode: GraphMetricEdgeFilterMode | Unset + if isinstance(_mode, Unset): + mode = UNSET + else: + mode = GraphMetricEdgeFilterMode(_mode) + + types = cast(list[str], d.pop("types", UNSET)) + + graph_metric_edge_filter = cls( + mode=mode, + types=types, + ) + + return graph_metric_edge_filter diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_mode.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_mode.py new file mode 100644 index 0000000000..ddd612c3bc --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_mode.py @@ -0,0 +1,8 @@ +from enum import StrEnum + + +class GraphMetricEdgeFilterMode(StrEnum): + ALL = "all" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status.py new file mode 100644 index 0000000000..013eab90be --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_edge_filter_status_mode import GraphMetricEdgeFilterStatusMode +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricEdgeFilterStatus") + + +@_attrs_define +class GraphMetricEdgeFilterStatus: + """ + Attributes: + mode (GraphMetricEdgeFilterStatusMode): + types (list[str] | Unset): + """ + + mode: GraphMetricEdgeFilterStatusMode + types: list[str] | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + mode = self.mode.value + + types: list[str] | Unset = UNSET + if not isinstance(self.types, Unset): + types = self.types + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "mode": mode, + } + ) + if types is not UNSET: + field_dict["types"] = types + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + mode = GraphMetricEdgeFilterStatusMode(d.pop("mode")) + + types = cast(list[str], d.pop("types", UNSET)) + + graph_metric_edge_filter_status = cls( + mode=mode, + types=types, + ) + + graph_metric_edge_filter_status.additional_properties = d + return graph_metric_edge_filter_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status_mode.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status_mode.py new file mode 100644 index 0000000000..715114caee --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_edge_filter_status_mode.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphMetricEdgeFilterStatusMode(StrEnum): + ALL = "all" + TYPES = "types" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event.py new file mode 100644 index 0000000000..47f68445f5 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_event_kind import GraphMetricEventKind + +T = TypeVar("T", bound="GraphMetricEvent") + + +@_attrs_define +class GraphMetricEvent: + """ + Attributes: + sequence (int): + kind (GraphMetricEventKind): + at_ms (int): + target_edge_generation (int): + published_generation (int): + score_count (int): + """ + + sequence: int + kind: GraphMetricEventKind + at_ms: int + target_edge_generation: int + published_generation: int + score_count: int + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + sequence = self.sequence + + kind = self.kind.value + + at_ms = self.at_ms + + target_edge_generation = self.target_edge_generation + + published_generation = self.published_generation + + score_count = self.score_count + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "sequence": sequence, + "kind": kind, + "at_ms": at_ms, + "target_edge_generation": target_edge_generation, + "published_generation": published_generation, + "score_count": score_count, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + sequence = d.pop("sequence") + + kind = GraphMetricEventKind(d.pop("kind")) + + at_ms = d.pop("at_ms") + + target_edge_generation = d.pop("target_edge_generation") + + published_generation = d.pop("published_generation") + + score_count = d.pop("score_count") + + graph_metric_event = cls( + sequence=sequence, + kind=kind, + at_ms=at_ms, + target_edge_generation=target_edge_generation, + published_generation=published_generation, + score_count=score_count, + ) + + graph_metric_event.additional_properties = d + return graph_metric_event + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event_kind.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event_kind.py new file mode 100644 index 0000000000..105074ab14 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_event_kind.py @@ -0,0 +1,12 @@ +from enum import StrEnum + + +class GraphMetricEventKind(StrEnum): + DELETE = "delete" + FAILED = "failed" + PAUSE = "pause" + PUBLISH = "publish" + RESUME = "resume" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter.py new file mode 100644 index 0000000000..6b3f58f1ea --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_filter_op import GraphMetricFilterOp + +T = TypeVar("T", bound="GraphMetricFilter") + + +@_attrs_define +class GraphMetricFilter: + """ + Attributes: + metric (str): + op (GraphMetricFilterOp): Semantic comparison operator. Named values keep generated SDK enums portable and + readable. + value (float): + """ + + metric: str + op: GraphMetricFilterOp + value: float + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + metric = self.metric + + op = self.op.value + + value = self.value + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "metric": metric, + "op": op, + "value": value, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + metric = d.pop("metric") + + op = GraphMetricFilterOp(d.pop("op")) + + value = d.pop("value") + + graph_metric_filter = cls( + metric=metric, + op=op, + value=value, + ) + + graph_metric_filter.additional_properties = d + return graph_metric_filter + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter_op.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter_op.py new file mode 100644 index 0000000000..1f9e24f636 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_filter_op.py @@ -0,0 +1,13 @@ +from enum import StrEnum + + +class GraphMetricFilterOp(StrEnum): + EQ = "eq" + GT = "gt" + GTE = "gte" + LT = "lt" + LTE = "lte" + NEQ = "neq" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order.py new file mode 100644 index 0000000000..2cde368f78 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_order_direction import GraphMetricOrderDirection +from ..models.graph_metric_order_nulls import GraphMetricOrderNulls +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricOrder") + + +@_attrs_define +class GraphMetricOrder: + """ + Attributes: + metric (str): + direction (GraphMetricOrderDirection | Unset): + nulls (GraphMetricOrderNulls | Unset): + """ + + metric: str + direction: GraphMetricOrderDirection | Unset = UNSET + nulls: GraphMetricOrderNulls | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + metric = self.metric + + direction: str | Unset = UNSET + if not isinstance(self.direction, Unset): + direction = self.direction.value + + nulls: str | Unset = UNSET + if not isinstance(self.nulls, Unset): + nulls = self.nulls.value + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "metric": metric, + } + ) + if direction is not UNSET: + field_dict["direction"] = direction + if nulls is not UNSET: + field_dict["nulls"] = nulls + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + metric = d.pop("metric") + + _direction = d.pop("direction", UNSET) + direction: GraphMetricOrderDirection | Unset + if isinstance(_direction, Unset): + direction = UNSET + else: + direction = GraphMetricOrderDirection(_direction) + + _nulls = d.pop("nulls", UNSET) + nulls: GraphMetricOrderNulls | Unset + if isinstance(_nulls, Unset): + nulls = UNSET + else: + nulls = GraphMetricOrderNulls(_nulls) + + graph_metric_order = cls( + metric=metric, + direction=direction, + nulls=nulls, + ) + + graph_metric_order.additional_properties = d + return graph_metric_order + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_direction.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_direction.py new file mode 100644 index 0000000000..493ea44585 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_direction.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphMetricOrderDirection(StrEnum): + ASC = "asc" + DESC = "desc" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_nulls.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_nulls.py new file mode 100644 index 0000000000..e31857f943 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_order_nulls.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class GraphMetricOrderNulls(StrEnum): + FIRST = "first" + LAST = "last" + NULLS_FIRST = "nulls_first" + NULLS_LAST = "nulls_last" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_profile.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_profile.py new file mode 100644 index 0000000000..3e9ccd6cea --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_profile.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_status import GraphMetricStatus + + +T = TypeVar("T", bound="GraphMetricProfile") + + +@_attrs_define +class GraphMetricProfile: + """ + Attributes: + query_name (str): Name of the graph query or graph metric query that used the metric. + source (str): Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + index_name (str): Graph index that owns the metric. + metric_name (str): Graph metric name within the index. + freshness (str): Effective freshness mode requested for this metric use. + status (GraphMetricStatus): + """ + + query_name: str + source: str + index_name: str + metric_name: str + freshness: str + status: GraphMetricStatus + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + query_name = self.query_name + + source = self.source + + index_name = self.index_name + + metric_name = self.metric_name + + freshness = self.freshness + + status = self.status.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "query_name": query_name, + "source": source, + "index_name": index_name, + "metric_name": metric_name, + "freshness": freshness, + "status": status, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_status import GraphMetricStatus + + d = dict(src_dict) + query_name = d.pop("query_name") + + source = d.pop("source") + + index_name = d.pop("index_name") + + metric_name = d.pop("metric_name") + + freshness = d.pop("freshness") + + status = GraphMetricStatus.from_dict(d.pop("status")) + + graph_metric_profile = cls( + query_name=query_name, + source=source, + index_name=index_name, + metric_name=metric_name, + freshness=freshness, + status=status, + ) + + graph_metric_profile.additional_properties = d + return graph_metric_profile + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query.py new file mode 100644 index 0000000000..644acaffbe --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_query_metric_freshness import GraphMetricQueryMetricFreshness +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricQuery") + + +@_attrs_define +class GraphMetricQuery: + """Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally + coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging + mathematically incompatible shard-local scores. + + Attributes: + index (str): Graph index that owns the published metric. + metric (str): Graph metric to read. + name (str | Unset): Optional result key. Defaults to the metric name. + top_k (int | Unset): Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated + metric snapshot. Default: 10. + metric_freshness (GraphMetricQueryMetricFreshness | Unset): Whether the latest published generation may be stale + or must match the graph edge generation. Default: GraphMetricQueryMetricFreshness.PUBLISHED. + """ + + index: str + metric: str + name: str | Unset = UNSET + top_k: int | Unset = 10 + metric_freshness: GraphMetricQueryMetricFreshness | Unset = GraphMetricQueryMetricFreshness.PUBLISHED + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + index = self.index + + metric = self.metric + + name = self.name + + top_k = self.top_k + + metric_freshness: str | Unset = UNSET + if not isinstance(self.metric_freshness, Unset): + metric_freshness = self.metric_freshness.value + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "index": index, + "metric": metric, + } + ) + if name is not UNSET: + field_dict["name"] = name + if top_k is not UNSET: + field_dict["top_k"] = top_k + if metric_freshness is not UNSET: + field_dict["metric_freshness"] = metric_freshness + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + index = d.pop("index") + + metric = d.pop("metric") + + name = d.pop("name", UNSET) + + top_k = d.pop("top_k", UNSET) + + _metric_freshness = d.pop("metric_freshness", UNSET) + metric_freshness: GraphMetricQueryMetricFreshness | Unset + if isinstance(_metric_freshness, Unset): + metric_freshness = UNSET + else: + metric_freshness = GraphMetricQueryMetricFreshness(_metric_freshness) + + graph_metric_query = cls( + index=index, + metric=metric, + name=name, + top_k=top_k, + metric_freshness=metric_freshness, + ) + + graph_metric_query.additional_properties = d + return graph_metric_query + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query_metric_freshness.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query_metric_freshness.py new file mode 100644 index 0000000000..8c0b187260 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_query_metric_freshness.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphMetricQueryMetricFreshness(StrEnum): + FRESH = "fresh" + PUBLISHED = "published" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank.py new file mode 100644 index 0000000000..28bb2f0208 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_rerank_metric_freshness import GraphMetricRerankMetricFreshness +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricRerank") + + +@_attrs_define +class GraphMetricRerank: + """Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot + and otherwise return graph_metric_global_materialization_required. + + Attributes: + index (str): Graph index that owns the published metric. + metric (str): Graph metric name to blend into the search hit score. + candidate_count (int | Unset): Bounded retrieval window scored by the graph metric before offset and limit are + applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. An explicit + value must cover offset plus limit. Larger windows improve promotion recall at predictable linear score-read + cost. + base_weight (float | Unset): Multiplier applied to the existing hit score before adding the graph metric + feature. Default: 1.0. + weight (float | Unset): Multiplier applied to the graph metric score before it is added to the existing hit + score. Default: 1.0. + missing_score (float | Unset): Metric feature value to use for hits that do not have a score in the published + metric generation. Default: 0.0. + metric_freshness (GraphMetricRerankMetricFreshness | Unset): Whether stale published generations are acceptable + or the metric must be fresh. Default: GraphMetricRerankMetricFreshness.PUBLISHED. + """ + + index: str + metric: str + candidate_count: int | Unset = UNSET + base_weight: float | Unset = 1.0 + weight: float | Unset = 1.0 + missing_score: float | Unset = 0.0 + metric_freshness: GraphMetricRerankMetricFreshness | Unset = GraphMetricRerankMetricFreshness.PUBLISHED + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + index = self.index + + metric = self.metric + + candidate_count = self.candidate_count + + base_weight = self.base_weight + + weight = self.weight + + missing_score = self.missing_score + + metric_freshness: str | Unset = UNSET + if not isinstance(self.metric_freshness, Unset): + metric_freshness = self.metric_freshness.value + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "index": index, + "metric": metric, + } + ) + if candidate_count is not UNSET: + field_dict["candidate_count"] = candidate_count + if base_weight is not UNSET: + field_dict["base_weight"] = base_weight + if weight is not UNSET: + field_dict["weight"] = weight + if missing_score is not UNSET: + field_dict["missing_score"] = missing_score + if metric_freshness is not UNSET: + field_dict["metric_freshness"] = metric_freshness + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + index = d.pop("index") + + metric = d.pop("metric") + + candidate_count = d.pop("candidate_count", UNSET) + + base_weight = d.pop("base_weight", UNSET) + + weight = d.pop("weight", UNSET) + + missing_score = d.pop("missing_score", UNSET) + + _metric_freshness = d.pop("metric_freshness", UNSET) + metric_freshness: GraphMetricRerankMetricFreshness | Unset + if isinstance(_metric_freshness, Unset): + metric_freshness = UNSET + else: + metric_freshness = GraphMetricRerankMetricFreshness(_metric_freshness) + + graph_metric_rerank = cls( + index=index, + metric=metric, + candidate_count=candidate_count, + base_weight=base_weight, + weight=weight, + missing_score=missing_score, + metric_freshness=metric_freshness, + ) + + graph_metric_rerank.additional_properties = d + return graph_metric_rerank + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_metric_freshness.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_metric_freshness.py new file mode 100644 index 0000000000..613406df3d --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_metric_freshness.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphMetricRerankMetricFreshness(StrEnum): + FRESH = "fresh" + PUBLISHED = "published" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_score_details.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_score_details.py new file mode 100644 index 0000000000..5e22a959df --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_rerank_score_details.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricRerankScoreDetails") + + +@_attrs_define +class GraphMetricRerankScoreDetails: + """ + Attributes: + index_name (str): Graph index that provided the metric score. + metric_name (str): Graph metric used as a score feature. + base_score (float): Hit score before graph metric rerank composition. + base_weight (float): Weight applied to the base score. + metric_score_used (float): Metric feature value used in the formula after applying missing_score fallback if + needed. + metric_weight (float): Weight applied to the metric score feature. + missing_score_used (bool): True when metric_score was missing and the request's missing_score fallback was used. + final_score (float): Final hit score after graph metric rerank composition. + published_generation (int): Published graph metric score generation used for this hit. + metric_score (float | None | Unset): Published metric score for this hit, or null when the hit was missing from + the metric generation. + """ + + index_name: str + metric_name: str + base_score: float + base_weight: float + metric_score_used: float + metric_weight: float + missing_score_used: bool + final_score: float + published_generation: int + metric_score: float | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + index_name = self.index_name + + metric_name = self.metric_name + + base_score = self.base_score + + base_weight = self.base_weight + + metric_score_used = self.metric_score_used + + metric_weight = self.metric_weight + + missing_score_used = self.missing_score_used + + final_score = self.final_score + + published_generation = self.published_generation + + metric_score: float | None | Unset + if isinstance(self.metric_score, Unset): + metric_score = UNSET + else: + metric_score = self.metric_score + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "index_name": index_name, + "metric_name": metric_name, + "base_score": base_score, + "base_weight": base_weight, + "metric_score_used": metric_score_used, + "metric_weight": metric_weight, + "missing_score_used": missing_score_used, + "final_score": final_score, + "published_generation": published_generation, + } + ) + if metric_score is not UNSET: + field_dict["metric_score"] = metric_score + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + index_name = d.pop("index_name") + + metric_name = d.pop("metric_name") + + base_score = d.pop("base_score") + + base_weight = d.pop("base_weight") + + metric_score_used = d.pop("metric_score_used") + + metric_weight = d.pop("metric_weight") + + missing_score_used = d.pop("missing_score_used") + + final_score = d.pop("final_score") + + published_generation = d.pop("published_generation") + + def _parse_metric_score(data: object) -> float | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(float | None | Unset, data) + + metric_score = _parse_metric_score(d.pop("metric_score", UNSET)) + + graph_metric_rerank_score_details = cls( + index_name=index_name, + metric_name=metric_name, + base_score=base_score, + base_weight=base_weight, + metric_score_used=metric_score_used, + metric_weight=metric_weight, + missing_score_used=missing_score_used, + final_score=final_score, + published_generation=published_generation, + metric_score=metric_score, + ) + + graph_metric_rerank_score_details.additional_properties = d + return graph_metric_rerank_score_details + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_result.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_result.py new file mode 100644 index 0000000000..bd4dd35c83 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_result.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_score import GraphMetricScore + from ..models.graph_metric_status import GraphMetricStatus + + +T = TypeVar("T", bound="GraphMetricResult") + + +@_attrs_define +class GraphMetricResult: + """ + Attributes: + index_name (str): + metric (str): + scores (list[GraphMetricScore]): + status (GraphMetricStatus): + """ + + index_name: str + metric: str + scores: list[GraphMetricScore] + status: GraphMetricStatus + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + index_name = self.index_name + + metric = self.metric + + scores = [] + for scores_item_data in self.scores: + scores_item = scores_item_data.to_dict() + scores.append(scores_item) + + status = self.status.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "index_name": index_name, + "metric": metric, + "scores": scores, + "status": status, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_score import GraphMetricScore + from ..models.graph_metric_status import GraphMetricStatus + + d = dict(src_dict) + index_name = d.pop("index_name") + + metric = d.pop("metric") + + scores = [] + _scores = d.pop("scores") + for scores_item_data in _scores: + scores_item = GraphMetricScore.from_dict(scores_item_data) + + scores.append(scores_item) + + status = GraphMetricStatus.from_dict(d.pop("status")) + + graph_metric_result = cls( + index_name=index_name, + metric=metric, + scores=scores, + status=status, + ) + + graph_metric_result.additional_properties = d + return graph_metric_result + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats.py new file mode 100644 index 0000000000..af9b45a43f --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats.py @@ -0,0 +1,506 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_runtime_stats_role import GraphMetricRuntimeStatsRole +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GraphMetricRuntimeStats") + + +@_attrs_define +class GraphMetricRuntimeStats: + """Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner + identifiers. + + Attributes: + enabled (bool | Unset): + role (GraphMetricRuntimeStatsRole | Unset): + runtime_id_hash (int | Unset): + owner_id_hash (int | Unset): + lease_key_hash (int | Unset): + worker_id_hash (int | Unset): + worker_count (int | Unset): + lease_owned (bool | Unset): + has_lease (bool | Unset): + acquisition_count (int | Unset): + takeover_count (int | Unset): + lease_acquire_failures (int | Unset): + lost_leases (int | Unset): + last_acquired_ms (int | Unset): + lease_expires_at_ms (int | Unset): Cached expiry of the currently held maintenance lease, or zero when no lease + is held. + lease_renew_after_ms (int | Unset): Earliest time the runtime will renew its maintenance lease, or zero when no + lease is held. + renewal_count (int | Unset): Number of durable maintenance lease renewals completed by this runtime. + started (bool | Unset): + shutdown (bool | Unset): + notified (bool | Unset): + ticks_started (int | Unset): + ticks_completed (int | Unset): + durable_progress_ticks (int | Unset): + idle_ticks (int | Unset): + error_ticks (int | Unset): + last_error_name (str | Unset): + total_metrics_scanned (int | Unset): + total_active_builds (int | Unset): + total_builds_started (int | Unset): + total_worker_steps (int | Unset): + total_coordinator_steps (int | Unset): + total_retired_input_records (int | Unset): Consumed intermediate records retired at completed reduction + barriers. + total_pages_claimed (int | Unset): + total_pages_completed (int | Unset): + total_phases_advanced (int | Unset): + total_published (int | Unset): + total_failed_builds (int | Unset): + last_metrics_scanned (int | Unset): + last_active_builds (int | Unset): + last_builds_started (int | Unset): + last_worker_steps (int | Unset): + last_coordinator_steps (int | Unset): + last_retired_input_records (int | Unset): Consumed intermediate records retired in the latest maintenance tick. + last_pages_claimed (int | Unset): + last_pages_completed (int | Unset): + last_phases_advanced (int | Unset): + last_published (int | Unset): + last_failed_builds (int | Unset): + last_budget_exhausted (bool | Unset): + """ + + enabled: bool | Unset = UNSET + role: GraphMetricRuntimeStatsRole | Unset = UNSET + runtime_id_hash: int | Unset = UNSET + owner_id_hash: int | Unset = UNSET + lease_key_hash: int | Unset = UNSET + worker_id_hash: int | Unset = UNSET + worker_count: int | Unset = UNSET + lease_owned: bool | Unset = UNSET + has_lease: bool | Unset = UNSET + acquisition_count: int | Unset = UNSET + takeover_count: int | Unset = UNSET + lease_acquire_failures: int | Unset = UNSET + lost_leases: int | Unset = UNSET + last_acquired_ms: int | Unset = UNSET + lease_expires_at_ms: int | Unset = UNSET + lease_renew_after_ms: int | Unset = UNSET + renewal_count: int | Unset = UNSET + started: bool | Unset = UNSET + shutdown: bool | Unset = UNSET + notified: bool | Unset = UNSET + ticks_started: int | Unset = UNSET + ticks_completed: int | Unset = UNSET + durable_progress_ticks: int | Unset = UNSET + idle_ticks: int | Unset = UNSET + error_ticks: int | Unset = UNSET + last_error_name: str | Unset = UNSET + total_metrics_scanned: int | Unset = UNSET + total_active_builds: int | Unset = UNSET + total_builds_started: int | Unset = UNSET + total_worker_steps: int | Unset = UNSET + total_coordinator_steps: int | Unset = UNSET + total_retired_input_records: int | Unset = UNSET + total_pages_claimed: int | Unset = UNSET + total_pages_completed: int | Unset = UNSET + total_phases_advanced: int | Unset = UNSET + total_published: int | Unset = UNSET + total_failed_builds: int | Unset = UNSET + last_metrics_scanned: int | Unset = UNSET + last_active_builds: int | Unset = UNSET + last_builds_started: int | Unset = UNSET + last_worker_steps: int | Unset = UNSET + last_coordinator_steps: int | Unset = UNSET + last_retired_input_records: int | Unset = UNSET + last_pages_claimed: int | Unset = UNSET + last_pages_completed: int | Unset = UNSET + last_phases_advanced: int | Unset = UNSET + last_published: int | Unset = UNSET + last_failed_builds: int | Unset = UNSET + last_budget_exhausted: bool | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + enabled = self.enabled + + role: str | Unset = UNSET + if not isinstance(self.role, Unset): + role = self.role.value + + runtime_id_hash = self.runtime_id_hash + + owner_id_hash = self.owner_id_hash + + lease_key_hash = self.lease_key_hash + + worker_id_hash = self.worker_id_hash + + worker_count = self.worker_count + + lease_owned = self.lease_owned + + has_lease = self.has_lease + + acquisition_count = self.acquisition_count + + takeover_count = self.takeover_count + + lease_acquire_failures = self.lease_acquire_failures + + lost_leases = self.lost_leases + + last_acquired_ms = self.last_acquired_ms + + lease_expires_at_ms = self.lease_expires_at_ms + + lease_renew_after_ms = self.lease_renew_after_ms + + renewal_count = self.renewal_count + + started = self.started + + shutdown = self.shutdown + + notified = self.notified + + ticks_started = self.ticks_started + + ticks_completed = self.ticks_completed + + durable_progress_ticks = self.durable_progress_ticks + + idle_ticks = self.idle_ticks + + error_ticks = self.error_ticks + + last_error_name = self.last_error_name + + total_metrics_scanned = self.total_metrics_scanned + + total_active_builds = self.total_active_builds + + total_builds_started = self.total_builds_started + + total_worker_steps = self.total_worker_steps + + total_coordinator_steps = self.total_coordinator_steps + + total_retired_input_records = self.total_retired_input_records + + total_pages_claimed = self.total_pages_claimed + + total_pages_completed = self.total_pages_completed + + total_phases_advanced = self.total_phases_advanced + + total_published = self.total_published + + total_failed_builds = self.total_failed_builds + + last_metrics_scanned = self.last_metrics_scanned + + last_active_builds = self.last_active_builds + + last_builds_started = self.last_builds_started + + last_worker_steps = self.last_worker_steps + + last_coordinator_steps = self.last_coordinator_steps + + last_retired_input_records = self.last_retired_input_records + + last_pages_claimed = self.last_pages_claimed + + last_pages_completed = self.last_pages_completed + + last_phases_advanced = self.last_phases_advanced + + last_published = self.last_published + + last_failed_builds = self.last_failed_builds + + last_budget_exhausted = self.last_budget_exhausted + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if enabled is not UNSET: + field_dict["enabled"] = enabled + if role is not UNSET: + field_dict["role"] = role + if runtime_id_hash is not UNSET: + field_dict["runtime_id_hash"] = runtime_id_hash + if owner_id_hash is not UNSET: + field_dict["owner_id_hash"] = owner_id_hash + if lease_key_hash is not UNSET: + field_dict["lease_key_hash"] = lease_key_hash + if worker_id_hash is not UNSET: + field_dict["worker_id_hash"] = worker_id_hash + if worker_count is not UNSET: + field_dict["worker_count"] = worker_count + if lease_owned is not UNSET: + field_dict["lease_owned"] = lease_owned + if has_lease is not UNSET: + field_dict["has_lease"] = has_lease + if acquisition_count is not UNSET: + field_dict["acquisition_count"] = acquisition_count + if takeover_count is not UNSET: + field_dict["takeover_count"] = takeover_count + if lease_acquire_failures is not UNSET: + field_dict["lease_acquire_failures"] = lease_acquire_failures + if lost_leases is not UNSET: + field_dict["lost_leases"] = lost_leases + if last_acquired_ms is not UNSET: + field_dict["last_acquired_ms"] = last_acquired_ms + if lease_expires_at_ms is not UNSET: + field_dict["lease_expires_at_ms"] = lease_expires_at_ms + if lease_renew_after_ms is not UNSET: + field_dict["lease_renew_after_ms"] = lease_renew_after_ms + if renewal_count is not UNSET: + field_dict["renewal_count"] = renewal_count + if started is not UNSET: + field_dict["started"] = started + if shutdown is not UNSET: + field_dict["shutdown"] = shutdown + if notified is not UNSET: + field_dict["notified"] = notified + if ticks_started is not UNSET: + field_dict["ticks_started"] = ticks_started + if ticks_completed is not UNSET: + field_dict["ticks_completed"] = ticks_completed + if durable_progress_ticks is not UNSET: + field_dict["durable_progress_ticks"] = durable_progress_ticks + if idle_ticks is not UNSET: + field_dict["idle_ticks"] = idle_ticks + if error_ticks is not UNSET: + field_dict["error_ticks"] = error_ticks + if last_error_name is not UNSET: + field_dict["last_error_name"] = last_error_name + if total_metrics_scanned is not UNSET: + field_dict["total_metrics_scanned"] = total_metrics_scanned + if total_active_builds is not UNSET: + field_dict["total_active_builds"] = total_active_builds + if total_builds_started is not UNSET: + field_dict["total_builds_started"] = total_builds_started + if total_worker_steps is not UNSET: + field_dict["total_worker_steps"] = total_worker_steps + if total_coordinator_steps is not UNSET: + field_dict["total_coordinator_steps"] = total_coordinator_steps + if total_retired_input_records is not UNSET: + field_dict["total_retired_input_records"] = total_retired_input_records + if total_pages_claimed is not UNSET: + field_dict["total_pages_claimed"] = total_pages_claimed + if total_pages_completed is not UNSET: + field_dict["total_pages_completed"] = total_pages_completed + if total_phases_advanced is not UNSET: + field_dict["total_phases_advanced"] = total_phases_advanced + if total_published is not UNSET: + field_dict["total_published"] = total_published + if total_failed_builds is not UNSET: + field_dict["total_failed_builds"] = total_failed_builds + if last_metrics_scanned is not UNSET: + field_dict["last_metrics_scanned"] = last_metrics_scanned + if last_active_builds is not UNSET: + field_dict["last_active_builds"] = last_active_builds + if last_builds_started is not UNSET: + field_dict["last_builds_started"] = last_builds_started + if last_worker_steps is not UNSET: + field_dict["last_worker_steps"] = last_worker_steps + if last_coordinator_steps is not UNSET: + field_dict["last_coordinator_steps"] = last_coordinator_steps + if last_retired_input_records is not UNSET: + field_dict["last_retired_input_records"] = last_retired_input_records + if last_pages_claimed is not UNSET: + field_dict["last_pages_claimed"] = last_pages_claimed + if last_pages_completed is not UNSET: + field_dict["last_pages_completed"] = last_pages_completed + if last_phases_advanced is not UNSET: + field_dict["last_phases_advanced"] = last_phases_advanced + if last_published is not UNSET: + field_dict["last_published"] = last_published + if last_failed_builds is not UNSET: + field_dict["last_failed_builds"] = last_failed_builds + if last_budget_exhausted is not UNSET: + field_dict["last_budget_exhausted"] = last_budget_exhausted + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + enabled = d.pop("enabled", UNSET) + + _role = d.pop("role", UNSET) + role: GraphMetricRuntimeStatsRole | Unset + if isinstance(_role, Unset): + role = UNSET + else: + role = GraphMetricRuntimeStatsRole(_role) + + runtime_id_hash = d.pop("runtime_id_hash", UNSET) + + owner_id_hash = d.pop("owner_id_hash", UNSET) + + lease_key_hash = d.pop("lease_key_hash", UNSET) + + worker_id_hash = d.pop("worker_id_hash", UNSET) + + worker_count = d.pop("worker_count", UNSET) + + lease_owned = d.pop("lease_owned", UNSET) + + has_lease = d.pop("has_lease", UNSET) + + acquisition_count = d.pop("acquisition_count", UNSET) + + takeover_count = d.pop("takeover_count", UNSET) + + lease_acquire_failures = d.pop("lease_acquire_failures", UNSET) + + lost_leases = d.pop("lost_leases", UNSET) + + last_acquired_ms = d.pop("last_acquired_ms", UNSET) + + lease_expires_at_ms = d.pop("lease_expires_at_ms", UNSET) + + lease_renew_after_ms = d.pop("lease_renew_after_ms", UNSET) + + renewal_count = d.pop("renewal_count", UNSET) + + started = d.pop("started", UNSET) + + shutdown = d.pop("shutdown", UNSET) + + notified = d.pop("notified", UNSET) + + ticks_started = d.pop("ticks_started", UNSET) + + ticks_completed = d.pop("ticks_completed", UNSET) + + durable_progress_ticks = d.pop("durable_progress_ticks", UNSET) + + idle_ticks = d.pop("idle_ticks", UNSET) + + error_ticks = d.pop("error_ticks", UNSET) + + last_error_name = d.pop("last_error_name", UNSET) + + total_metrics_scanned = d.pop("total_metrics_scanned", UNSET) + + total_active_builds = d.pop("total_active_builds", UNSET) + + total_builds_started = d.pop("total_builds_started", UNSET) + + total_worker_steps = d.pop("total_worker_steps", UNSET) + + total_coordinator_steps = d.pop("total_coordinator_steps", UNSET) + + total_retired_input_records = d.pop("total_retired_input_records", UNSET) + + total_pages_claimed = d.pop("total_pages_claimed", UNSET) + + total_pages_completed = d.pop("total_pages_completed", UNSET) + + total_phases_advanced = d.pop("total_phases_advanced", UNSET) + + total_published = d.pop("total_published", UNSET) + + total_failed_builds = d.pop("total_failed_builds", UNSET) + + last_metrics_scanned = d.pop("last_metrics_scanned", UNSET) + + last_active_builds = d.pop("last_active_builds", UNSET) + + last_builds_started = d.pop("last_builds_started", UNSET) + + last_worker_steps = d.pop("last_worker_steps", UNSET) + + last_coordinator_steps = d.pop("last_coordinator_steps", UNSET) + + last_retired_input_records = d.pop("last_retired_input_records", UNSET) + + last_pages_claimed = d.pop("last_pages_claimed", UNSET) + + last_pages_completed = d.pop("last_pages_completed", UNSET) + + last_phases_advanced = d.pop("last_phases_advanced", UNSET) + + last_published = d.pop("last_published", UNSET) + + last_failed_builds = d.pop("last_failed_builds", UNSET) + + last_budget_exhausted = d.pop("last_budget_exhausted", UNSET) + + graph_metric_runtime_stats = cls( + enabled=enabled, + role=role, + runtime_id_hash=runtime_id_hash, + owner_id_hash=owner_id_hash, + lease_key_hash=lease_key_hash, + worker_id_hash=worker_id_hash, + worker_count=worker_count, + lease_owned=lease_owned, + has_lease=has_lease, + acquisition_count=acquisition_count, + takeover_count=takeover_count, + lease_acquire_failures=lease_acquire_failures, + lost_leases=lost_leases, + last_acquired_ms=last_acquired_ms, + lease_expires_at_ms=lease_expires_at_ms, + lease_renew_after_ms=lease_renew_after_ms, + renewal_count=renewal_count, + started=started, + shutdown=shutdown, + notified=notified, + ticks_started=ticks_started, + ticks_completed=ticks_completed, + durable_progress_ticks=durable_progress_ticks, + idle_ticks=idle_ticks, + error_ticks=error_ticks, + last_error_name=last_error_name, + total_metrics_scanned=total_metrics_scanned, + total_active_builds=total_active_builds, + total_builds_started=total_builds_started, + total_worker_steps=total_worker_steps, + total_coordinator_steps=total_coordinator_steps, + total_retired_input_records=total_retired_input_records, + total_pages_claimed=total_pages_claimed, + total_pages_completed=total_pages_completed, + total_phases_advanced=total_phases_advanced, + total_published=total_published, + total_failed_builds=total_failed_builds, + last_metrics_scanned=last_metrics_scanned, + last_active_builds=last_active_builds, + last_builds_started=last_builds_started, + last_worker_steps=last_worker_steps, + last_coordinator_steps=last_coordinator_steps, + last_retired_input_records=last_retired_input_records, + last_pages_claimed=last_pages_claimed, + last_pages_completed=last_pages_completed, + last_phases_advanced=last_phases_advanced, + last_published=last_published, + last_failed_builds=last_failed_builds, + last_budget_exhausted=last_budget_exhausted, + ) + + graph_metric_runtime_stats.additional_properties = d + return graph_metric_runtime_stats + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats_role.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats_role.py new file mode 100644 index 0000000000..ab3379a584 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_runtime_stats_role.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class GraphMetricRuntimeStatsRole(StrEnum): + COMBINED = "combined" + COORDINATOR = "coordinator" + WORKER = "worker" + WORKER_POOL = "worker_pool" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_score.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_score.py new file mode 100644 index 0000000000..b083a923f6 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_score.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="GraphMetricScore") + + +@_attrs_define +class GraphMetricScore: + """ + Attributes: + node (str): + score (float): + """ + + node: str + score: float + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + node = self.node + + score = self.score + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "node": node, + "score": score, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + node = d.pop("node") + + score = d.pop("score") + + graph_metric_score = cls( + node=node, + score=score, + ) + + graph_metric_score.additional_properties = d + return graph_metric_score + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status.py new file mode 100644 index 0000000000..205daf2edd --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status.py @@ -0,0 +1,382 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.graph_metric_status_phase import GraphMetricStatusPhase +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.graph_metric_build_page_status import GraphMetricBuildPageStatus + from ..models.graph_metric_edge_filter_status import GraphMetricEdgeFilterStatus + from ..models.graph_metric_event import GraphMetricEvent + + +T = TypeVar("T", bound="GraphMetricStatus") + + +@_attrs_define +class GraphMetricStatus: + """ + Attributes: + state (str): + phase (GraphMetricStatusPhase): + build_queued (bool): Whether a local or distributed build is queued after the currently published or building + generation. + published_generation (int): + edge_generation (int): + target_edge_generation (int): + progress (float): Build progress for the target edge generation, from 0.0 to 1.0 + converged (bool): + iterations_completed (int): + delta (float): + computed_at_ms (int): + edge_filter (GraphMetricEdgeFilterStatus | Unset): + metadata_version (int | Unset): Version of the published graph metric metadata schema. + config_fingerprint (str | Unset): Deterministic configuration fingerprint encoded as fixed-width hexadecimal so + every SDK preserves all 64 bits. + maintenance_paused (bool | Unset): + queued_generation (int | Unset): Pending edge generation waiting to build, or 0 when no build is queued. + building_generation (int | Unset): Edge generation currently held by an active build lease, or 0 when idle. + build_job_id (int | Unset): Durable identifier for the active graph metric build job, or 0 when idle. + build_started_at_ms (int | Unset): Unix epoch milliseconds when the active graph metric build started, or 0 when + idle. + build_iteration (int | Unset): Iteration number reported by the active build lease, or 0 when idle or not + iterative. + build_lease_expires_at_ms (int | Unset): Unix epoch milliseconds when the active build lease expires, or 0 when + idle. + build_worker_id (str | Unset): Worker id that owns the active build lease. Local builds use `local`. + build_cursor (str | Unset): Opaque resumable cursor for the active build phase. Empty or omitted when idle or + when the phase has no cursor. + build_completed_units (int | Unset): Completed work units for the active graph metric build, or 0 when idle or + unknown. + build_total_units (int | Unset): Estimated total work units for the active graph metric build, or 0 when idle or + unknown. + build_pages (list[GraphMetricBuildPageStatus] | Unset): Active leased or failed build pages for the current + build phase, capped and ordered by durable page key. + build_pages_truncated (bool | Unset): Whether build_pages was capped before every active page could be included. + retry_count (int | Unset): Number of consecutive failed build attempts for the current target generation, or 0 + when no failure applies. + last_error (str | Unset): Last build error for the current failed target generation. + last_event (GraphMetricEvent | Unset): + recent_events (list[GraphMetricEvent] | Unset): Recent graph metric events, newest first. + """ + + state: str + phase: GraphMetricStatusPhase + build_queued: bool + published_generation: int + edge_generation: int + target_edge_generation: int + progress: float + converged: bool + iterations_completed: int + delta: float + computed_at_ms: int + edge_filter: GraphMetricEdgeFilterStatus | Unset = UNSET + metadata_version: int | Unset = UNSET + config_fingerprint: str | Unset = UNSET + maintenance_paused: bool | Unset = UNSET + queued_generation: int | Unset = UNSET + building_generation: int | Unset = UNSET + build_job_id: int | Unset = UNSET + build_started_at_ms: int | Unset = UNSET + build_iteration: int | Unset = UNSET + build_lease_expires_at_ms: int | Unset = UNSET + build_worker_id: str | Unset = UNSET + build_cursor: str | Unset = UNSET + build_completed_units: int | Unset = UNSET + build_total_units: int | Unset = UNSET + build_pages: list[GraphMetricBuildPageStatus] | Unset = UNSET + build_pages_truncated: bool | Unset = UNSET + retry_count: int | Unset = UNSET + last_error: str | Unset = UNSET + last_event: GraphMetricEvent | Unset = UNSET + recent_events: list[GraphMetricEvent] | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + state = self.state + + phase = self.phase.value + + build_queued = self.build_queued + + published_generation = self.published_generation + + edge_generation = self.edge_generation + + target_edge_generation = self.target_edge_generation + + progress = self.progress + + converged = self.converged + + iterations_completed = self.iterations_completed + + delta = self.delta + + computed_at_ms = self.computed_at_ms + + edge_filter: dict[str, Any] | Unset = UNSET + if not isinstance(self.edge_filter, Unset): + edge_filter = self.edge_filter.to_dict() + + metadata_version = self.metadata_version + + config_fingerprint = self.config_fingerprint + + maintenance_paused = self.maintenance_paused + + queued_generation = self.queued_generation + + building_generation = self.building_generation + + build_job_id = self.build_job_id + + build_started_at_ms = self.build_started_at_ms + + build_iteration = self.build_iteration + + build_lease_expires_at_ms = self.build_lease_expires_at_ms + + build_worker_id = self.build_worker_id + + build_cursor = self.build_cursor + + build_completed_units = self.build_completed_units + + build_total_units = self.build_total_units + + build_pages: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.build_pages, Unset): + build_pages = [] + for build_pages_item_data in self.build_pages: + build_pages_item = build_pages_item_data.to_dict() + build_pages.append(build_pages_item) + + build_pages_truncated = self.build_pages_truncated + + retry_count = self.retry_count + + last_error = self.last_error + + last_event: dict[str, Any] | Unset = UNSET + if not isinstance(self.last_event, Unset): + last_event = self.last_event.to_dict() + + recent_events: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.recent_events, Unset): + recent_events = [] + for recent_events_item_data in self.recent_events: + recent_events_item = recent_events_item_data.to_dict() + recent_events.append(recent_events_item) + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "state": state, + "phase": phase, + "build_queued": build_queued, + "published_generation": published_generation, + "edge_generation": edge_generation, + "target_edge_generation": target_edge_generation, + "progress": progress, + "converged": converged, + "iterations_completed": iterations_completed, + "delta": delta, + "computed_at_ms": computed_at_ms, + } + ) + if edge_filter is not UNSET: + field_dict["edge_filter"] = edge_filter + if metadata_version is not UNSET: + field_dict["metadata_version"] = metadata_version + if config_fingerprint is not UNSET: + field_dict["config_fingerprint"] = config_fingerprint + if maintenance_paused is not UNSET: + field_dict["maintenance_paused"] = maintenance_paused + if queued_generation is not UNSET: + field_dict["queued_generation"] = queued_generation + if building_generation is not UNSET: + field_dict["building_generation"] = building_generation + if build_job_id is not UNSET: + field_dict["build_job_id"] = build_job_id + if build_started_at_ms is not UNSET: + field_dict["build_started_at_ms"] = build_started_at_ms + if build_iteration is not UNSET: + field_dict["build_iteration"] = build_iteration + if build_lease_expires_at_ms is not UNSET: + field_dict["build_lease_expires_at_ms"] = build_lease_expires_at_ms + if build_worker_id is not UNSET: + field_dict["build_worker_id"] = build_worker_id + if build_cursor is not UNSET: + field_dict["build_cursor"] = build_cursor + if build_completed_units is not UNSET: + field_dict["build_completed_units"] = build_completed_units + if build_total_units is not UNSET: + field_dict["build_total_units"] = build_total_units + if build_pages is not UNSET: + field_dict["build_pages"] = build_pages + if build_pages_truncated is not UNSET: + field_dict["build_pages_truncated"] = build_pages_truncated + if retry_count is not UNSET: + field_dict["retry_count"] = retry_count + if last_error is not UNSET: + field_dict["last_error"] = last_error + if last_event is not UNSET: + field_dict["last_event"] = last_event + if recent_events is not UNSET: + field_dict["recent_events"] = recent_events + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_build_page_status import GraphMetricBuildPageStatus + from ..models.graph_metric_edge_filter_status import GraphMetricEdgeFilterStatus + from ..models.graph_metric_event import GraphMetricEvent + + d = dict(src_dict) + state = d.pop("state") + + phase = GraphMetricStatusPhase(d.pop("phase")) + + build_queued = d.pop("build_queued") + + published_generation = d.pop("published_generation") + + edge_generation = d.pop("edge_generation") + + target_edge_generation = d.pop("target_edge_generation") + + progress = d.pop("progress") + + converged = d.pop("converged") + + iterations_completed = d.pop("iterations_completed") + + delta = d.pop("delta") + + computed_at_ms = d.pop("computed_at_ms") + + _edge_filter = d.pop("edge_filter", UNSET) + edge_filter: GraphMetricEdgeFilterStatus | Unset + if isinstance(_edge_filter, Unset): + edge_filter = UNSET + else: + edge_filter = GraphMetricEdgeFilterStatus.from_dict(_edge_filter) + + metadata_version = d.pop("metadata_version", UNSET) + + config_fingerprint = d.pop("config_fingerprint", UNSET) + + maintenance_paused = d.pop("maintenance_paused", UNSET) + + queued_generation = d.pop("queued_generation", UNSET) + + building_generation = d.pop("building_generation", UNSET) + + build_job_id = d.pop("build_job_id", UNSET) + + build_started_at_ms = d.pop("build_started_at_ms", UNSET) + + build_iteration = d.pop("build_iteration", UNSET) + + build_lease_expires_at_ms = d.pop("build_lease_expires_at_ms", UNSET) + + build_worker_id = d.pop("build_worker_id", UNSET) + + build_cursor = d.pop("build_cursor", UNSET) + + build_completed_units = d.pop("build_completed_units", UNSET) + + build_total_units = d.pop("build_total_units", UNSET) + + _build_pages = d.pop("build_pages", UNSET) + build_pages: list[GraphMetricBuildPageStatus] | Unset = UNSET + if _build_pages is not UNSET: + build_pages = [] + for build_pages_item_data in _build_pages: + build_pages_item = GraphMetricBuildPageStatus.from_dict(build_pages_item_data) + + build_pages.append(build_pages_item) + + build_pages_truncated = d.pop("build_pages_truncated", UNSET) + + retry_count = d.pop("retry_count", UNSET) + + last_error = d.pop("last_error", UNSET) + + _last_event = d.pop("last_event", UNSET) + last_event: GraphMetricEvent | Unset + if isinstance(_last_event, Unset): + last_event = UNSET + else: + last_event = GraphMetricEvent.from_dict(_last_event) + + _recent_events = d.pop("recent_events", UNSET) + recent_events: list[GraphMetricEvent] | Unset = UNSET + if _recent_events is not UNSET: + recent_events = [] + for recent_events_item_data in _recent_events: + recent_events_item = GraphMetricEvent.from_dict(recent_events_item_data) + + recent_events.append(recent_events_item) + + graph_metric_status = cls( + state=state, + phase=phase, + build_queued=build_queued, + published_generation=published_generation, + edge_generation=edge_generation, + target_edge_generation=target_edge_generation, + progress=progress, + converged=converged, + iterations_completed=iterations_completed, + delta=delta, + computed_at_ms=computed_at_ms, + edge_filter=edge_filter, + metadata_version=metadata_version, + config_fingerprint=config_fingerprint, + maintenance_paused=maintenance_paused, + queued_generation=queued_generation, + building_generation=building_generation, + build_job_id=build_job_id, + build_started_at_ms=build_started_at_ms, + build_iteration=build_iteration, + build_lease_expires_at_ms=build_lease_expires_at_ms, + build_worker_id=build_worker_id, + build_cursor=build_cursor, + build_completed_units=build_completed_units, + build_total_units=build_total_units, + build_pages=build_pages, + build_pages_truncated=build_pages_truncated, + retry_count=retry_count, + last_error=last_error, + last_event=last_event, + recent_events=recent_events, + ) + + graph_metric_status.additional_properties = d + return graph_metric_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status_phase.py b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status_phase.py new file mode 100644 index 0000000000..5170a447c2 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_metric_status_phase.py @@ -0,0 +1,21 @@ +from enum import StrEnum + + +class GraphMetricStatusPhase(StrEnum): + CHECK_CONVERGENCE = "check_convergence" + CLEANUP_OLD_GENERATIONS = "cleanup_old_generations" + COMPLETE = "complete" + COMPUTING = "computing" + HITS_HUB_CONTRIBUTIONS = "hits_hub_contributions" + HITS_HUB_REDUCE_RANKS = "hits_hub_reduce_ranks" + IDLE = "idle" + INITIALIZE_RANKS = "initialize_ranks" + ITERATE_CONTRIBUTIONS = "iterate_contributions" + PREPARE_GENERATION = "prepare_generation" + PUBLISHING = "publishing" + PUBLISH_GENERATION = "publish_generation" + REDUCE_RANKS = "reduce_ranks" + SCAN_EDGES_AND_OUT_DEGREE = "scan_edges_and_out_degree" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result.py b/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result.py index bb9e8e2b28..0da387e4c3 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result.py +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result.py @@ -6,8 +6,10 @@ from attrs import define as _attrs_define from ..models.graph_nodes_result_kind import GraphNodesResultKind +from ..types import UNSET, Unset if TYPE_CHECKING: + from ..models.graph_nodes_result_metric_status import GraphNodesResultMetricStatus from ..models.graph_result_node import GraphResultNode from ..models.graph_result_stats import GraphResultStats @@ -23,11 +25,14 @@ class GraphNodesResult: kind (GraphNodesResultKind): Stable discriminator for the graph result shape. nodes (list[GraphResultNode]): Traversal result nodes; requested paths are stored on each node. stats (GraphResultStats): Completion statistics for a bounded graph result. + metric_status (GraphNodesResultMetricStatus | Unset): Graph metric status metadata keyed by metric name when + requested. """ kind: GraphNodesResultKind nodes: list[GraphResultNode] stats: GraphResultStats + metric_status: GraphNodesResultMetricStatus | Unset = UNSET def to_dict(self) -> dict[str, Any]: kind = self.kind.value @@ -39,6 +44,10 @@ def to_dict(self) -> dict[str, Any]: stats = self.stats.to_dict() + metric_status: dict[str, Any] | Unset = UNSET + if not isinstance(self.metric_status, Unset): + metric_status = self.metric_status.to_dict() + field_dict: dict[str, Any] = {} field_dict.update( @@ -48,11 +57,14 @@ def to_dict(self) -> dict[str, Any]: "stats": stats, } ) + if metric_status is not UNSET: + field_dict["metric_status"] = metric_status return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_nodes_result_metric_status import GraphNodesResultMetricStatus from ..models.graph_result_node import GraphResultNode from ..models.graph_result_stats import GraphResultStats @@ -68,10 +80,18 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: stats = GraphResultStats.from_dict(d.pop("stats")) + _metric_status = d.pop("metric_status", UNSET) + metric_status: GraphNodesResultMetricStatus | Unset + if isinstance(_metric_status, Unset): + metric_status = UNSET + else: + metric_status = GraphNodesResultMetricStatus.from_dict(_metric_status) + graph_nodes_result = cls( kind=kind, nodes=nodes, stats=stats, + metric_status=metric_status, ) return graph_nodes_result diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result_metric_status.py b/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result_metric_status.py new file mode 100644 index 0000000000..434dccd6e1 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_nodes_result_metric_status.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_status import GraphMetricStatus + + +T = TypeVar("T", bound="GraphNodesResultMetricStatus") + + +@_attrs_define +class GraphNodesResultMetricStatus: + """Graph metric status metadata keyed by metric name when requested.""" + + additional_properties: dict[str, GraphMetricStatus] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_status import GraphMetricStatus + + d = dict(src_dict) + graph_nodes_result_metric_status = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = GraphMetricStatus.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + graph_nodes_result_metric_status.additional_properties = additional_properties + return graph_nodes_result_metric_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> GraphMetricStatus: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: GraphMetricStatus) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_result_node.py b/py/packages/sdk/src/antfly/client_generated/models/graph_result_node.py index 3f948857d8..2d3b2eec64 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/graph_result_node.py +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_result_node.py @@ -12,6 +12,7 @@ from ..models.graph_path_endpoint import GraphPathEndpoint from ..models.graph_result_node_document import GraphResultNodeDocument from ..models.graph_result_node_evidence import GraphResultNodeEvidence + from ..models.graph_result_node_metrics import GraphResultNodeMetrics T = TypeVar("T", bound="GraphResultNode") @@ -34,6 +35,8 @@ class GraphResultNode: path for traversal queries. provenance (list[str] | Unset): Algebraic provenance labels folded into this result, when requested by an algebraic graph executor + metrics (GraphResultNodeMetrics | Unset): Projected graph metric scores keyed by metric name. Values are numbers + or null when a requested metric has no score for the node. evidence (GraphResultNodeEvidence | Unset): Parsed evidence envelope for provenance labels and edge metadata """ @@ -44,6 +47,7 @@ class GraphResultNode: path: list[GraphPathEndpoint] | Unset = UNSET path_edges: list[GraphPathEdge] | Unset = UNSET provenance: list[str] | Unset = UNSET + metrics: GraphResultNodeMetrics | Unset = UNSET evidence: GraphResultNodeEvidence | Unset = UNSET def to_dict(self) -> dict[str, Any]: @@ -75,6 +79,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.provenance, Unset): provenance = self.provenance + metrics: dict[str, Any] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics.to_dict() + evidence: dict[str, Any] | Unset = UNSET if not isinstance(self.evidence, Unset): evidence = self.evidence.to_dict() @@ -97,6 +105,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["path_edges"] = path_edges if provenance is not UNSET: field_dict["provenance"] = provenance + if metrics is not UNSET: + field_dict["metrics"] = metrics if evidence is not UNSET: field_dict["evidence"] = evidence @@ -108,6 +118,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.graph_path_endpoint import GraphPathEndpoint from ..models.graph_result_node_document import GraphResultNodeDocument from ..models.graph_result_node_evidence import GraphResultNodeEvidence + from ..models.graph_result_node_metrics import GraphResultNodeMetrics d = dict(src_dict) key = d.pop("key") @@ -143,6 +154,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: provenance = cast(list[str], d.pop("provenance", UNSET)) + _metrics = d.pop("metrics", UNSET) + metrics: GraphResultNodeMetrics | Unset + if isinstance(_metrics, Unset): + metrics = UNSET + else: + metrics = GraphResultNodeMetrics.from_dict(_metrics) + _evidence = d.pop("evidence", UNSET) evidence: GraphResultNodeEvidence | Unset if isinstance(_evidence, Unset): @@ -158,6 +176,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: path=path, path_edges=path_edges, provenance=provenance, + metrics=metrics, evidence=evidence, ) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_result_node_metrics.py b/py/packages/sdk/src/antfly/client_generated/models/graph_result_node_metrics.py new file mode 100644 index 0000000000..508e89ffb5 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_result_node_metrics.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="GraphResultNodeMetrics") + + +@_attrs_define +class GraphResultNodeMetrics: + """Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score + for the node. + + """ + + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + graph_result_node_metrics = cls() + + graph_result_node_metrics.additional_properties = d + return graph_result_node_metrics + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_traversal.py b/py/packages/sdk/src/antfly/client_generated/models/graph_traversal.py index cef3cb1194..b096088e94 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/graph_traversal.py +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_traversal.py @@ -6,6 +6,7 @@ from attrs import define as _attrs_define from ..models.edge_direction import EdgeDirection +from ..models.graph_traversal_metric_freshness import GraphTraversalMetricFreshness from ..types import UNSET, Unset if TYPE_CHECKING: @@ -27,6 +28,8 @@ from ..models.graph_edge_weight_range import GraphEdgeWeightRange from ..models.graph_identity_node_selector import GraphIdentityNodeSelector from ..models.graph_key_node_selector import GraphKeyNodeSelector + from ..models.graph_metric_filter import GraphMetricFilter + from ..models.graph_metric_order import GraphMetricOrder from ..models.graph_result_ref_node_selector import GraphResultRefNodeSelector @@ -55,6 +58,15 @@ class GraphTraversal: include_documents (bool | Unset): Include each result node's stored document when it exists at the pinned snapshot. A dangling graph identity omits document. When false, document is always omitted. Default: False. fields (list[str] | Unset): Requires include_documents=true. Omit to include all document fields. + metrics (list[str] | Unset): Graph metric names to project onto returned traversal nodes. + order_by (list[GraphMetricOrder] | Unset): Sort traversal candidates by graph metric score before applying + limit. + where_metric (list[GraphMetricFilter] | Unset): Filter traversal candidates by graph metric score before + applying limit. + metric_freshness (GraphTraversalMetricFreshness | Unset): Freshness required for projected, ordered, and + filtered graph metrics. Default: GraphTraversalMetricFreshness.PUBLISHED. + include_metric_status (bool | Unset): Include graph metric status metadata in the traversal profile. Default: + False. filter_ (GraphDocumentBoolFieldFilter | GraphDocumentDateRangeFilter | GraphDocumentFilterBoolean | GraphDocumentFilterConjunction | GraphDocumentFilterDisjunction | GraphDocumentFuzzyFilter | GraphDocumentIdsFilter | GraphDocumentMatchAllFilter | GraphDocumentMatchNoneFilter | @@ -76,6 +88,11 @@ class GraphTraversal: include_paths: bool | Unset = False include_documents: bool | Unset = False fields: list[str] | Unset = UNSET + metrics: list[str] | Unset = UNSET + order_by: list[GraphMetricOrder] | Unset = UNSET + where_metric: list[GraphMetricFilter] | Unset = UNSET + metric_freshness: GraphTraversalMetricFreshness | Unset = GraphTraversalMetricFreshness.PUBLISHED + include_metric_status: bool | Unset = False filter_: ( GraphDocumentBoolFieldFilter | GraphDocumentDateRangeFilter @@ -145,6 +162,30 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.fields, Unset): fields = self.fields + metrics: list[str] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics + + order_by: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.order_by, Unset): + order_by = [] + for order_by_item_data in self.order_by: + order_by_item = order_by_item_data.to_dict() + order_by.append(order_by_item) + + where_metric: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.where_metric, Unset): + where_metric = [] + for where_metric_item_data in self.where_metric: + where_metric_item = where_metric_item_data.to_dict() + where_metric.append(where_metric_item) + + metric_freshness: str | Unset = UNSET + if not isinstance(self.metric_freshness, Unset): + metric_freshness = self.metric_freshness.value + + include_metric_status = self.include_metric_status + filter_: dict[str, Any] | Unset if isinstance(self.filter_, Unset): filter_ = UNSET @@ -202,6 +243,16 @@ def to_dict(self) -> dict[str, Any]: field_dict["include_documents"] = include_documents if fields is not UNSET: field_dict["fields"] = fields + if metrics is not UNSET: + field_dict["metrics"] = metrics + if order_by is not UNSET: + field_dict["order_by"] = order_by + if where_metric is not UNSET: + field_dict["where_metric"] = where_metric + if metric_freshness is not UNSET: + field_dict["metric_freshness"] = metric_freshness + if include_metric_status is not UNSET: + field_dict["include_metric_status"] = include_metric_status if filter_ is not UNSET: field_dict["filter"] = filter_ @@ -227,6 +278,8 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.graph_edge_weight_range import GraphEdgeWeightRange from ..models.graph_identity_node_selector import GraphIdentityNodeSelector from ..models.graph_key_node_selector import GraphKeyNodeSelector + from ..models.graph_metric_filter import GraphMetricFilter + from ..models.graph_metric_order import GraphMetricOrder from ..models.graph_result_ref_node_selector import GraphResultRefNodeSelector d = dict(src_dict) @@ -282,6 +335,35 @@ def _parse_start(data: object) -> GraphIdentityNodeSelector | GraphKeyNodeSelect fields = cast(list[str], d.pop("fields", UNSET)) + metrics = cast(list[str], d.pop("metrics", UNSET)) + + _order_by = d.pop("order_by", UNSET) + order_by: list[GraphMetricOrder] | Unset = UNSET + if _order_by is not UNSET: + order_by = [] + for order_by_item_data in _order_by: + order_by_item = GraphMetricOrder.from_dict(order_by_item_data) + + order_by.append(order_by_item) + + _where_metric = d.pop("where_metric", UNSET) + where_metric: list[GraphMetricFilter] | Unset = UNSET + if _where_metric is not UNSET: + where_metric = [] + for where_metric_item_data in _where_metric: + where_metric_item = GraphMetricFilter.from_dict(where_metric_item_data) + + where_metric.append(where_metric_item) + + _metric_freshness = d.pop("metric_freshness", UNSET) + metric_freshness: GraphTraversalMetricFreshness | Unset + if isinstance(_metric_freshness, Unset): + metric_freshness = UNSET + else: + metric_freshness = GraphTraversalMetricFreshness(_metric_freshness) + + include_metric_status = d.pop("include_metric_status", UNSET) + def _parse_filter_( data: object, ) -> ( @@ -434,6 +516,11 @@ def _parse_filter_( include_paths=include_paths, include_documents=include_documents, fields=fields, + metrics=metrics, + order_by=order_by, + where_metric=where_metric, + metric_freshness=metric_freshness, + include_metric_status=include_metric_status, filter_=filter_, ) diff --git a/py/packages/sdk/src/antfly/client_generated/models/graph_traversal_metric_freshness.py b/py/packages/sdk/src/antfly/client_generated/models/graph_traversal_metric_freshness.py new file mode 100644 index 0000000000..a5b92ef281 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/graph_traversal_metric_freshness.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class GraphTraversalMetricFreshness(StrEnum): + FRESH = "fresh" + PUBLISHED = "published" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query.py b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query.py index f5ef7a4da4..edc695a872 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query.py +++ b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query.py @@ -7,9 +7,12 @@ from attrs import field as _attrs_field from ..models.graph_query_type import GraphQueryType +from ..models.legacy_graph_query_metric_freshness import LegacyGraphQueryMetricFreshness from ..types import UNSET, Unset if TYPE_CHECKING: + from ..models.graph_metric_filter import GraphMetricFilter + from ..models.graph_metric_order import GraphMetricOrder from ..models.graph_query_params import GraphQueryParams from ..models.legacy_graph_node_selector import LegacyGraphNodeSelector from ..models.pattern_step import PatternStep @@ -35,6 +38,12 @@ class LegacyGraphQuery: include_documents (bool | Unset): include_edges (bool | Unset): fields (list[str] | Unset): + metrics (list[str] | Unset): Graph metric names to project onto legacy graph_searches result nodes. + order_by (list[GraphMetricOrder] | Unset): Sort legacy graph_searches result nodes by graph metric score. + where_metric (list[GraphMetricFilter] | Unset): Filter legacy graph_searches result nodes by graph metric score. + metric_freshness (LegacyGraphQueryMetricFreshness | Unset): Freshness required for projected, ordered, and + filtered graph metrics. + include_metric_status (bool | Unset): Include graph metric status metadata in the legacy graph_searches result. """ type_: GraphQueryType @@ -47,6 +56,11 @@ class LegacyGraphQuery: include_documents: bool | Unset = UNSET include_edges: bool | Unset = UNSET fields: list[str] | Unset = UNSET + metrics: list[str] | Unset = UNSET + order_by: list[GraphMetricOrder] | Unset = UNSET + where_metric: list[GraphMetricFilter] | Unset = UNSET + metric_freshness: LegacyGraphQueryMetricFreshness | Unset = UNSET + include_metric_status: bool | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -85,6 +99,30 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.fields, Unset): fields = self.fields + metrics: list[str] | Unset = UNSET + if not isinstance(self.metrics, Unset): + metrics = self.metrics + + order_by: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.order_by, Unset): + order_by = [] + for order_by_item_data in self.order_by: + order_by_item = order_by_item_data.to_dict() + order_by.append(order_by_item) + + where_metric: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.where_metric, Unset): + where_metric = [] + for where_metric_item_data in self.where_metric: + where_metric_item = where_metric_item_data.to_dict() + where_metric.append(where_metric_item) + + metric_freshness: str | Unset = UNSET + if not isinstance(self.metric_freshness, Unset): + metric_freshness = self.metric_freshness.value + + include_metric_status = self.include_metric_status + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -109,11 +147,23 @@ def to_dict(self) -> dict[str, Any]: field_dict["include_edges"] = include_edges if fields is not UNSET: field_dict["fields"] = fields + if metrics is not UNSET: + field_dict["metrics"] = metrics + if order_by is not UNSET: + field_dict["order_by"] = order_by + if where_metric is not UNSET: + field_dict["where_metric"] = where_metric + if metric_freshness is not UNSET: + field_dict["metric_freshness"] = metric_freshness + if include_metric_status is not UNSET: + field_dict["include_metric_status"] = include_metric_status return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_filter import GraphMetricFilter + from ..models.graph_metric_order import GraphMetricOrder from ..models.graph_query_params import GraphQueryParams from ..models.legacy_graph_node_selector import LegacyGraphNodeSelector from ..models.pattern_step import PatternStep @@ -161,6 +211,35 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: fields = cast(list[str], d.pop("fields", UNSET)) + metrics = cast(list[str], d.pop("metrics", UNSET)) + + _order_by = d.pop("order_by", UNSET) + order_by: list[GraphMetricOrder] | Unset = UNSET + if _order_by is not UNSET: + order_by = [] + for order_by_item_data in _order_by: + order_by_item = GraphMetricOrder.from_dict(order_by_item_data) + + order_by.append(order_by_item) + + _where_metric = d.pop("where_metric", UNSET) + where_metric: list[GraphMetricFilter] | Unset = UNSET + if _where_metric is not UNSET: + where_metric = [] + for where_metric_item_data in _where_metric: + where_metric_item = GraphMetricFilter.from_dict(where_metric_item_data) + + where_metric.append(where_metric_item) + + _metric_freshness = d.pop("metric_freshness", UNSET) + metric_freshness: LegacyGraphQueryMetricFreshness | Unset + if isinstance(_metric_freshness, Unset): + metric_freshness = UNSET + else: + metric_freshness = LegacyGraphQueryMetricFreshness(_metric_freshness) + + include_metric_status = d.pop("include_metric_status", UNSET) + legacy_graph_query = cls( type_=type_, index_name=index_name, @@ -172,6 +251,11 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: include_documents=include_documents, include_edges=include_edges, fields=fields, + metrics=metrics, + order_by=order_by, + where_metric=where_metric, + metric_freshness=metric_freshness, + include_metric_status=include_metric_status, ) legacy_graph_query.additional_properties = d diff --git a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query_metric_freshness.py b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query_metric_freshness.py new file mode 100644 index 0000000000..ec832b6d2e --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_query_metric_freshness.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class LegacyGraphQueryMetricFreshness(StrEnum): + FRESH = "fresh" + PUBLISHED = "published" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result.py b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result.py index fa2aafe052..2d7c4bc051 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result.py +++ b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from ..models.legacy_graph_result_node import LegacyGraphResultNode + from ..models.legacy_graph_search_result_metric_status import LegacyGraphSearchResultMetricStatus from ..models.path import Path from ..models.pattern_match import PatternMatch @@ -33,6 +34,7 @@ class LegacyGraphSearchResult: matches (list[PatternMatch] | Unset): Deprecated graph_searches pattern results; use rows for graph_queries. took (int | Unset): Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field. + metric_status (LegacyGraphSearchResultMetricStatus | Unset): Graph metric status metadata keyed by metric name. """ type_: GraphQueryType @@ -42,6 +44,7 @@ class LegacyGraphSearchResult: paths: list[Path] | Unset = UNSET matches: list[PatternMatch] | Unset = UNSET took: int | Unset = UNSET + metric_status: LegacyGraphSearchResultMetricStatus | Unset = UNSET def to_dict(self) -> dict[str, Any]: type_ = self.type_.value @@ -75,6 +78,10 @@ def to_dict(self) -> dict[str, Any]: took = self.took + metric_status: dict[str, Any] | Unset = UNSET + if not isinstance(self.metric_status, Unset): + metric_status = self.metric_status.to_dict() + field_dict: dict[str, Any] = {} field_dict.update( @@ -93,12 +100,15 @@ def to_dict(self) -> dict[str, Any]: field_dict["matches"] = matches if took is not UNSET: field_dict["took"] = took + if metric_status is not UNSET: + field_dict["metric_status"] = metric_status return field_dict @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.legacy_graph_result_node import LegacyGraphResultNode + from ..models.legacy_graph_search_result_metric_status import LegacyGraphSearchResultMetricStatus from ..models.path import Path from ..models.pattern_match import PatternMatch @@ -143,6 +153,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: took = d.pop("took", UNSET) + _metric_status = d.pop("metric_status", UNSET) + metric_status: LegacyGraphSearchResultMetricStatus | Unset + if isinstance(_metric_status, Unset): + metric_status = UNSET + else: + metric_status = LegacyGraphSearchResultMetricStatus.from_dict(_metric_status) + legacy_graph_search_result = cls( type_=type_, total=total, @@ -151,6 +168,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: paths=paths, matches=matches, took=took, + metric_status=metric_status, ) return legacy_graph_search_result diff --git a/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result_metric_status.py b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result_metric_status.py new file mode 100644 index 0000000000..8ebb21c153 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/legacy_graph_search_result_metric_status.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_status import GraphMetricStatus + + +T = TypeVar("T", bound="LegacyGraphSearchResultMetricStatus") + + +@_attrs_define +class LegacyGraphSearchResultMetricStatus: + """Graph metric status metadata keyed by metric name.""" + + additional_properties: dict[str, GraphMetricStatus] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_status import GraphMetricStatus + + d = dict(src_dict) + legacy_graph_search_result_metric_status = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = GraphMetricStatus.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + legacy_graph_search_result_metric_status.additional_properties = additional_properties + return legacy_graph_search_result_metric_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> GraphMetricStatus: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: GraphMetricStatus) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_hit.py b/py/packages/sdk/src/antfly/client_generated/models/query_hit.py index d457f8b469..d7e9e260e5 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/query_hit.py +++ b/py/packages/sdk/src/antfly/client_generated/models/query_hit.py @@ -12,6 +12,7 @@ from ..models.query_hit_hierarchy import QueryHitHierarchy from ..models.query_hit_index_scores import QueryHitIndexScores from ..models.query_hit_source import QueryHitSource + from ..models.query_score_details import QueryScoreDetails T = TypeVar("T", bound="QueryHit") @@ -29,6 +30,8 @@ class QueryHit: the best matching descendant that supplied the group score. Omitted for non-dense and fused results. field_index_scores (QueryHitIndexScores | Unset): Scores partitioned by index when using RRF search. + field_score_details (QueryScoreDetails | Unset): Optional score provenance for ranking features that changed the + final hit score. field_source (QueryHitSource | Unset): hierarchy (QueryHitHierarchy | Unset): field_sort (list[Any] | Unset): Sort key values for this hit. Pass as search_after or search_before @@ -41,6 +44,7 @@ class QueryHit: field_score: float field_distance: float | Unset = UNSET field_index_scores: QueryHitIndexScores | Unset = UNSET + field_score_details: QueryScoreDetails | Unset = UNSET field_source: QueryHitSource | Unset = UNSET hierarchy: QueryHitHierarchy | Unset = UNSET field_sort: list[Any] | Unset = UNSET @@ -57,6 +61,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.field_index_scores, Unset): field_index_scores = self.field_index_scores.to_dict() + field_score_details: dict[str, Any] | Unset = UNSET + if not isinstance(self.field_score_details, Unset): + field_score_details = self.field_score_details.to_dict() + field_source: dict[str, Any] | Unset = UNSET if not isinstance(self.field_source, Unset): field_source = self.field_source.to_dict() @@ -81,6 +89,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["_distance"] = field_distance if field_index_scores is not UNSET: field_dict["_index_scores"] = field_index_scores + if field_score_details is not UNSET: + field_dict["_score_details"] = field_score_details if field_source is not UNSET: field_dict["_source"] = field_source if hierarchy is not UNSET: @@ -95,6 +105,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.query_hit_hierarchy import QueryHitHierarchy from ..models.query_hit_index_scores import QueryHitIndexScores from ..models.query_hit_source import QueryHitSource + from ..models.query_score_details import QueryScoreDetails d = dict(src_dict) field_id = d.pop("_id") @@ -110,6 +121,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: field_index_scores = QueryHitIndexScores.from_dict(_field_index_scores) + _field_score_details = d.pop("_score_details", UNSET) + field_score_details: QueryScoreDetails | Unset + if isinstance(_field_score_details, Unset): + field_score_details = UNSET + else: + field_score_details = QueryScoreDetails.from_dict(_field_score_details) + _field_source = d.pop("_source", UNSET) field_source: QueryHitSource | Unset if isinstance(_field_source, Unset): @@ -131,6 +149,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: field_score=field_score, field_distance=field_distance, field_index_scores=field_index_scores, + field_score_details=field_score_details, field_source=field_source, hierarchy=hierarchy, field_sort=field_sort, diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_profile.py b/py/packages/sdk/src/antfly/client_generated/models/query_profile.py index a708859416..acdf40f344 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/query_profile.py +++ b/py/packages/sdk/src/antfly/client_generated/models/query_profile.py @@ -9,6 +9,7 @@ from ..types import UNSET, Unset if TYPE_CHECKING: + from ..models.graph_metric_profile import GraphMetricProfile from ..models.join_profile import JoinProfile from ..models.merge_profile import MergeProfile from ..models.reranker_profile import RerankerProfile @@ -29,6 +30,8 @@ class QueryProfile: join (JoinProfile | Unset): Join execution statistics. reranker (RerankerProfile | Unset): Reranking execution statistics. merge (MergeProfile | Unset): Result merge statistics for hybrid search. + graph_metrics (list[GraphMetricProfile] | Unset): Graph metric freshness and generation details for metric-aware + query work. sort (SortProfile | Unset): Sort execution profile. These fields are the stable public diagnostic surface. Low-level implementation counters such as doc-value load timings, stored-source loads, collector/window internals, cost-model @@ -41,6 +44,7 @@ class QueryProfile: join: JoinProfile | Unset = UNSET reranker: RerankerProfile | Unset = UNSET merge: MergeProfile | Unset = UNSET + graph_metrics: list[GraphMetricProfile] | Unset = UNSET sort: SortProfile | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -61,6 +65,13 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.merge, Unset): merge = self.merge.to_dict() + graph_metrics: list[dict[str, Any]] | Unset = UNSET + if not isinstance(self.graph_metrics, Unset): + graph_metrics = [] + for graph_metrics_item_data in self.graph_metrics: + graph_metrics_item = graph_metrics_item_data.to_dict() + graph_metrics.append(graph_metrics_item) + sort: dict[str, Any] | Unset = UNSET if not isinstance(self.sort, Unset): sort = self.sort.to_dict() @@ -76,6 +87,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["reranker"] = reranker if merge is not UNSET: field_dict["merge"] = merge + if graph_metrics is not UNSET: + field_dict["graph_metrics"] = graph_metrics if sort is not UNSET: field_dict["sort"] = sort @@ -83,6 +96,7 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_profile import GraphMetricProfile from ..models.join_profile import JoinProfile from ..models.merge_profile import MergeProfile from ..models.reranker_profile import RerankerProfile @@ -118,6 +132,15 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: merge = MergeProfile.from_dict(_merge) + _graph_metrics = d.pop("graph_metrics", UNSET) + graph_metrics: list[GraphMetricProfile] | Unset = UNSET + if _graph_metrics is not UNSET: + graph_metrics = [] + for graph_metrics_item_data in _graph_metrics: + graph_metrics_item = GraphMetricProfile.from_dict(graph_metrics_item_data) + + graph_metrics.append(graph_metrics_item) + _sort = d.pop("sort", UNSET) sort: SortProfile | Unset if isinstance(_sort, Unset): @@ -130,6 +153,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: join=join, reranker=reranker, merge=merge, + graph_metrics=graph_metrics, sort=sort, ) diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_request.py b/py/packages/sdk/src/antfly/client_generated/models/query_request.py index e1f956da35..f480b0acd8 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/query_request.py +++ b/py/packages/sdk/src/antfly/client_generated/models/query_request.py @@ -21,6 +21,8 @@ from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -295,6 +297,12 @@ class QueryRequest: Has minor performance overhead — not recommended for production traffic. reranker (RerankerConfig | Unset): A unified configuration for a reranking provider. Example: {'provider': 'cohere', 'model': 'rerank-v4.0-pro', 'field': 'content'}. + graph_metric (GraphMetricQuery | Unset): Reads a published graph metric. Score-bearing graph metric queries on + multi-shard tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + graph_metric_rerank (GraphMetricRerank | Unset): Blends a published graph metric into hit scores. Multi-shard + tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required. analyses (Analyses | Unset): graph_queries (GraphQueries | Unset): Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH @@ -484,6 +492,8 @@ class QueryRequest: count: bool | Unset = UNSET profile: bool | Unset = UNSET reranker: RerankerConfig | Unset = UNSET + graph_metric: GraphMetricQuery | Unset = UNSET + graph_metric_rerank: GraphMetricRerank | Unset = UNSET analyses: Analyses | Unset = UNSET graph_queries: GraphQueries | Unset = UNSET document_renderer: str | Unset = UNSET @@ -760,6 +770,14 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.reranker, Unset): reranker = self.reranker.to_dict() + graph_metric: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric, Unset): + graph_metric = self.graph_metric.to_dict() + + graph_metric_rerank: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_rerank, Unset): + graph_metric_rerank = self.graph_metric_rerank.to_dict() + analyses: dict[str, Any] | Unset = UNSET if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() @@ -839,6 +857,10 @@ def to_dict(self) -> dict[str, Any]: field_dict["profile"] = profile if reranker is not UNSET: field_dict["reranker"] = reranker + if graph_metric is not UNSET: + field_dict["graph_metric"] = graph_metric + if graph_metric_rerank is not UNSET: + field_dict["graph_metric_rerank"] = graph_metric_rerank if analyses is not UNSET: field_dict["analyses"] = analyses if graph_queries is not UNSET: @@ -868,6 +890,8 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -1704,6 +1728,20 @@ def _parse_exclusion_query( else: reranker = RerankerConfig.from_dict(_reranker) + _graph_metric = d.pop("graph_metric", UNSET) + graph_metric: GraphMetricQuery | Unset + if isinstance(_graph_metric, Unset): + graph_metric = UNSET + else: + graph_metric = GraphMetricQuery.from_dict(_graph_metric) + + _graph_metric_rerank = d.pop("graph_metric_rerank", UNSET) + graph_metric_rerank: GraphMetricRerank | Unset + if isinstance(_graph_metric_rerank, Unset): + graph_metric_rerank = UNSET + else: + graph_metric_rerank = GraphMetricRerank.from_dict(_graph_metric_rerank) + _analyses = d.pop("analyses", UNSET) analyses: Analyses | Unset if isinstance(_analyses, Unset): @@ -1769,6 +1807,8 @@ def _parse_exclusion_query( count=count, profile=profile, reranker=reranker, + graph_metric=graph_metric, + graph_metric_rerank=graph_metric_rerank, analyses=analyses, graph_queries=graph_queries, document_renderer=document_renderer, diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_result.py b/py/packages/sdk/src/antfly/client_generated/models/query_result.py index 6498ba8f73..f5c881c348 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/query_result.py +++ b/py/packages/sdk/src/antfly/client_generated/models/query_result.py @@ -14,6 +14,7 @@ from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults T = TypeVar("T", bound="QueryResult") @@ -31,6 +32,7 @@ class QueryResult: names from the request. Contains computed metrics or buckets depending on the aggregation type. analyses (QueryResultBaseAnalyses | Unset): Analysis results like PCA and t-SNE per index embeddings. + graph_metric_results (QueryResultBaseGraphMetricResults | Unset): Results from direct graph metric reads. profile (QueryProfile | Unset): Detailed execution profiling for a query. Present in the response when the request sets `profile: true`. error (str | Unset): Error message if the query failed. @@ -44,6 +46,7 @@ class QueryResult: hits: QueryHits | Unset = UNSET aggregations: QueryResultBaseAggregations | Unset = UNSET analyses: QueryResultBaseAnalyses | Unset = UNSET + graph_metric_results: QueryResultBaseGraphMetricResults | Unset = UNSET profile: QueryProfile | Unset = UNSET error: str | Unset = UNSET table: str | Unset = UNSET @@ -67,6 +70,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() + graph_metric_results: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_results, Unset): + graph_metric_results = self.graph_metric_results.to_dict() + profile: dict[str, Any] | Unset = UNSET if not isinstance(self.profile, Unset): profile = self.profile.to_dict() @@ -93,6 +100,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["aggregations"] = aggregations if analyses is not UNSET: field_dict["analyses"] = analyses + if graph_metric_results is not UNSET: + field_dict["graph_metric_results"] = graph_metric_results if profile is not UNSET: field_dict["profile"] = profile if error is not UNSET: @@ -111,6 +120,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults d = dict(src_dict) took = d.pop("took") @@ -138,6 +148,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: analyses = QueryResultBaseAnalyses.from_dict(_analyses) + _graph_metric_results = d.pop("graph_metric_results", UNSET) + graph_metric_results: QueryResultBaseGraphMetricResults | Unset + if isinstance(_graph_metric_results, Unset): + graph_metric_results = UNSET + else: + graph_metric_results = QueryResultBaseGraphMetricResults.from_dict(_graph_metric_results) + _profile = d.pop("profile", UNSET) profile: QueryProfile | Unset if isinstance(_profile, Unset): @@ -162,6 +179,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: hits=hits, aggregations=aggregations, analyses=analyses, + graph_metric_results=graph_metric_results, profile=profile, error=error, table=table, diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_result_base.py b/py/packages/sdk/src/antfly/client_generated/models/query_result_base.py index 9893694920..eb5fa55c69 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/query_result_base.py +++ b/py/packages/sdk/src/antfly/client_generated/models/query_result_base.py @@ -13,6 +13,7 @@ from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults T = TypeVar("T", bound="QueryResultBase") @@ -30,6 +31,7 @@ class QueryResultBase: names from the request. Contains computed metrics or buckets depending on the aggregation type. analyses (QueryResultBaseAnalyses | Unset): Analysis results like PCA and t-SNE per index embeddings. + graph_metric_results (QueryResultBaseGraphMetricResults | Unset): Results from direct graph metric reads. profile (QueryProfile | Unset): Detailed execution profiling for a query. Present in the response when the request sets `profile: true`. error (str | Unset): Error message if the query failed. @@ -41,6 +43,7 @@ class QueryResultBase: hits: QueryHits | Unset = UNSET aggregations: QueryResultBaseAggregations | Unset = UNSET analyses: QueryResultBaseAnalyses | Unset = UNSET + graph_metric_results: QueryResultBaseGraphMetricResults | Unset = UNSET profile: QueryProfile | Unset = UNSET error: str | Unset = UNSET table: str | Unset = UNSET @@ -63,6 +66,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() + graph_metric_results: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_results, Unset): + graph_metric_results = self.graph_metric_results.to_dict() + profile: dict[str, Any] | Unset = UNSET if not isinstance(self.profile, Unset): profile = self.profile.to_dict() @@ -85,6 +92,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["aggregations"] = aggregations if analyses is not UNSET: field_dict["analyses"] = analyses + if graph_metric_results is not UNSET: + field_dict["graph_metric_results"] = graph_metric_results if profile is not UNSET: field_dict["profile"] = profile if error is not UNSET: @@ -100,6 +109,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults d = dict(src_dict) took = d.pop("took") @@ -127,6 +137,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: analyses = QueryResultBaseAnalyses.from_dict(_analyses) + _graph_metric_results = d.pop("graph_metric_results", UNSET) + graph_metric_results: QueryResultBaseGraphMetricResults | Unset + if isinstance(_graph_metric_results, Unset): + graph_metric_results = UNSET + else: + graph_metric_results = QueryResultBaseGraphMetricResults.from_dict(_graph_metric_results) + _profile = d.pop("profile", UNSET) profile: QueryProfile | Unset if isinstance(_profile, Unset): @@ -144,6 +161,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: hits=hits, aggregations=aggregations, analyses=analyses, + graph_metric_results=graph_metric_results, profile=profile, error=error, table=table, diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_result_base_graph_metric_results.py b/py/packages/sdk/src/antfly/client_generated/models/query_result_base_graph_metric_results.py new file mode 100644 index 0000000000..e67f768fc1 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/query_result_base_graph_metric_results.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.graph_metric_result import GraphMetricResult + + +T = TypeVar("T", bound="QueryResultBaseGraphMetricResults") + + +@_attrs_define +class QueryResultBaseGraphMetricResults: + """Results from direct graph metric reads.""" + + additional_properties: dict[str, GraphMetricResult] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_result import GraphMetricResult + + d = dict(src_dict) + query_result_base_graph_metric_results = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = GraphMetricResult.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + query_result_base_graph_metric_results.additional_properties = additional_properties + return query_result_base_graph_metric_results + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> GraphMetricResult: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: GraphMetricResult) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/query_score_details.py b/py/packages/sdk/src/antfly/client_generated/models/query_score_details.py new file mode 100644 index 0000000000..ac36aaae8d --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/query_score_details.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.graph_metric_rerank_score_details import GraphMetricRerankScoreDetails + + +T = TypeVar("T", bound="QueryScoreDetails") + + +@_attrs_define +class QueryScoreDetails: + """Optional score provenance for ranking features that changed the final hit score. + + Attributes: + graph_metric_rerank (GraphMetricRerankScoreDetails | Unset): + """ + + graph_metric_rerank: GraphMetricRerankScoreDetails | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + graph_metric_rerank: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_rerank, Unset): + graph_metric_rerank = self.graph_metric_rerank.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if graph_metric_rerank is not UNSET: + field_dict["graph_metric_rerank"] = graph_metric_rerank + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.graph_metric_rerank_score_details import GraphMetricRerankScoreDetails + + d = dict(src_dict) + _graph_metric_rerank = d.pop("graph_metric_rerank", UNSET) + graph_metric_rerank: GraphMetricRerankScoreDetails | Unset + if isinstance(_graph_metric_rerank, Unset): + graph_metric_rerank = UNSET + else: + graph_metric_rerank = GraphMetricRerankScoreDetails.from_dict(_graph_metric_rerank) + + query_score_details = cls( + graph_metric_rerank=graph_metric_rerank, + ) + + query_score_details.additional_properties = d + return query_score_details + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/retrieval_query_request.py b/py/packages/sdk/src/antfly/client_generated/models/retrieval_query_request.py index a71a63ec52..73cc2b2591 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/retrieval_query_request.py +++ b/py/packages/sdk/src/antfly/client_generated/models/retrieval_query_request.py @@ -21,6 +21,8 @@ from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -302,6 +304,12 @@ class RetrievalQueryRequest: Has minor performance overhead — not recommended for production traffic. reranker (RerankerConfig | Unset): A unified configuration for a reranking provider. Example: {'provider': 'cohere', 'model': 'rerank-v4.0-pro', 'field': 'content'}. + graph_metric (GraphMetricQuery | Unset): Reads a published graph metric. Score-bearing graph metric queries on + multi-shard tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + graph_metric_rerank (GraphMetricRerank | Unset): Blends a published graph metric into hit scores. Multi-shard + tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required. analyses (Analyses | Unset): graph_queries (GraphQueries | Unset): Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH @@ -494,6 +502,8 @@ class RetrievalQueryRequest: count: bool | Unset = UNSET profile: bool | Unset = UNSET reranker: RerankerConfig | Unset = UNSET + graph_metric: GraphMetricQuery | Unset = UNSET + graph_metric_rerank: GraphMetricRerank | Unset = UNSET analyses: Analyses | Unset = UNSET graph_queries: GraphQueries | Unset = UNSET document_renderer: str | Unset = UNSET @@ -771,6 +781,14 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.reranker, Unset): reranker = self.reranker.to_dict() + graph_metric: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric, Unset): + graph_metric = self.graph_metric.to_dict() + + graph_metric_rerank: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_rerank, Unset): + graph_metric_rerank = self.graph_metric_rerank.to_dict() + analyses: dict[str, Any] | Unset = UNSET if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() @@ -854,6 +872,10 @@ def to_dict(self) -> dict[str, Any]: field_dict["profile"] = profile if reranker is not UNSET: field_dict["reranker"] = reranker + if graph_metric is not UNSET: + field_dict["graph_metric"] = graph_metric + if graph_metric_rerank is not UNSET: + field_dict["graph_metric_rerank"] = graph_metric_rerank if analyses is not UNSET: field_dict["analyses"] = analyses if graph_queries is not UNSET: @@ -885,6 +907,8 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -1722,6 +1746,20 @@ def _parse_exclusion_query( else: reranker = RerankerConfig.from_dict(_reranker) + _graph_metric = d.pop("graph_metric", UNSET) + graph_metric: GraphMetricQuery | Unset + if isinstance(_graph_metric, Unset): + graph_metric = UNSET + else: + graph_metric = GraphMetricQuery.from_dict(_graph_metric) + + _graph_metric_rerank = d.pop("graph_metric_rerank", UNSET) + graph_metric_rerank: GraphMetricRerank | Unset + if isinstance(_graph_metric_rerank, Unset): + graph_metric_rerank = UNSET + else: + graph_metric_rerank = GraphMetricRerank.from_dict(_graph_metric_rerank) + _analyses = d.pop("analyses", UNSET) analyses: Analyses | Unset if isinstance(_analyses, Unset): @@ -1794,6 +1832,8 @@ def _parse_exclusion_query( count=count, profile=profile, reranker=reranker, + graph_metric=graph_metric, + graph_metric_rerank=graph_metric_rerank, analyses=analyses, graph_queries=graph_queries, document_renderer=document_renderer, diff --git a/py/packages/sdk/src/antfly/client_generated/models/stateful_query_request.py b/py/packages/sdk/src/antfly/client_generated/models/stateful_query_request.py index f16625e3aa..7db9098861 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/stateful_query_request.py +++ b/py/packages/sdk/src/antfly/client_generated/models/stateful_query_request.py @@ -22,6 +22,8 @@ from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -299,6 +301,12 @@ class StatefulQueryRequest: Has minor performance overhead — not recommended for production traffic. reranker (RerankerConfig | Unset): A unified configuration for a reranking provider. Example: {'provider': 'cohere', 'model': 'rerank-v4.0-pro', 'field': 'content'}. + graph_metric (GraphMetricQuery | Unset): Reads a published graph metric. Score-bearing graph metric queries on + multi-shard tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + graph_metric_rerank (GraphMetricRerank | Unset): Blends a published graph metric into hit scores. Multi-shard + tables require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required. analyses (Analyses | Unset): graph_queries (GraphQueries | Unset): Named canonical graph operations. When graph_queries is present it must contain at least one operation. A request may contain at most 64 operations, of which at most eight may be MATCH @@ -503,6 +511,8 @@ class StatefulQueryRequest: count: bool | Unset = UNSET profile: bool | Unset = UNSET reranker: RerankerConfig | Unset = UNSET + graph_metric: GraphMetricQuery | Unset = UNSET + graph_metric_rerank: GraphMetricRerank | Unset = UNSET analyses: Analyses | Unset = UNSET graph_queries: GraphQueries | Unset = UNSET document_renderer: str | Unset = UNSET @@ -781,6 +791,14 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.reranker, Unset): reranker = self.reranker.to_dict() + graph_metric: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric, Unset): + graph_metric = self.graph_metric.to_dict() + + graph_metric_rerank: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_rerank, Unset): + graph_metric_rerank = self.graph_metric_rerank.to_dict() + analyses: dict[str, Any] | Unset = UNSET if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() @@ -868,6 +886,10 @@ def to_dict(self) -> dict[str, Any]: field_dict["profile"] = profile if reranker is not UNSET: field_dict["reranker"] = reranker + if graph_metric is not UNSET: + field_dict["graph_metric"] = graph_metric + if graph_metric_rerank is not UNSET: + field_dict["graph_metric_rerank"] = graph_metric_rerank if analyses is not UNSET: field_dict["analyses"] = analyses if graph_queries is not UNSET: @@ -901,6 +923,8 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.geo_bounding_polygon_query import GeoBoundingPolygonQuery from ..models.geo_distance_query import GeoDistanceQuery from ..models.geo_shape_query import GeoShapeQuery + from ..models.graph_metric_query import GraphMetricQuery + from ..models.graph_metric_rerank import GraphMetricRerank from ..models.graph_queries import GraphQueries from ..models.ip_range_query import IPRangeQuery from ..models.join_clause import JoinClause @@ -1738,6 +1762,20 @@ def _parse_exclusion_query( else: reranker = RerankerConfig.from_dict(_reranker) + _graph_metric = d.pop("graph_metric", UNSET) + graph_metric: GraphMetricQuery | Unset + if isinstance(_graph_metric, Unset): + graph_metric = UNSET + else: + graph_metric = GraphMetricQuery.from_dict(_graph_metric) + + _graph_metric_rerank = d.pop("graph_metric_rerank", UNSET) + graph_metric_rerank: GraphMetricRerank | Unset + if isinstance(_graph_metric_rerank, Unset): + graph_metric_rerank = UNSET + else: + graph_metric_rerank = GraphMetricRerank.from_dict(_graph_metric_rerank) + _analyses = d.pop("analyses", UNSET) analyses: Analyses | Unset if isinstance(_analyses, Unset): @@ -1817,6 +1855,8 @@ def _parse_exclusion_query( count=count, profile=profile, reranker=reranker, + graph_metric=graph_metric, + graph_metric_rerank=graph_metric_rerank, analyses=analyses, graph_queries=graph_queries, document_renderer=document_renderer, diff --git a/py/packages/sdk/src/antfly/client_generated/models/stateful_query_result.py b/py/packages/sdk/src/antfly/client_generated/models/stateful_query_result.py index 3173329bb9..0701cb05e9 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/stateful_query_result.py +++ b/py/packages/sdk/src/antfly/client_generated/models/stateful_query_result.py @@ -13,6 +13,7 @@ from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults from ..models.stateful_graph_query_results import StatefulGraphQueryResults @@ -31,6 +32,7 @@ class StatefulQueryResult: names from the request. Contains computed metrics or buckets depending on the aggregation type. analyses (QueryResultBaseAnalyses | Unset): Analysis results like PCA and t-SNE per index embeddings. + graph_metric_results (QueryResultBaseGraphMetricResults | Unset): Results from direct graph metric reads. profile (QueryProfile | Unset): Detailed execution profiling for a query. Present in the response when the request sets `profile: true`. error (str | Unset): Error message if the query failed. @@ -44,6 +46,7 @@ class StatefulQueryResult: hits: QueryHits | Unset = UNSET aggregations: QueryResultBaseAggregations | Unset = UNSET analyses: QueryResultBaseAnalyses | Unset = UNSET + graph_metric_results: QueryResultBaseGraphMetricResults | Unset = UNSET profile: QueryProfile | Unset = UNSET error: str | Unset = UNSET table: str | Unset = UNSET @@ -67,6 +70,10 @@ def to_dict(self) -> dict[str, Any]: if not isinstance(self.analyses, Unset): analyses = self.analyses.to_dict() + graph_metric_results: dict[str, Any] | Unset = UNSET + if not isinstance(self.graph_metric_results, Unset): + graph_metric_results = self.graph_metric_results.to_dict() + profile: dict[str, Any] | Unset = UNSET if not isinstance(self.profile, Unset): profile = self.profile.to_dict() @@ -93,6 +100,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["aggregations"] = aggregations if analyses is not UNSET: field_dict["analyses"] = analyses + if graph_metric_results is not UNSET: + field_dict["graph_metric_results"] = graph_metric_results if profile is not UNSET: field_dict["profile"] = profile if error is not UNSET: @@ -110,6 +119,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.query_profile import QueryProfile from ..models.query_result_base_aggregations import QueryResultBaseAggregations from ..models.query_result_base_analyses import QueryResultBaseAnalyses + from ..models.query_result_base_graph_metric_results import QueryResultBaseGraphMetricResults from ..models.stateful_graph_query_results import StatefulGraphQueryResults d = dict(src_dict) @@ -138,6 +148,13 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: else: analyses = QueryResultBaseAnalyses.from_dict(_analyses) + _graph_metric_results = d.pop("graph_metric_results", UNSET) + graph_metric_results: QueryResultBaseGraphMetricResults | Unset + if isinstance(_graph_metric_results, Unset): + graph_metric_results = UNSET + else: + graph_metric_results = QueryResultBaseGraphMetricResults.from_dict(_graph_metric_results) + _profile = d.pop("profile", UNSET) profile: QueryProfile | Unset if isinstance(_profile, Unset): @@ -162,6 +179,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: hits=hits, aggregations=aggregations, analyses=analyses, + graph_metric_results=graph_metric_results, profile=profile, error=error, table=table, diff --git a/py/packages/sdk/tests/test_client.py b/py/packages/sdk/tests/test_client.py index 50d795c856..12f93a8068 100644 --- a/py/packages/sdk/tests/test_client.py +++ b/py/packages/sdk/tests/test_client.py @@ -35,8 +35,20 @@ from antfly.client_generated.models.graph_document_term_filter import ( # noqa: E402 GraphDocumentTermFilter, ) +from antfly.client_generated.models.graph_index_stats import GraphIndexStats # noqa: E402 +from antfly.client_generated.models.graph_index_stats_index_type import GraphIndexStatsIndexType # noqa: E402 from antfly.client_generated.models.graph_match_node import GraphMatchNode # noqa: E402 from antfly.client_generated.models.graph_match_query import GraphMatchQuery # noqa: E402 +from antfly.client_generated.models.graph_metric_build_page_status import GraphMetricBuildPageStatus # noqa: E402 +from antfly.client_generated.models.graph_metric_build_page_status_range_kind import ( + GraphMetricBuildPageStatusRangeKind, # noqa: E402 +) +from antfly.client_generated.models.graph_metric_query import GraphMetricQuery # noqa: E402 +from antfly.client_generated.models.graph_metric_query_metric_freshness import ( + GraphMetricQueryMetricFreshness, # noqa: E402 +) +from antfly.client_generated.models.graph_metric_runtime_stats import GraphMetricRuntimeStats # noqa: E402 +from antfly.client_generated.models.graph_metric_runtime_stats_role import GraphMetricRuntimeStatsRole # noqa: E402 from antfly.client_generated.models.inference_chat_message import InferenceChatMessage # noqa: E402 from antfly.client_generated.models.inference_generate_request import InferenceGenerateRequest # noqa: E402 from antfly.client_generated.models.inference_role import InferenceRole # noqa: E402 @@ -88,6 +100,19 @@ def test_transform_operator_names_are_stable(self) -> None: "MAX": "$max", } + def test_graph_metric_summary_build_page_deserializes(self) -> None: + page = GraphMetricBuildPageStatus.from_dict( + { + "phase": "reduce_ranks", + "iteration": 2, + "page_id": 0, + "state": "complete", + "range_kind": "summary", + } + ) + + assert page.range_kind is GraphMetricBuildPageStatusRangeKind.SUMMARY + def test_generated_create_index_request_is_discriminated_and_path_owned(self) -> None: request = CreateEmbeddingsIndexRequest( type_=CreateEmbeddingsIndexRequestType.EMBEDDINGS, @@ -197,6 +222,53 @@ def test_normalize_base_url(self) -> None: == "https://platform.antfly.io/cloud/v1/instance" ) + def test_graph_index_stats_model_serializes_metric_runtime(self) -> None: + stats = GraphIndexStats( + index_type=GraphIndexStatsIndexType.GRAPH, + total_edges=4, + graph_metric_runtime=GraphMetricRuntimeStats( + enabled=True, + role=GraphMetricRuntimeStatsRole.WORKER_POOL, + owner_id_hash=17, + worker_count=3, + takeover_count=2, + lost_leases=1, + total_pages_claimed=6, + last_pages_completed=3, + last_budget_exhausted=True, + ), + ) + + stats_dict = stats.to_dict() + assert stats_dict["graph_metric_runtime"]["role"] == "worker_pool" + assert stats_dict["graph_metric_runtime"]["owner_id_hash"] == 17 + assert stats_dict["graph_metric_runtime"]["last_budget_exhausted"] is True + + round_tripped = GraphIndexStats.from_dict(stats_dict) + assert isinstance(round_tripped.graph_metric_runtime, GraphMetricRuntimeStats) + assert round_tripped.graph_metric_runtime.role == GraphMetricRuntimeStatsRole.WORKER_POOL + assert round_tripped.graph_metric_runtime.worker_count == 3 + assert round_tripped.graph_metric_runtime.total_pages_claimed == 6 + + def test_direct_graph_metric_query_round_trip(self) -> None: + query = GraphMetricQuery( + name="central", + index="graph_idx", + metric="pagerank", + top_k=25, + metric_freshness=GraphMetricQueryMetricFreshness.FRESH, + ) + + payload = query.to_dict() + assert payload == { + "name": "central", + "index": "graph_idx", + "metric": "pagerank", + "top_k": 25, + "metric_freshness": "fresh", + } + assert GraphMetricQuery.from_dict(payload) == query + def test_response_limits_must_be_positive(self) -> None: with pytest.raises(ValueError, match="response byte limits must be positive"): AntflyClient("http://localhost:8080", max_json_response_bytes=0) diff --git a/rs/crates/sdk/src/lib.rs b/rs/crates/sdk/src/lib.rs index 5df8b78dfd..75d1009e4a 100644 --- a/rs/crates/sdk/src/lib.rs +++ b/rs/crates/sdk/src/lib.rs @@ -660,6 +660,7 @@ impl Default for types::CreateGraphIndexRequest { edge_types: Vec::new(), enrichments: Vec::new(), max_edges_per_document: None, + metrics: Default::default(), resolvers: Vec::new(), source: None, sources: None, @@ -708,6 +709,22 @@ impl types::CreateIndexError { #[cfg(test)] mod tests { + #[test] + fn graph_index_default_and_metric_configuration_round_trip() { + let default = super::types::CreateGraphIndexRequest::default(); + let mut json = serde_json::to_value(default).unwrap(); + assert!( + json.get("metrics").is_none_or(|value| value.is_null() + || value.as_object().is_some_and(|object| object.is_empty())) + ); + json["metrics"] = serde_json::json!({"rank": {"kind": "pagerank"}}); + let configured: super::types::CreateGraphIndexRequest = + serde_json::from_value(json).unwrap(); + assert_eq!( + serde_json::to_value(configured).unwrap()["metrics"]["rank"]["kind"], + "pagerank" + ); + } use super::{ ArtifactEmbeddingIndexOptions, ArtifactEmbeddingSourceSpec, ArtifactFullTextIndexOptions, FullTextArtifactSourceSpec, GraphArtifactFormat, GraphContextMappingSpec, diff --git a/scripts/packaging/test_cabi_packaging.py b/scripts/packaging/test_cabi_packaging.py index 67e2f6134c..14562695b1 100644 --- a/scripts/packaging/test_cabi_packaging.py +++ b/scripts/packaging/test_cabi_packaging.py @@ -128,7 +128,7 @@ def test_homebrew_job_provisions_packaging_toolchains(self) -> None: self.assertIn("steps.toolchain.outputs.zig_nixpkgs_revision", bootstrap) self.assertIn("steps.toolchain.outputs.zig_nix_attribute", bootstrap) self.assertIn("steps.toolchain.outputs.zig_version", bootstrap) - self.assertIn("nix-build '' -A \"$ZIG_NIX_ATTRIBUTE\"", bootstrap) + self.assertIn("nix-build '' -A " + '"$ZIG_NIX_ATTRIBUTE"', bootstrap) self.assertIn('echo "$zig_path/bin" >> "$GITHUB_PATH"', bootstrap) self.assertIn("grep -q 'dynamically linked'", bootstrap) diff --git a/specs/openapi/antfly/indexes.yaml b/specs/openapi/antfly/indexes.yaml index 25f1c334e5..0ff985dcf7 100644 --- a/specs/openapi/antfly/indexes.yaml +++ b/specs/openapi/antfly/indexes.yaml @@ -472,6 +472,9 @@ components: type: object description: "Credential-free normalized graph configuration returned after creation." properties: + metrics: + type: object + additionalProperties: { $ref: "#/components/schemas/GraphMetricConfig" } summarizer: { $ref: "#/components/schemas/CreatedProviderConfig" } template: { type: string } edge_types: @@ -2351,6 +2354,10 @@ components: x-antfly-mutually-exclusive: - [sources, source] properties: + metrics: + type: object + description: "Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name." + additionalProperties: { $ref: "#/components/schemas/GraphMetricConfig" } sources: type: array minItems: 1 @@ -2394,6 +2401,53 @@ components: type: array items: $ref: "#/components/schemas/GraphResolverConfig" + GraphMetricConfig: + type: object + additionalProperties: false + description: "Published metric configuration. If kind is omitted, the metric name must be a supported kind." + properties: + enabled: { type: boolean, default: true } + kind: + type: string + enum: [pagerank, degree, eigenvector, hits_authority, hits_hub] + refresh: + type: string + enum: [background, manual] + default: background + description: "Serverless accepts background only." + damping: + type: number + format: double + minimum: 0 + exclusiveMinimum: true + maximum: 1 + exclusiveMaximum: true + default: 0.85 + tolerance: + type: number + format: double + minimum: 0 + exclusiveMinimum: true + default: 0.000001 + max_iterations: + type: integer + format: int32 + minimum: 1 + maximum: 1000 + default: 50 + edge_filter: + $ref: "#/components/schemas/GraphMetricEdgeFilter" + GraphMetricEdgeFilter: + type: object + additionalProperties: false + description: "Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied." + properties: + mode: { type: string, enum: [all] } + types: + type: array + minItems: 1 + uniqueItems: true + items: { $ref: "#/components/schemas/GraphEdgeType" } GraphEdgeType: type: string minLength: 1 @@ -2667,6 +2721,161 @@ components: result_nodes: type: integer format: uint64 + graph_metric_runtime: + x-go-type-skip-optional-pointer: true + $ref: "#/components/schemas/GraphMetricRuntimeStats" + GraphMetricRuntimeStats: + type: object + description: "Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers." + properties: + enabled: + type: boolean + role: + type: string + enum: + - combined + - coordinator + - worker + - worker_pool + runtime_id_hash: + type: integer + format: uint64 + owner_id_hash: + type: integer + format: uint64 + lease_key_hash: + type: integer + format: uint64 + worker_id_hash: + type: integer + format: uint64 + worker_count: + type: integer + format: uint64 + lease_owned: + type: boolean + has_lease: + type: boolean + acquisition_count: + type: integer + format: uint64 + takeover_count: + type: integer + format: uint64 + lease_acquire_failures: + type: integer + format: uint64 + lost_leases: + type: integer + format: uint64 + last_acquired_ms: + type: integer + format: uint64 + lease_expires_at_ms: + type: integer + format: uint64 + description: "Cached expiry of the currently held maintenance lease, or zero when no lease is held." + lease_renew_after_ms: + type: integer + format: uint64 + description: "Earliest time the runtime will renew its maintenance lease, or zero when no lease is held." + renewal_count: + type: integer + format: uint64 + description: "Number of durable maintenance lease renewals completed by this runtime." + started: + type: boolean + shutdown: + type: boolean + notified: + type: boolean + ticks_started: + type: integer + format: uint64 + ticks_completed: + type: integer + format: uint64 + durable_progress_ticks: + type: integer + format: uint64 + idle_ticks: + type: integer + format: uint64 + error_ticks: + type: integer + format: uint64 + last_error_name: + type: string + total_metrics_scanned: + type: integer + format: uint64 + total_active_builds: + type: integer + format: uint64 + total_builds_started: + type: integer + format: uint64 + total_worker_steps: + type: integer + format: uint64 + total_coordinator_steps: + type: integer + format: uint64 + total_retired_input_records: + description: Consumed intermediate records retired at completed reduction barriers. + type: integer + format: uint64 + total_pages_claimed: + type: integer + format: uint64 + total_pages_completed: + type: integer + format: uint64 + total_phases_advanced: + type: integer + format: uint64 + total_published: + type: integer + format: uint64 + total_failed_builds: + type: integer + format: uint64 + last_metrics_scanned: + type: integer + format: uint64 + last_active_builds: + type: integer + format: uint64 + last_builds_started: + type: integer + format: uint64 + last_worker_steps: + type: integer + format: uint64 + last_coordinator_steps: + type: integer + format: uint64 + last_retired_input_records: + description: Consumed intermediate records retired in the latest maintenance tick. + type: integer + format: uint64 + last_pages_claimed: + type: integer + format: uint64 + last_pages_completed: + type: integer + format: uint64 + last_phases_advanced: + type: integer + format: uint64 + last_published: + type: integer + format: uint64 + last_failed_builds: + type: integer + format: uint64 + last_budget_exhausted: + type: boolean Edge: x-go-type-skip-optional-pointer: true type: object @@ -3216,6 +3425,39 @@ components: x-go-type-skip-optional-pointer: true type: array items: { type: string } + metrics: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 16 + uniqueItems: true + items: + type: string + minLength: 1 + description: Graph metric names to project onto legacy graph_searches result nodes. + order_by: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 8 + uniqueItems: true + items: + $ref: '#/components/schemas/GraphMetricOrder' + description: Sort legacy graph_searches result nodes by graph metric score. + where_metric: + x-go-type-skip-optional-pointer: true + type: array + maxItems: 32 + items: + $ref: '#/components/schemas/GraphMetricFilter' + description: Filter legacy graph_searches result nodes by graph metric score. + metric_freshness: + x-go-type-skip-optional-pointer: true + type: string + enum: [published, fresh] + description: Freshness required for projected, ordered, and filtered graph metrics. + include_metric_status: + x-go-type-skip-optional-pointer: true + type: boolean + description: Include graph metric status metadata in the legacy graph_searches result. LegacyGraphDocumentQuery: type: object additionalProperties: true @@ -3586,6 +3828,32 @@ components: type: array items: { type: string } description: Requires include_documents=true. Omit to include all document fields. + metrics: + type: array + maxItems: 16 + uniqueItems: true + items: { type: string, minLength: 1 } + description: Graph metric names to project onto returned traversal nodes. + order_by: + type: array + maxItems: 8 + uniqueItems: true + items: { $ref: '#/components/schemas/GraphMetricOrder' } + description: Sort traversal candidates by graph metric score before applying limit. + where_metric: + type: array + maxItems: 32 + items: { $ref: '#/components/schemas/GraphMetricFilter' } + description: Filter traversal candidates by graph metric score before applying limit. + metric_freshness: + type: string + enum: [published, fresh] + default: published + description: Freshness required for projected, ordered, and filtered graph metrics. + include_metric_status: + type: boolean + default: false + description: Include graph metric status metadata in the traversal profile. filter: $ref: '#/components/schemas/GraphDocumentFilter' description: Non-scoring structured stored-document predicate for reached nodes. @@ -4110,6 +4378,11 @@ components: items: type: string description: "Algebraic provenance labels folded into this result, when requested by an algebraic graph executor" + metrics: + x-go-type-skip-optional-pointer: true + type: object + additionalProperties: true + description: "Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score for the node." evidence: x-go-type-skip-optional-pointer: true type: object @@ -4283,6 +4556,11 @@ components: maxItems: 10000 items: { $ref: '#/components/schemas/GraphResultNode' } description: Traversal result nodes; requested paths are stored on each node. + metric_status: + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricStatus' + description: Graph metric status metadata keyed by metric name when requested. stats: { $ref: '#/components/schemas/GraphResultStats' } GraphPathResult: x-go-type-skip-optional-pointer: true @@ -4364,6 +4642,12 @@ components: format: int64 deprecated: true description: "Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field." + metric_status: + x-go-type-skip-optional-pointer: true + type: object + additionalProperties: + $ref: '#/components/schemas/GraphMetricStatus' + description: Graph metric status metadata keyed by metric name. GraphResult: description: >- A canonical result produced by graph_queries. Bindings, exact @@ -4446,3 +4730,397 @@ components: additionalProperties: { $ref: '#/components/schemas/GraphResultBinding' } x-antfly-property-name-schema: $ref: 'generated/graph_identifier.yaml#/components/schemas/GraphIdentifier' + GraphMetricStatus: + type: object + required: + - state + - phase + - published_generation + - edge_generation + - target_edge_generation + - build_queued + - progress + - converged + - iterations_completed + - delta + - computed_at_ms + properties: + state: + type: string + phase: + type: string + enum: + - idle + - computing + - publishing + - complete + - prepare_generation + - scan_edges_and_out_degree + - initialize_ranks + - iterate_contributions + - reduce_ranks + - hits_hub_contributions + - hits_hub_reduce_ranks + - check_convergence + - publish_generation + - cleanup_old_generations + edge_filter: + $ref: '#/components/schemas/GraphMetricEdgeFilterStatus' + metadata_version: + type: integer + format: int64 + description: "Version of the published graph metric metadata schema." + config_fingerprint: + type: string + pattern: "^[0-9a-f]{16}$" + description: "Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every SDK preserves all 64 bits." + maintenance_paused: + type: boolean + build_queued: + type: boolean + description: "Whether a local or distributed build is queued after the currently published or building generation." + published_generation: + type: integer + format: int64 + edge_generation: + type: integer + format: int64 + target_edge_generation: + type: integer + format: int64 + queued_generation: + type: integer + format: int64 + description: "Pending edge generation waiting to build, or 0 when no build is queued." + building_generation: + type: integer + format: int64 + description: "Edge generation currently held by an active build lease, or 0 when idle." + build_job_id: + type: integer + format: int64 + description: "Durable identifier for the active graph metric build job, or 0 when idle." + build_started_at_ms: + type: integer + format: int64 + description: "Unix epoch milliseconds when the active graph metric build started, or 0 when idle." + build_iteration: + type: integer + format: int64 + description: "Iteration number reported by the active build lease, or 0 when idle or not iterative." + build_lease_expires_at_ms: + type: integer + format: int64 + description: "Unix epoch milliseconds when the active build lease expires, or 0 when idle." + build_worker_id: + type: string + description: "Worker id that owns the active build lease. Local builds use `local`." + build_cursor: + type: string + description: "Opaque resumable cursor for the active build phase. Empty or omitted when idle or when the phase has no cursor." + build_completed_units: + type: integer + format: int64 + description: "Completed work units for the active graph metric build, or 0 when idle or unknown." + build_total_units: + type: integer + format: int64 + description: "Estimated total work units for the active graph metric build, or 0 when idle or unknown." + build_pages: + type: array + description: "Active leased or failed build pages for the current build phase, capped and ordered by durable page key." + items: + $ref: '#/components/schemas/GraphMetricBuildPageStatus' + build_pages_truncated: + type: boolean + description: "Whether build_pages was capped before every active page could be included." + retry_count: + type: integer + format: int64 + description: "Number of consecutive failed build attempts for the current target generation, or 0 when no failure applies." + last_error: + type: string + description: "Last build error for the current failed target generation." + progress: + type: number + format: double + description: "Build progress for the target edge generation, from 0.0 to 1.0" + converged: + type: boolean + iterations_completed: + type: integer + format: int64 + delta: + type: number + format: double + computed_at_ms: + type: integer + format: int64 + last_event: + $ref: '#/components/schemas/GraphMetricEvent' + recent_events: + type: array + description: "Recent graph metric events, newest first." + items: + $ref: '#/components/schemas/GraphMetricEvent' + GraphMetricBuildPageStatus: + type: object + required: + - phase + - iteration + - page_id + - state + - range_kind + properties: + phase: + type: string + iteration: + type: integer + format: int64 + page_id: + type: integer + format: int64 + state: + type: string + enum: + - pending + - leased + - complete + - failed + range_kind: + type: string + enum: + - full + - reverse_edges + - nodes + - scores + - contributions + - job_control + - summary + worker_id: + type: string + description: "Worker id that owns or last failed this page." + lease_expires_at_ms: + type: integer + format: int64 + description: "Unix epoch milliseconds when the page lease expires, or 0 when not leased." + attempt: + type: integer + format: int64 + description: "Current attempt number for this page." + cursor: + type: string + description: "Opaque resumable cursor for this page." + completed_units: + type: integer + format: int64 + description: "Completed work units for this page." + total_units: + type: integer + format: int64 + description: "Estimated total work units for this page." + last_error: + type: string + description: "Last page-level error." + GraphMetricEvent: + type: object + required: + - sequence + - kind + - at_ms + - target_edge_generation + - published_generation + - score_count + properties: + sequence: + type: integer + format: int64 + kind: + type: string + enum: + - publish + - delete + - pause + - resume + - failed + at_ms: + type: integer + format: int64 + target_edge_generation: + type: integer + format: int64 + published_generation: + type: integer + format: int64 + score_count: + type: integer + format: int64 + GraphMetricEdgeFilterStatus: + type: object + required: + - mode + properties: + mode: + type: string + enum: + - all + - types + types: + type: array + items: + type: string + GraphMetricScore: + type: object + required: + - node + - score + properties: + node: + type: string + score: + type: number + format: double + GraphMetricQuery: + type: object + description: >- + Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables + require a globally coordinated metric snapshot and otherwise return + graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. + required: + - index + - metric + properties: + name: + type: string + minLength: 1 + description: Optional result key. Defaults to the metric name. + index: + type: string + minLength: 1 + description: Graph index that owns the published metric. + metric: + type: string + minLength: 1 + description: Graph metric to read. + top_k: + type: integer + format: int32 + minimum: 1 + maximum: 10000 + default: 10 + description: Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated metric snapshot. + metric_freshness: + type: string + enum: + - published + - fresh + default: published + description: Whether the latest published generation may be stale or must match the graph edge generation. + GraphMetricRerank: + type: object + description: >- + Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated + metric snapshot and otherwise return graph_metric_global_materialization_required. + required: + - index + - metric + properties: + index: + type: string + description: Graph index that owns the published metric. + metric: + type: string + description: Graph metric name to blend into the search hit score. + candidate_count: + type: integer + format: int32 + minimum: 1 + maximum: 10000 + description: >- + Bounded retrieval window scored by the graph metric before offset and limit are applied. + When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. + An explicit value must cover offset plus limit. Larger windows improve promotion recall at + predictable linear score-read cost. + base_weight: + type: number + format: double + default: 1.0 + description: Multiplier applied to the existing hit score before adding the graph metric feature. + weight: + type: number + format: double + default: 1.0 + description: Multiplier applied to the graph metric score before it is added to the existing hit score. + missing_score: + type: number + format: double + default: 0.0 + description: Metric feature value to use for hits that do not have a score in the published metric generation. + metric_freshness: + type: string + enum: + - published + - fresh + default: published + description: Whether stale published generations are acceptable or the metric must be fresh. + GraphMetricResult: + type: object + required: + - index_name + - metric + - scores + - status + properties: + index_name: + type: string + metric: + type: string + scores: + type: array + items: + $ref: '#/components/schemas/GraphMetricScore' + status: + $ref: '#/components/schemas/GraphMetricStatus' + GraphMetricOrder: + type: object + required: + - metric + properties: + metric: + type: string + minLength: 1 + direction: + type: string + enum: + - asc + - desc + nulls: + type: string + enum: + - first + - last + - nulls_first + - nulls_last + GraphMetricFilter: + type: object + required: + - metric + - op + - value + properties: + metric: + type: string + minLength: 1 + op: + type: string + description: Semantic comparison operator. Named values keep generated SDK enums portable and readable. + enum: + - gt + - gte + - lt + - lte + - eq + - neq + value: + type: number + format: double diff --git a/specs/openapi/antfly/metadata.yaml b/specs/openapi/antfly/metadata.yaml index b0c07a54ca..ee7174d83f 100644 --- a/specs/openapi/antfly/metadata.yaml +++ b/specs/openapi/antfly/metadata.yaml @@ -4428,6 +4428,13 @@ components: $ref: "indexes.yaml#/components/schemas/CreatedIndex" status: $ref: "indexes.yaml#/components/schemas/IndexStats" + GraphMetricActionResponse: + type: object + required: + - status + properties: + status: + $ref: "indexes.yaml#/components/schemas/GraphMetricStatus" LsmStorageStatus: type: object description: Compact LSM backend operational status. Detailed low-level counters are available through metrics. @@ -5326,8 +5333,9 @@ components: description: | Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` - means the primary write committed, but a terminal enrichment failure needs - operator repair and will not be retried indefinitely. + means the primary write committed, but a terminal background materialization + failure needs operator repair and will not be retried indefinitely. Inspect + `failure` when present; retrying the document write is unnecessary. inserted: type: integer description: Number of documents successfully inserted @@ -5337,6 +5345,28 @@ components: transformed: type: integer description: Number of documents successfully transformed + failure: + $ref: "#/components/schemas/BatchCommittedFailure" + BatchCommittedFailure: + type: object + description: | + Additive details for a committed batch that needs operator action. The + open string code is forward-compatible with older SDKs; clients should + treat unknown codes as non-retryable when `retryable` is false. + required: [code, message, retryable] + properties: + code: + type: string + description: Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + message: + type: string + description: Actionable operator guidance. + reason: + type: string + description: Optional stable reason within the failure category, such as `build_budget_exceeded`. + retryable: + type: boolean + description: Whether replaying the document mutation is safe. Committed repair outcomes are false. DenseRepairBackpressureError: type: object description: A dense-index rebuild is retaining replay history and the node has reached its hard safety budget. @@ -7510,6 +7540,8 @@ components: - required: [count] - required: [profile] - required: [reranker] + - required: [graph_metric] + - required: [graph_metric_rerank] - required: [analyses] - required: [aggregations] - required: [graph_queries] @@ -7912,6 +7944,18 @@ components: "field": "content" } ``` + graph_metric: + $ref: "indexes.yaml#/components/schemas/GraphMetricQuery" + description: >- + Direct top-k read from a published graph metric generation. Results + are returned in graph_metric_results under the requested name or the + metric name when no explicit name is supplied. + graph_metric_rerank: + $ref: "indexes.yaml#/components/schemas/GraphMetricRerank" + description: >- + Blend a published graph metric feature into ordinary search hit + scores. Requests may require either any published generation or a + generation that is fresh with respect to graph writes. analyses: $ref: "#/components/schemas/Analyses" graph_queries: @@ -8289,9 +8333,42 @@ components: merge: $ref: "#/components/schemas/MergeProfile" description: Result merge statistics (present for hybrid search). + graph_metrics: + type: array + items: + $ref: "#/components/schemas/GraphMetricProfile" + description: Graph metric freshness and generation details for metric-aware query work. sort: $ref: "#/components/schemas/SortProfile" description: Sort execution statistics (present when the query used ordered page options and profiling was enabled). + GraphMetricProfile: + type: object + required: + - query_name + - source + - index_name + - metric_name + - freshness + - status + properties: + query_name: + type: string + description: Name of the graph query or graph metric query that used the metric. + source: + type: string + description: Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + index_name: + type: string + description: Graph index that owns the metric. + metric_name: + type: string + description: Graph metric name within the index. + freshness: + type: string + description: Effective freshness mode requested for this metric use. + status: + $ref: "indexes.yaml#/components/schemas/GraphMetricStatus" + description: Published generation and freshness status observed by the query. SortProfile: type: object additionalProperties: false @@ -8785,6 +8862,64 @@ components: items: $ref: "#/components/schemas/HierarchyMatchHit" + QueryScoreDetails: + type: object + description: Optional score provenance for ranking features that changed the final hit score. + properties: + graph_metric_rerank: + $ref: "#/components/schemas/GraphMetricRerankScoreDetails" + description: Score contribution from an explicit graph_metric_rerank request. + GraphMetricRerankScoreDetails: + type: object + required: + - index_name + - metric_name + - base_score + - base_weight + - metric_score_used + - metric_weight + - missing_score_used + - final_score + - published_generation + properties: + index_name: + type: string + description: Graph index that provided the metric score. + metric_name: + type: string + description: Graph metric used as a score feature. + base_score: + type: number + format: double + description: Hit score before graph metric rerank composition. + base_weight: + type: number + format: double + description: Weight applied to the base score. + metric_score: + type: number + format: double + nullable: true + description: Published metric score for this hit, or null when the hit was missing from the metric generation. + metric_score_used: + type: number + format: double + description: Metric feature value used in the formula after applying missing_score fallback if needed. + metric_weight: + type: number + format: double + description: Weight applied to the metric score feature. + missing_score_used: + type: boolean + description: True when metric_score was missing and the request's missing_score fallback was used. + final_score: + type: number + format: double + description: Final hit score after graph metric rerank composition. + published_generation: + type: integer + format: int64 + description: Published graph metric score generation used for this hit. QueryHit: type: object description: A single query result hit @@ -8820,6 +8955,10 @@ components: format: double description: Scores partitioned by index when using RRF search. x-go-name: indexScores + _score_details: + $ref: "#/components/schemas/QueryScoreDetails" + description: Optional score provenance for ranking features applied to this hit. + x-go-name: scoreDetails _source: type: object additionalProperties: true # map[string]any for field values @@ -8924,6 +9063,11 @@ components: additionalProperties: $ref: "#/components/schemas/AnalysesResult" description: Analysis results like PCA and t-SNE per index embeddings. + graph_metric_results: + type: object + additionalProperties: + $ref: "indexes.yaml#/components/schemas/GraphMetricResult" + description: Results from direct graph metric reads. profile: x-go-type-skip-optional-pointer: false allOf: @@ -9995,6 +10139,77 @@ paths: type: string "500": $ref: "#/components/responses/StatelessTransactionFailure" + /tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action}: + parameters: + - name: tableName + in: path + required: true + description: Name of the table + schema: + type: string + - name: indexName + in: path + required: true + description: Name of the graph index + schema: + type: string + - name: metricName + in: path + required: true + description: Name of the configured graph metric + schema: + type: string + - name: action + in: path + required: true + description: Operational action to apply to the graph metric materialization + schema: + type: string + enum: [refresh, rebuild, delete, pause, resume] + post: + summary: Execute a graph metric operational action + description: | + Refresh, rebuild, delete, pause, or resume maintenance for a configured + graph metric. The metric configuration remains owned by the graph index. + Refresh and rebuild durably enqueue bounded, resumable maintenance and + return the aggregate shard status without waiting for graph-sized work. + `delete` clears materialized metric state and durably disables automatic + maintenance. A later refresh, rebuild, or resume action re-enables the + metric and can publish a new generation. + tags: + - index_management + operationId: executeGraphMetricAction + responses: + "200": + description: Aggregate graph metric status after the action is durably accepted + content: + application/json: + schema: + $ref: "#/components/schemas/GraphMetricActionResponse" + "400": + $ref: "#/components/responses/BadRequest" + "409": + description: The table topology or write-owner generation changed, or only some shards accepted the action; retrying is consistency-safe and reuses any still-active build + content: + text/plain: + schema: + type: string + "429": + description: Local storage resources are temporarily exhausted + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "404": + $ref: "#/components/responses/NotFound" + "405": + description: Graph metric actions are unavailable for this runtime + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + $ref: "#/components/responses/InternalServerError" /transactions/commit: post: summary: Commit an OCC transaction diff --git a/ts/packages/sdk/src/index.ts b/ts/packages/sdk/src/index.ts index bd7410c742..39e4ceed60 100644 --- a/ts/packages/sdk/src/index.ts +++ b/ts/packages/sdk/src/index.ts @@ -267,6 +267,7 @@ export type { GraphExactResultStats, GraphIdentityNodeSelector, GraphIndexConfig, + GraphIndexStats, GraphKeyNodeSelector, GraphKShortestPaths, GraphKShortestPathsQuery, @@ -274,6 +275,21 @@ export type { GraphMatchEdge, GraphMatchNode, GraphMatchQuery, + GraphMetricActionResponse, + GraphMetricBuildPageStatus, + GraphMetricEdgeFilterStatus, + GraphMetricEvent, + GraphMetricFilter, + GraphMetricFreshness, + GraphMetricOrder, + GraphMetricProfile, + GraphMetricQuery, + GraphMetricRerank, + GraphMetricRerankScoreDetails, + GraphMetricResult, + GraphMetricRuntimeStats, + GraphMetricScore, + GraphMetricStatus, GraphNodeSelector, GraphNodesResult, GraphNotEqualPredicate, @@ -341,6 +357,7 @@ export type { // Core types QueryResponses, QueryResult, + QueryScoreDetails, QueryStrategy, RerankerConfig, RerankerProfile, diff --git a/ts/packages/sdk/src/public-api.d.ts b/ts/packages/sdk/src/public-api.d.ts index a33d786893..05463a8dc8 100644 --- a/ts/packages/sdk/src/public-api.d.ts +++ b/ts/packages/sdk/src/public-api.d.ts @@ -175,6 +175,41 @@ export interface paths { patch?: never; trace?: never; }; + "/db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action}": { + parameters: { + query?: never; + header?: never; + path: { + /** @description Name of the table */ + tableName: string; + /** @description Name of the graph index */ + indexName: string; + /** @description Name of the configured graph metric */ + metricName: string; + /** @description Operational action to apply to the graph metric materialization */ + action: "refresh" | "rebuild" | "delete" | "pause" | "resume"; + }; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Execute a graph metric operational action + * @description Refresh, rebuild, delete, pause, or resume maintenance for a configured + * graph metric. The metric configuration remains owned by the graph index. + * Refresh and rebuild durably enqueue bounded, resumable maintenance and + * return the aggregate shard status without waiting for graph-sized work. + * `delete` clears materialized metric state and durably disables automatic + * maintenance. A later refresh, rebuild, or resume action re-enables the + * metric and can publish a new generation. + */ + post: operations["executeGraphMetricAction"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/db/v1/transactions/commit": { parameters: { query?: never; @@ -5014,6 +5049,9 @@ export interface components { config: components["schemas"]["CreatedIndex"]; status: components["schemas"]["IndexStats"]; }; + GraphMetricActionResponse: { + status: components["schemas"]["GraphMetricStatus"]; + }; /** @description Compact LSM backend operational status. Detailed low-level counters are available through metrics. */ LsmStorageStatus: { /** Format: uint64 */ @@ -5790,8 +5828,9 @@ export interface components { /** * @description Durable commit outcome. `committed_pending` means requested visibility or * participant propagation is still completing. `committed_repair_required` - * means the primary write committed, but a terminal enrichment failure needs - * operator repair and will not be retried indefinitely. + * means the primary write committed, but a terminal background materialization + * failure needs operator repair and will not be retried indefinitely. Inspect + * `failure` when present; retrying the document write is unnecessary. * @enum {string} */ status?: "committed" | "committed_pending" | "committed_repair_required"; @@ -5801,6 +5840,22 @@ export interface components { deleted?: number; /** @description Number of documents successfully transformed */ transformed?: number; + failure?: components["schemas"]["BatchCommittedFailure"]; + }; + /** + * @description Additive details for a committed batch that needs operator action. The + * open string code is forward-compatible with older SDKs; clients should + * treat unknown codes as non-retryable when `retryable` is false. + */ + BatchCommittedFailure: { + /** @description Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. */ + code: string; + /** @description Actionable operator guidance. */ + message: string; + /** @description Optional stable reason within the failure category, such as `build_budget_exceeded`. */ + reason?: string; + /** @description Whether replaying the document mutation is safe. Committed repair outcomes are false. */ + retryable: boolean; }; /** @description A dense-index rebuild is retaining replay history and the node has reached its hard safety budget. */ DenseRepairBackpressureError: { @@ -7637,6 +7692,10 @@ export interface components { * ``` */ reranker?: components["schemas"]["RerankerConfig"]; + /** @description Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. */ + graph_metric?: components["schemas"]["GraphMetricQuery"]; + /** @description Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. */ + graph_metric_rerank?: components["schemas"]["GraphMetricRerank"]; analyses?: components["schemas"]["Analyses"]; /** * @description Declarative graph matching, traversal, and path queries. A nested node @@ -7952,9 +8011,25 @@ export interface components { reranker?: components["schemas"]["RerankerProfile"]; /** @description Result merge statistics (present for hybrid search). */ merge?: components["schemas"]["MergeProfile"]; + /** @description Graph metric freshness and generation details for metric-aware query work. */ + graph_metrics?: components["schemas"]["GraphMetricProfile"][]; /** @description Sort execution statistics (present when the query used ordered page options and profiling was enabled). */ sort?: components["schemas"]["SortProfile"]; }; + GraphMetricProfile: { + /** @description Name of the graph query or graph metric query that used the metric. */ + query_name: string; + /** @description Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. */ + source: string; + /** @description Graph index that owns the metric. */ + index_name: string; + /** @description Graph metric name within the index. */ + metric_name: string; + /** @description Effective freshness mode requested for this metric use. */ + freshness: string; + /** @description Published generation and freshness status observed by the query. */ + status: components["schemas"]["GraphMetricStatus"]; + }; /** * @description Sort execution profile. These fields are the stable public diagnostic * surface. Low-level implementation counters such as doc-value load @@ -8348,6 +8423,54 @@ export interface components { */ chunks?: components["schemas"]["HierarchyMatchHit"][]; }; + /** @description Optional score provenance for ranking features that changed the final hit score. */ + QueryScoreDetails: { + /** @description Score contribution from an explicit graph_metric_rerank request. */ + graph_metric_rerank?: components["schemas"]["GraphMetricRerankScoreDetails"]; + }; + GraphMetricRerankScoreDetails: { + /** @description Graph index that provided the metric score. */ + index_name: string; + /** @description Graph metric used as a score feature. */ + metric_name: string; + /** + * Format: double + * @description Hit score before graph metric rerank composition. + */ + base_score: number; + /** + * Format: double + * @description Weight applied to the base score. + */ + base_weight: number; + /** + * Format: double + * @description Published metric score for this hit, or null when the hit was missing from the metric generation. + */ + metric_score?: number | null; + /** + * Format: double + * @description Metric feature value used in the formula after applying missing_score fallback if needed. + */ + metric_score_used: number; + /** + * Format: double + * @description Weight applied to the metric score feature. + */ + metric_weight: number; + /** @description True when metric_score was missing and the request's missing_score fallback was used. */ + missing_score_used: boolean; + /** + * Format: double + * @description Final hit score after graph metric rerank composition. + */ + final_score: number; + /** + * Format: int64 + * @description Published graph metric score generation used for this hit. + */ + published_generation: number; + }; /** @description A single query result hit */ QueryHit: { /** @description ID of the record. */ @@ -8369,6 +8492,8 @@ export interface components { _index_scores?: { [key: string]: number; }; + /** @description Optional score provenance for ranking features applied to this hit. */ + _score_details?: components["schemas"]["QueryScoreDetails"]; _source?: { [key: string]: unknown; }; @@ -8436,6 +8561,10 @@ export interface components { analyses?: { [key: string]: components["schemas"]["AnalysesResult"]; }; + /** @description Results from direct graph metric reads. */ + graph_metric_results?: { + [key: string]: components["schemas"]["GraphMetricResult"]; + }; /** @description Detailed execution profile (present when `profile: true` in request). */ profile?: components["schemas"]["QueryProfile"]; /** @@ -9882,6 +10011,41 @@ export interface components { }; /** @description Durable graph edge type. Values must be valid UTF-8 and encode to at most 64 KiB; `maxLength` is the standard-schema code-point ceiling and `x-antfly-max-utf8-bytes` carries the exact wire-byte limit. */ GraphEdgeType: string; + /** @description Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied. */ + GraphMetricEdgeFilter: { + /** @enum {string} */ + mode?: "all"; + types?: components["schemas"]["GraphEdgeType"][]; + }; + /** @description Published metric configuration. If kind is omitted, the metric name must be a supported kind. */ + GraphMetricConfig: { + /** @default true */ + enabled?: boolean; + /** @enum {string} */ + kind?: "pagerank" | "degree" | "eigenvector" | "hits_authority" | "hits_hub"; + /** + * @description Serverless accepts background only. + * @default background + * @enum {string} + */ + refresh?: "background" | "manual"; + /** + * Format: double + * @default 0.85 + */ + damping?: number; + /** + * Format: double + * @default 0.000001 + */ + tolerance?: number; + /** + * Format: int32 + * @default 50 + */ + max_iterations?: number; + edge_filter?: components["schemas"]["GraphMetricEdgeFilter"]; + }; /** @description A literal string or finite numeric value, or a Handlebars template evaluated for each materialized graph item. */ GraphTemplateValue: string | number; /** @description Maps each artifact item to graph node identifiers. */ @@ -10193,6 +10357,10 @@ export interface components { }; /** @description Configuration for graph index type */ GraphIndexConfig: { + /** @description Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. */ + metrics?: { + [key: string]: components["schemas"]["GraphMetricConfig"]; + }; /** @description Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires index_capabilities.artifact_sources=true and is rejected by serverless deployments. */ sources?: components["schemas"]["GraphArtifactSourceConfig"][]; /** @description Configuration for generating node summaries (enables tree navigation in Retrieval Agent) */ @@ -10684,6 +10852,9 @@ export interface components { }; /** @description Credential-free normalized graph configuration returned after creation. */ CreatedGraphIndexConfig: { + metrics?: { + [key: string]: components["schemas"]["GraphMetricConfig"]; + }; summarizer?: components["schemas"]["CreatedProviderConfig"]; template?: string; edge_types?: components["schemas"]["EdgeTypeConfig"][]; @@ -11377,6 +11548,114 @@ export interface components { [key: string]: unknown; }; }; + /** @description Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers. */ + GraphMetricRuntimeStats: { + enabled?: boolean; + /** @enum {string} */ + role?: "combined" | "coordinator" | "worker" | "worker_pool"; + /** Format: uint64 */ + runtime_id_hash?: number; + /** Format: uint64 */ + owner_id_hash?: number; + /** Format: uint64 */ + lease_key_hash?: number; + /** Format: uint64 */ + worker_id_hash?: number; + /** Format: uint64 */ + worker_count?: number; + lease_owned?: boolean; + has_lease?: boolean; + /** Format: uint64 */ + acquisition_count?: number; + /** Format: uint64 */ + takeover_count?: number; + /** Format: uint64 */ + lease_acquire_failures?: number; + /** Format: uint64 */ + lost_leases?: number; + /** Format: uint64 */ + last_acquired_ms?: number; + /** + * Format: uint64 + * @description Cached expiry of the currently held maintenance lease, or zero when no lease is held. + */ + lease_expires_at_ms?: number; + /** + * Format: uint64 + * @description Earliest time the runtime will renew its maintenance lease, or zero when no lease is held. + */ + lease_renew_after_ms?: number; + /** + * Format: uint64 + * @description Number of durable maintenance lease renewals completed by this runtime. + */ + renewal_count?: number; + started?: boolean; + shutdown?: boolean; + notified?: boolean; + /** Format: uint64 */ + ticks_started?: number; + /** Format: uint64 */ + ticks_completed?: number; + /** Format: uint64 */ + durable_progress_ticks?: number; + /** Format: uint64 */ + idle_ticks?: number; + /** Format: uint64 */ + error_ticks?: number; + last_error_name?: string; + /** Format: uint64 */ + total_metrics_scanned?: number; + /** Format: uint64 */ + total_active_builds?: number; + /** Format: uint64 */ + total_builds_started?: number; + /** Format: uint64 */ + total_worker_steps?: number; + /** Format: uint64 */ + total_coordinator_steps?: number; + /** + * Format: uint64 + * @description Consumed intermediate records retired at completed reduction barriers. + */ + total_retired_input_records?: number; + /** Format: uint64 */ + total_pages_claimed?: number; + /** Format: uint64 */ + total_pages_completed?: number; + /** Format: uint64 */ + total_phases_advanced?: number; + /** Format: uint64 */ + total_published?: number; + /** Format: uint64 */ + total_failed_builds?: number; + /** Format: uint64 */ + last_metrics_scanned?: number; + /** Format: uint64 */ + last_active_builds?: number; + /** Format: uint64 */ + last_builds_started?: number; + /** Format: uint64 */ + last_worker_steps?: number; + /** Format: uint64 */ + last_coordinator_steps?: number; + /** + * Format: uint64 + * @description Consumed intermediate records retired in the latest maintenance tick. + */ + last_retired_input_records?: number; + /** Format: uint64 */ + last_pages_claimed?: number; + /** Format: uint64 */ + last_pages_completed?: number; + /** Format: uint64 */ + last_phases_advanced?: number; + /** Format: uint64 */ + last_published?: number; + /** Format: uint64 */ + last_failed_builds?: number; + last_budget_exhausted?: boolean; + }; /** @description Statistics for graph index */ GraphIndexStats: { /** @@ -11531,6 +11810,7 @@ export interface components { result_nodes?: number; }; }; + graph_metric_runtime?: components["schemas"]["GraphMetricRuntimeStats"]; }; /** @description Compact public statistics for an algebraic sidecar index. Detailed runtime, adaptive, and materialization records remain internal diagnostics. */ AlgebraicIndexStats: { @@ -11716,6 +11996,154 @@ export interface components { }; /** @description Statistics for an index */ IndexStats: components["schemas"]["FullTextIndexStats"] | components["schemas"]["EmbeddingsIndexStats"] | components["schemas"]["GraphIndexStats"] | components["schemas"]["AlgebraicIndexStats"]; + GraphMetricEdgeFilterStatus: { + /** @enum {string} */ + mode: "all" | "types"; + types?: string[]; + }; + GraphMetricBuildPageStatus: { + phase: string; + /** Format: int64 */ + iteration: number; + /** Format: int64 */ + page_id: number; + /** @enum {string} */ + state: "pending" | "leased" | "complete" | "failed"; + /** @enum {string} */ + range_kind: "full" | "reverse_edges" | "nodes" | "scores" | "contributions" | "job_control" | "summary"; + /** @description Worker id that owns or last failed this page. */ + worker_id?: string; + /** + * Format: int64 + * @description Unix epoch milliseconds when the page lease expires, or 0 when not leased. + */ + lease_expires_at_ms?: number; + /** + * Format: int64 + * @description Current attempt number for this page. + */ + attempt?: number; + /** @description Opaque resumable cursor for this page. */ + cursor?: string; + /** + * Format: int64 + * @description Completed work units for this page. + */ + completed_units?: number; + /** + * Format: int64 + * @description Estimated total work units for this page. + */ + total_units?: number; + /** @description Last page-level error. */ + last_error?: string; + }; + GraphMetricEvent: { + /** Format: int64 */ + sequence: number; + /** @enum {string} */ + kind: "publish" | "delete" | "pause" | "resume" | "failed"; + /** Format: int64 */ + at_ms: number; + /** Format: int64 */ + target_edge_generation: number; + /** Format: int64 */ + published_generation: number; + /** Format: int64 */ + score_count: number; + }; + GraphMetricStatus: { + state: string; + /** @enum {string} */ + phase: "idle" | "computing" | "publishing" | "complete" | "prepare_generation" | "scan_edges_and_out_degree" | "initialize_ranks" | "iterate_contributions" | "reduce_ranks" | "hits_hub_contributions" | "hits_hub_reduce_ranks" | "check_convergence" | "publish_generation" | "cleanup_old_generations"; + edge_filter?: components["schemas"]["GraphMetricEdgeFilterStatus"]; + /** + * Format: int64 + * @description Version of the published graph metric metadata schema. + */ + metadata_version?: number; + /** @description Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every SDK preserves all 64 bits. */ + config_fingerprint?: string; + maintenance_paused?: boolean; + /** @description Whether a local or distributed build is queued after the currently published or building generation. */ + build_queued: boolean; + /** Format: int64 */ + published_generation: number; + /** Format: int64 */ + edge_generation: number; + /** Format: int64 */ + target_edge_generation: number; + /** + * Format: int64 + * @description Pending edge generation waiting to build, or 0 when no build is queued. + */ + queued_generation?: number; + /** + * Format: int64 + * @description Edge generation currently held by an active build lease, or 0 when idle. + */ + building_generation?: number; + /** + * Format: int64 + * @description Durable identifier for the active graph metric build job, or 0 when idle. + */ + build_job_id?: number; + /** + * Format: int64 + * @description Unix epoch milliseconds when the active graph metric build started, or 0 when idle. + */ + build_started_at_ms?: number; + /** + * Format: int64 + * @description Iteration number reported by the active build lease, or 0 when idle or not iterative. + */ + build_iteration?: number; + /** + * Format: int64 + * @description Unix epoch milliseconds when the active build lease expires, or 0 when idle. + */ + build_lease_expires_at_ms?: number; + /** @description Worker id that owns the active build lease. Local builds use `local`. */ + build_worker_id?: string; + /** @description Opaque resumable cursor for the active build phase. Empty or omitted when idle or when the phase has no cursor. */ + build_cursor?: string; + /** + * Format: int64 + * @description Completed work units for the active graph metric build, or 0 when idle or unknown. + */ + build_completed_units?: number; + /** + * Format: int64 + * @description Estimated total work units for the active graph metric build, or 0 when idle or unknown. + */ + build_total_units?: number; + /** @description Active leased or failed build pages for the current build phase, capped and ordered by durable page key. */ + build_pages?: components["schemas"]["GraphMetricBuildPageStatus"][]; + /** @description Whether build_pages was capped before every active page could be included. */ + build_pages_truncated?: boolean; + /** + * Format: int64 + * @description Number of consecutive failed build attempts for the current target generation, or 0 when no failure applies. + */ + retry_count?: number; + /** @description Last build error for the current failed target generation. */ + last_error?: string; + /** + * Format: double + * @description Build progress for the target edge generation, from 0.0 to 1.0 + */ + progress: number; + converged: boolean; + /** Format: int64 */ + iterations_completed: number; + /** Format: double */ + delta: number; + /** Format: int64 */ + computed_at_ms: number; + last_event?: components["schemas"]["GraphMetricEvent"]; + /** @description Recent graph metric events, newest first. */ + recent_events?: components["schemas"]["GraphMetricEvent"][]; + }; /** * @description Available tool names for retrieval agents. * - add_filter: Add search filters (field constraints) @@ -12895,6 +13323,63 @@ export interface components { */ top_n?: number; } & (components["schemas"]["AntflyRerankerConfig"] | components["schemas"]["CohereRerankerConfig"] | components["schemas"]["VertexRerankerConfig"]); + /** @description Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. */ + GraphMetricQuery: { + /** @description Optional result key. Defaults to the metric name. */ + name?: string; + /** @description Graph index that owns the published metric. */ + index: string; + /** @description Graph metric to read. */ + metric: string; + /** + * Format: int32 + * @description Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated metric snapshot. + * @default 10 + */ + top_k?: number; + /** + * @description Whether the latest published generation may be stale or must match the graph edge generation. + * @default published + * @enum {string} + */ + metric_freshness?: "published" | "fresh"; + }; + /** @description Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. */ + GraphMetricRerank: { + /** @description Graph index that owns the published metric. */ + index: string; + /** @description Graph metric name to blend into the search hit score. */ + metric: string; + /** + * Format: int32 + * @description Bounded retrieval window scored by the graph metric before offset and limit are applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. An explicit value must cover offset plus limit. Larger windows improve promotion recall at predictable linear score-read cost. + */ + candidate_count?: number; + /** + * Format: double + * @description Multiplier applied to the existing hit score before adding the graph metric feature. + * @default 1 + */ + base_weight?: number; + /** + * Format: double + * @description Multiplier applied to the graph metric score before it is added to the existing hit score. + * @default 1 + */ + weight?: number; + /** + * Format: double + * @description Metric feature value to use for hits that do not have a score in the published metric generation. + * @default 0 + */ + missing_score?: number; + /** + * @description Whether stale published generations are acceptable or the metric must be fresh. + * @default published + * @enum {string} + */ + metric_freshness?: "published" | "fresh"; + }; /** @description User-visible graph alias or named result under Antfly graph identifier policy v1 (Unicode 15.0.0). Identifiers are exact UTF-8 strings and are not normalized. Ordinary internal ASCII spaces are allowed. The value must not equal `*`, begin with `$`, have leading or trailing spaces, contain non-ASCII Unicode White_Space, or contain Unicode Cc control or Cf format code points. UTF-8 encoding is limited to 512 bytes. */ GraphIdentifier: string; GraphDocumentFuzzyFilter: { @@ -13147,6 +13632,23 @@ export interface components { }; /** @description Select graph nodes using exactly one explicit, exact selector form. */ GraphNodeSelector: components["schemas"]["GraphKeyNodeSelector"] | components["schemas"]["GraphIdentityNodeSelector"] | components["schemas"]["GraphResultRefNodeSelector"]; + GraphMetricOrder: { + metric: string; + /** @enum {string} */ + direction?: "asc" | "desc"; + /** @enum {string} */ + nulls?: "first" | "last" | "nulls_first" | "nulls_last"; + }; + GraphMetricFilter: { + metric: string; + /** + * @description Semantic comparison operator. Named values keep generated SDK enums portable and readable. + * @enum {string} + */ + op: "gt" | "gte" | "lt" | "lte" | "eq" | "neq"; + /** Format: double */ + value: number; + }; /** @description Breadth-first traversal with request-wide deduplication by exact table-qualified node identity. Direction defaults to `out`; use `both` to traverse a relationship as undirected without storing a reciprocal edge. */ GraphTraversal: { start: components["schemas"]["GraphNodeSelector"]; @@ -13171,6 +13673,23 @@ export interface components { include_documents?: boolean; /** @description Requires include_documents=true. Omit to include all document fields. */ fields?: string[]; + /** @description Graph metric names to project onto returned traversal nodes. */ + metrics?: string[]; + /** @description Sort traversal candidates by graph metric score before applying limit. */ + order_by?: components["schemas"]["GraphMetricOrder"][]; + /** @description Filter traversal candidates by graph metric score before applying limit. */ + where_metric?: components["schemas"]["GraphMetricFilter"][]; + /** + * @description Freshness required for projected, ordered, and filtered graph metrics. + * @default published + * @enum {string} + */ + metric_freshness?: "published" | "fresh"; + /** + * @description Include graph metric status metadata in the traversal profile. + * @default false + */ + include_metric_status?: boolean; /** @description Non-scoring structured stored-document predicate for reached nodes. */ filter?: components["schemas"]["GraphDocumentFilter"]; }; @@ -13397,6 +13916,30 @@ export interface components { include_documents?: boolean; include_edges?: boolean; fields?: string[]; + /** @description Graph metric names to project onto legacy graph_searches result nodes. */ + metrics?: string[]; + /** @description Sort legacy graph_searches result nodes by graph metric score. */ + order_by?: components["schemas"]["GraphMetricOrder"][]; + /** @description Filter legacy graph_searches result nodes by graph metric score. */ + where_metric?: components["schemas"]["GraphMetricFilter"][]; + /** + * @description Freshness required for projected, ordered, and filtered graph metrics. + * @enum {string} + */ + metric_freshness?: "published" | "fresh"; + /** @description Include graph metric status metadata in the legacy graph_searches result. */ + include_metric_status?: boolean; + }; + GraphMetricScore: { + node: string; + /** Format: double */ + score: number; + }; + GraphMetricResult: { + index_name: string; + metric: string; + scores: components["schemas"]["GraphMetricScore"][]; + status: components["schemas"]["GraphMetricStatus"]; }; /** @description One exact node identity projected from a MATCH binding. Conjunctive bindings deliberately do not expose traversal depth, distance, or path: those values are not uniquely defined for branched patterns and may depend on execution order. */ GraphBindingNode: { @@ -13501,6 +14044,10 @@ export interface components { path_edges?: components["schemas"]["GraphPathEdge"][]; /** @description Algebraic provenance labels folded into this result, when requested by an algebraic graph executor */ provenance?: string[]; + /** @description Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score for the node. */ + metrics?: { + [key: string]: unknown; + }; /** @description Parsed evidence envelope for provenance labels and edge metadata */ evidence?: { [key: string]: unknown; @@ -13515,6 +14062,10 @@ export interface components { kind: "nodes"; /** @description Traversal result nodes; requested paths are stored on each node. */ nodes: components["schemas"]["GraphResultNode"][]; + /** @description Graph metric status metadata keyed by metric name when requested. */ + metric_status?: { + [key: string]: components["schemas"]["GraphMetricStatus"]; + }; stats: components["schemas"]["GraphResultStats"]; }; /** @description An ordered canonical graph path with table-qualified node identities and a self-describing ranking score. */ @@ -13635,6 +14186,10 @@ export interface components { * @description Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field. */ took?: number; + /** @description Graph metric status metadata keyed by metric name. */ + metric_status?: { + [key: string]: components["schemas"]["GraphMetricStatus"]; + }; }; /** @description Graph result emitted by the stateful compatibility transport. Canonical graph_queries produce GraphResult; deprecated graph_searches may produce LegacyGraphSearchResult during the compatibility window. */ StatefulGraphResult: components["schemas"]["GraphResult"] | components["schemas"]["LegacyGraphSearchResult"]; @@ -16104,6 +16659,65 @@ export interface operations { }; }; }; + executeGraphMetricAction: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Name of the table */ + tableName: string; + /** @description Name of the graph index */ + indexName: string; + /** @description Name of the configured graph metric */ + metricName: string; + /** @description Operational action to apply to the graph metric materialization */ + action: "refresh" | "rebuild" | "delete" | "pause" | "resume"; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Aggregate graph metric status after the action is durably accepted */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GraphMetricActionResponse"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 404: components["responses"]["NotFound"]; + /** @description Graph metric actions are unavailable for this runtime */ + 405: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + /** @description The table topology or write-owner generation changed, or only some shards accepted the action; retrying is consistency-safe and reuses any still-active build */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Local storage resources are temporarily exhausted */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + 500: components["responses"]["InternalServerError"]; + }; + }; commitTransaction: { parameters: { query?: never; diff --git a/ts/packages/sdk/src/types.ts b/ts/packages/sdk/src/types.ts index d58a24ebb9..75a963c71c 100644 --- a/ts/packages/sdk/src/types.ts +++ b/ts/packages/sdk/src/types.ts @@ -198,6 +198,25 @@ export type ClusterStatus = components["schemas"]["ClusterStatus"]; // Graph index types export type GraphIndexConfig = components["schemas"]["GraphIndexConfig"]; +export type GraphIndexStats = components["schemas"]["GraphIndexStats"]; +export type GraphMetricActionResponse = components["schemas"]["GraphMetricActionResponse"]; +export type GraphMetricBuildPageStatus = components["schemas"]["GraphMetricBuildPageStatus"]; +export type GraphMetricEdgeFilterStatus = components["schemas"]["GraphMetricEdgeFilterStatus"]; +export type GraphMetricEvent = components["schemas"]["GraphMetricEvent"]; +export type GraphMetricFilter = components["schemas"]["GraphMetricFilter"]; +export type GraphMetricOrder = components["schemas"]["GraphMetricOrder"]; +export type GraphMetricProfile = components["schemas"]["GraphMetricProfile"]; +export type GraphMetricQuery = components["schemas"]["GraphMetricQuery"]; +export type GraphMetricRerank = components["schemas"]["GraphMetricRerank"]; +export type GraphMetricRerankScoreDetails = components["schemas"]["GraphMetricRerankScoreDetails"]; +export type GraphMetricResult = components["schemas"]["GraphMetricResult"]; +export type GraphMetricRuntimeStats = components["schemas"]["GraphMetricRuntimeStats"]; +export type GraphMetricScore = components["schemas"]["GraphMetricScore"]; +export type GraphMetricStatus = components["schemas"]["GraphMetricStatus"]; +export type GraphMetricFreshness = NonNullable< + components["schemas"]["GraphTraversal"]["metric_freshness"] +>; +export type QueryScoreDetails = components["schemas"]["QueryScoreDetails"]; export type EdgeTypeConfig = components["schemas"]["EdgeTypeConfig"]; export type EdgeTopology = NonNullable; diff --git a/ts/packages/sdk/test/types.test.ts b/ts/packages/sdk/test/types.test.ts index e548dc4362..57a4fff846 100644 --- a/ts/packages/sdk/test/types.test.ts +++ b/ts/packages/sdk/test/types.test.ts @@ -21,7 +21,11 @@ import type { GraphAggregatesResult, GraphBindingsResult, GraphDocumentFilter, + GraphIndexStats, GraphMatchQuery, + GraphMetricBuildPageStatus, + GraphMetricQuery, + GraphMetricRuntimeStats, GraphNodesResult, IndexRuntimeCapabilities, LegacyGraphSearchResult, @@ -52,6 +56,18 @@ function generatedSortProfileDeclaration(): string { } describe("Antfly Query Type Integration", () => { + it("accepts summary graph metric build pages", () => { + const page: GraphMetricBuildPageStatus = { + phase: "reduce_ranks", + iteration: 2, + page_id: 0, + state: "complete", + range_kind: "summary", + }; + + expect(page.range_kind).toBe("summary"); + }); + describe("cluster status capabilities", () => { it("exports the typed artifact-source capability contract", () => { const capabilities: IndexRuntimeCapabilities = { @@ -223,6 +239,23 @@ describe("Antfly Query Type Integration", () => { }); describe("QueryRequest type safety", () => { + it("should expose direct graph metric reads", () => { + const graphMetric: GraphMetricQuery = { + name: "central", + index: "graph_idx", + metric: "pagerank", + top_k: 25, + metric_freshness: "fresh", + }; + const query: QueryRequest = { + table: "docs", + graph_metric: graphMetric, + }; + + expect(query.graph_metric?.name).toBe("central"); + expectTypeOf(query.graph_metric).toMatchTypeOf(); + }); + it("keeps graph filters in the stored-document predicate subset", () => { const filter: GraphDocumentFilter = { term: "active", path: "/status" }; const numeric: GraphDocumentFilter = { @@ -514,6 +547,31 @@ describe("Antfly Query Type Integration", () => { }); }); + describe("Graph index stats", () => { + it("should expose graph metric runtime telemetry", () => { + const runtime: GraphMetricRuntimeStats = { + enabled: true, + role: "worker_pool", + owner_id_hash: 17, + worker_count: 3, + takeover_count: 2, + lost_leases: 1, + total_pages_claimed: 6, + last_pages_completed: 3, + last_budget_exhausted: true, + }; + const stats: GraphIndexStats = { + index_type: "graph", + total_edges: 4, + graph_metric_runtime: runtime, + }; + + expect(stats.graph_metric_runtime?.role).toBe("worker_pool"); + expect(stats.graph_metric_runtime?.owner_id_hash).toBe(17); + expectTypeOf(stats.graph_metric_runtime).toMatchTypeOf(); + }); + }); + describe("Bool Field Query", () => { it("should create valid bool field query", () => { const query: BoolFieldQuery = { diff --git a/zig/GRAPH.md b/zig/GRAPH.md index e86667062c..f9470e2c38 100644 --- a/zig/GRAPH.md +++ b/zig/GRAPH.md @@ -736,3 +736,96 @@ Query tests: - External node ids can be returned without document hydration. - Hydration-required query over external nodes fails closed. - Entity/global projection query shapes are rejected in V1. + +## Immutable graph-metric execution + +Document and external-source serverless publications use the same request-wide +plan in `serverless/build/lake_graph_metric.zig`. Reusable metrics are resolved +first. Dirty requests are grouped by authenticated source identity, equivalent +edge filter, and exact metric computation parameters. Names and refresh policies +are not computation identity. Each source is fetched/prepared once. Compatible +filters share a union topology when the whole group fits its work and memory +budgets; otherwise the planner processes cheaper exact topology requirements +first. An unaffordable spectral sibling must not force an affordable degree +metric to build its adjacency lanes or inherit its rejection. Each unique metric +is computed, encoded, and uploaded once. Compatible HITS authority/hub metrics +share their kernel. + +Graph artifact wire v3 stores sorted node/type dictionaries and fixed-width +ordinal edge records (including weights and qualified-table ordinals). Metric +preparation reads validated borrowed views directly into compact outbound +topology, without per-edge string allocation, node hashing, or allocating then +discarding inbound edges. The graph-query reader reuses the same validated view +for memory admission and owned adjacency decoding. Encoders build dictionaries +once, check output limits before allocation, and observe cancellation. The +current score artifact remains v9: its prefix-compressed point/ranked blocks +remain independently readable without fetching another node dictionary. + +The plan retains only one source and one filtered projection at a time; alias +fanout retains lightweight references, not score vectors or encoded payloads. +References preserve request order and independently carry index names and +publication, topology, and computation provenance. Equivalent PageRank aliases +use the first available prior artifact in request order as their optional seed; +authentication or compatibility failure still cold-starts the shared computation. +Aggregate budgets count actual unique work, source reads, and output uploads. + +Both publication paths resolve the complete requested plan against a shared +inventory of prior computations. Equivalent new or renamed aliases reuse a ready +payload without source reads, kernel work, encoding, or uploads. Each immutable +prior payload is authenticated and its header read at most once per publication, +including failed verification. New aliases retain the original computation time +while carrying their own current publication and topology provenance. +Lake and sidecar manifest validation permits shared IDs for distinct graph/metric +names only when every immutable metadata field agrees; conflicting duplicate +declarations remain invalid. + +The preceding manifest's ordered metric references are the admission-plan +witness. Rejected computations remain reusable only while the complete plan, +source identities, and materializer policy are unchanged. Removing or changing a +budget-consuming sibling therefore retries previously rejected work; an unchanged +plan does not cause a retry loop. Missing or invalid prior payloads are rebuilt +with fresh publication/computation provenance. + +The storage-independent PageRank, eigenvector, and HITS kernels partition the +CSR vertex/edge work stream into fixed logical tiles, including boundaries inside +high-degree vertices. Complete rows remain target-owned. Only tile-boundary rows +need partial sums (at most 32 stack records), reduced in a fixed order independent +of the `std.Io` worker count. Large graphs have at most +`ceil((nodes + edges) / 16)` work units per logical tile; no extra edge-sized +scratch allocation or atomic floating-point updates are required. + +Graph-metric queries authenticate control, routing, primary-score, and ranked +blocks before publishing them to the bounded shared memory cache. Disk retention +is optional and asynchronous: one cache-owned `std.Io` worker drains at most +32 outstanding jobs / 16 MiB, independent of request allocator, executor, and +cancellation lifetimes. Queue pressure or disk failure does not fail a verified +read or make shared waiters download it again. Shutdown cancels pending retention +and joins the worker before destroying the cache. Pending bytes/jobs, failures, +and bypasses are exposed in `QueryCacheStats`; maintenance can explicitly drain +retention, but queries never wait for it. Local-cache read errors fall back to +authenticated origin reads; origin integrity failures remain fatal. + +Column queries resolve immutable physical computations before admission and +range planning. Equivalent aliases share routing, transport, and decode work, +while every logical output is admitted up front and owns its result array and +publication provenance. Conflicting immutable metadata cannot reuse another +column's validation. + +Non-serverless single- and multi-column reads use the same snapshot-local +physical-key reader in `graph/score_read.zig`. Status policies are checked before +score allocation. Only identical encoded metric/generation prefixes are aliases; +equal configurations with different durable publications remain independent. +Rows and physical columns are sorted independently, duplicate keys are read +once, and all logical results preserve input order and independent ownership. +One reusable key slab and result-vector pair serve batches of at most 4096 +storage keys, avoiding per-score prefix formatting and per-batch arena churn. +The existing durable ordinal/vector-chunk jobs and shared numerical kernels +remain the non-serverless computation path. + +Materializer epoch 13 invalidates earlier admission and preparation policies, +including rejections retained before adaptive topology grouping. Serverless is +unreleased and supports only the current artifact contract: old graph wire +versions are rejected, not migrated or silently decoded. + +See [preparation and score-reader benchmarks](bench/graph/METRIC_PREPARATION.md) +for reproducible phase-specific measurements and their limitations. diff --git a/zig/LAKES.md b/zig/LAKES.md index 72c1e7b18e..e1656a8045 100644 --- a/zig/LAKES.md +++ b/zig/LAKES.md @@ -111,6 +111,24 @@ The mapping is direct: artifact before manifest publication. - Antfly text, vector, sparse, graph, and algebraic accelerators map to serverless artifacts. +- Graph metrics map to immutable `graph_metric_segment` artifacts derived from + one named graph artifact. Logical graph/metric names remain in manifest + bindings, while each content-addressed payload records the source graph's + identity, configuration fingerprint, edge filter, convergence metadata, and + prefix-compressed node-sorted/ranked score blocks. Equivalent aliases reuse + the same payload. Query sessions reject a metric when + that source identity does not match the graph artifact pinned by the same + manifest, so published scores cannot be mixed across generations. Manifest + v17 is the first supported graph-metric publication format; v12 manifests + remain readable and graph-metric-free during rollout. +- Metric materialization uses storage-independent, cancellation-aware bounded + kernels for degree, PageRank, eigenvector centrality, and both HITS vectors. + The build call itself is synchronous and deterministic; deployments schedule + it through the existing `std.Io`/backend build runtime rather than creating + raw threads. Public serverless queries support direct top-K metric reads, + graph-metric reranking with score details, and traversal projection/filter/ + ordering with the same bounded full-candidate-before-limit semantics as the + embedded graph engine. - Query execution pins one manifest version before reading artifacts, matching the lake requirement that each query bind one immutable snapshot. - Artifact range reads and cache blocks map to Parquet footer, column-chunk, diff --git a/zig/SERVERLESS.md b/zig/SERVERLESS.md index ee8f486045..a8fef7bce7 100644 --- a/zig/SERVERLESS.md +++ b/zig/SERVERLESS.md @@ -667,6 +667,43 @@ The listener is configured with: - `ANTFLY_SERVERLESS_BIND_PORT` - `ANTFLY_SERVERLESS_TICK_INTERVAL_MS` +### Current-version-only storage contract + +Serverless is unreleased and supports only the current version. All readers and +writers use manifest version 20; graph metric segments use version 10 and graph +topology segments use version 5. Older +manifests are rejected, and no legacy writer or two-phase rollout gate is kept. +Graph metric materialization is enabled by default. Catalog reconciliation +detects configured metrics without artifacts and schedules their publication. +The public graph-index create/read contract exposes a typed `metrics` map; +Go, TypeScript, Python and Zig generated models carry the same configuration. +Metric objects and edge filters use closed field validation, while metric names +remain user-defined. Serverless accepts `background`, not `manual`, refresh. +Any explicit `ANTFLY_SERVERLESS_MANIFEST_WRITE_VERSION` must be `20`; other values +fail startup. All components must run the same current release, and old +development data must be rebuilt before use. + +Graph topology wire v5 retains adjacency traversal data and adds compact +per-type local edge runs (two u32 node ordinals per edge), an authenticated type +directory, and page offsets into the original node dictionary. Metric publication +reads the union of requested type runs and touched dictionary pages. Nearby pages +are coalesced into bounded range reads; dense selections use an ordinal map, +while sparse selections sort only their endpoints. Whole-source node/edge limits +still apply, but retained preparation memory scales with selected topology. +Cold exact-content verification may read the whole object and is byte-accounted; +warm verification identities permit true range-only preparation. + +Semantic fingerprints are produced during encoding with a bounded node-hash +cache, without allocating a graph-wide adjacency view or digest array. The +directory remains available for million-node graphs with ordinary type counts. +Its 1 MiB control-size limit can still explicitly omit indexing for unusually +large type dictionaries; those current-version artifacts use full preparation. + +The manifest authenticates the +point-lookup index and the bounded ranked routing root independently. Cold +top-K reads fetch at most 1,832 routing bytes, regardless of vector cardinality; +point reads authenticate both tiers in one routing fetch. + ## Image And CI Path The Zig runtime image is owned by `antfly-zig`, not by the Go control plane diff --git a/zig/bench/graph/METRIC_PREPARATION.md b/zig/bench/graph/METRIC_PREPARATION.md new file mode 100644 index 0000000000..e4826239d4 --- /dev/null +++ b/zig/bench/graph/METRIC_PREPARATION.md @@ -0,0 +1,749 @@ +# Graph metric execution and query benchmarks + +## Addressed adjacency and existence-only mutation probes (2026-09-10) + +Current graph wire is v7, manifest wire v22, materializer epoch 23. Serverless +supports this current layout only. Node-ordinal row offsets cost eight bytes per +dictionary node; dictionary fences add 68 bytes per 256-node page. Both are +authenticated by the manifest-bound control structure. The control directory +remains capped at 1 MiB; oversized controls explicitly omit the accelerator. + +Reproduce with: + +```sh +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --paged-only +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --presence-only +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --indexing-only +``` + +Apple M4 Max / Zig 0.16.0; medians of five measurements after one discarded +warmup. These are local microbenchmarks, not cloud end-to-end latency claims. + +| Cold one-edge lookup | Whole graph | Addressed row | +| --- | ---: | ---: | +| 16,384 nodes: fetched bytes | 1,627,870 | 333,504 | +| 16,384 nodes: local median | 1.341 ms | 0.160 ms | +| 100,000 nodes: fetched bytes | 9,934,770 | 296,884 | +| 100,000 nodes: local median | 10.893 ms | 0.177 ms | +| GETs per fixture | 1 | 6 | + +Every sample creates a fresh reader. The transport serves immutable memory and +counts exact GET bytes. Both paths assert the same neighbor. The full-decode +reference omits transport SHA verification, while the addressed path verifies +the footer, directory, and touched blocks. Network RTT can dominate six small +GETs; measure actual object-store latency before interpreting these CPU timings +as deployment speedups. Authenticated first-key fence prefixes avoid an object +GET for every dictionary binary-search comparison; common prefixes longer than +64 bytes fall back to a bounded search of the ambiguous pages. + +The many-type preparation fixture has two nodes and one relationship per type. +64/1,024/10,000 types require 3/3/6 range reads and 7,580/117,020/812,860 bytes. +Before boundary-block reuse, the 10,000-type probe exhausted its 512 MiB read +allowance after 8,184 calls for a roughly 1.14 MiB source. The regression now +requires completion within 2 MiB, at most eight calls, and no source-size read +amplification. Block retention is one 64 KiB block, or less for small sources. +Point and traversal readers instead retain up to eight blocks (512 KiB) per +source under the request's admitted allocator; a warm-hop regression verifies +that dictionary, routing, and row reads share that working set without extra GETs. + +| 1,024 existing-key presence probes | Scalar value reads | Sorted existence reads | +| --- | ---: | ---: | +| 256-byte values: local median | 5.771 ms | 0.526 ms | +| 256-byte values: extra peak allocation | 298,778 B | 160 B | +| 16-KiB values: local median | 1.287 ms | 0.693 ms | +| 16-KiB values: extra peak allocation | 16,810,398 B | 20,360 B | +| Retained value copies | 1,024 | 0 | + +This isolates presence checks on warm immutable LSM runs and block cache using +modeled storage. It measures batch lifetime, excluding fixture creation and +disk latency. Different value sizes create different run/block layouts, so +cross-row timing comparisons are not a payload-size scaling curve. The native +batch probes sorted keys in 256-key pages, releases temporary pins and decode +scratch per page, and returns only booleans. Tombstones and batch-local writes +override persisted data. Non-LSM backends use their existing get semantics. + +The separate default durable-LSM benchmark preserves identical topology over +65,536-edge insert/delete cycles: scalar presence plus per-edge global counters +takes 2.091 s, versus 1.277 s for sorted presence plus coalesced counters. That +comparison includes WAL and both directional commits and combines the two +optimizations; it is not an isolated estimate of the presence-check gain. + +## Block-authenticated preparation and committed counters (2026-09-09) + +Run `zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --indexing-only`. +At that measurement, graph wire was v6, manifest wire v21. Both ingestion paths emitted the same +authenticated block table, and the published manifest binds its control root. +Selected preparation retains authenticated semantic digests instead of hashing +the selected graph again. Each cold sample uses a fresh verifier, not a cold OS +page cache. Local filesystem timings are not cloud request-latency measurements. + +Apple M4 Max / Zig 0.16.0 ReleaseFast, shared host, after merging main +`9f192f9be`; six samples with the first discarded. Each reference prepares the same current-wire artifact and checks +the same selected semantic identity. Preparation excludes the numerical kernel. + +| Phase | Reference | Current | Tracked peak / reads | +| --- | ---: | ---: | --- | +| Stateful committed insert + delete | 1,719.468 ms | 980.983 ms | Endpoint counter reads 262,144 → 2,048 | +| Serverless narrow, warm identity | 12.781 ms | 0.107 ms | Peak 16,140,777 → 99,141 B; reads 11,780,488 → 115,080 B | +| Serverless narrow, cold identity | 12.759 ms | 0.105 ms | Same peak and reads as warm | +| Serverless all types, warm identity | 12.437 ms | 2.710 ms | Peak 16,140,777 → 5,854,947 B; reads 11,780,488 → 3,434,931 B | +| Serverless all types, cold identity | 12.625 ms | 2.480 ms | Same peak and reads as warm | + +Two pre-merge runs had identical counted bytes and allocation peaks. Committed +cycles measured 2,150.950 → 1,198.386 ms and 2,158.953 → 1,185.736 ms; cold narrow +preparation measured 14.511 → 0.115 ms and 14.483 → 0.133 ms. All three complete +runs checked semantic/encoded parity across all 24 benchmark records. + +Compared with the historical v5 run below, warm narrow reads rise from 19,677 +to 115,080 bytes because of block alignment; cold narrow reads fall from +11,794,405 to 115,080 bytes. The source grows by 5,760 bytes for its block table. +Cross-run timing comparisons are approximate on this shared host; within-run +reference parity and counted bytes are the stronger evidence. + +The committed-counter fixture uses 65,536 edges and 1,024 nodes on the default +durable LSM. Both paths execute six complete insert/delete cycles, discarding the +first, and check edge/node counts after both commits. Only global incidence +maintenance differs; topology invalidation, directional writes and WAL/commit +remain in the timer. Coalescing reduces endpoint counter reads per cycle from +262,144 to 2,048. This does not imply fewer WAL records: repeated mutable-key +updates were already coalesced by the LSM. Forced compaction, reopening and +fixture construction are excluded. + +The block table costs 32 bytes per 64 KiB covered, at most 128 KiB for a 256 MiB +artifact, plus 32 bytes in each graph manifest reference. The existing eight-byte +ordinal edge index and dictionary/type directory are still present. Directory +control remains capped at 1 MiB, with an explicit full-preparation fallback. +Aligned reads can overfetch compared with v5's warm range path, but no longer +require cold full-object authentication. Actual overfetched bytes count against +publication's shared read allowance; separate filter groups share one control +object and retain only their own selected topology. + +## Earlier v5 addressed plans and topology preparation (2026-09-09) + +Run `zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --indexing-only`. +Apple M4 Max / Zig 0.16.0, shared host; six samples, first discarded. This run +includes the merge of main `aa44bddd1`. That run used wire v5. The following are +local phase measurements, not cloud/HTTP latency. + +| Phase | Reference | Addressed | Tracked peak / reads | +| --- | ---: | ---: | --- | +| Stateful plan validation | 129.812 µs | 9.546 µs | Fixed 76-byte control; no boundary allocations on control path | +| Serverless narrow filter, warm identity | 12.296 ms | 0.074 ms | Peak 16,135,017 → 64,884 B; reads 11,774,728 → 19,677 B | +| Serverless narrow filter, cold identity | 12.538 ms | 4.107 ms | Same phase peak; reads 11,774,728 → 11,794,405 B | +| Serverless all types, warm identity | 12.509 ms | 7.490 ms | Peak 16,135,017 → 5,849,203 B; reads 11,774,728 → 3,181,277 B | +| Serverless all types, cold identity | 12.775 ms | 11.519 ms | Same phase peak; reads 11,774,728 → 14,956,005 B | + +The stateful reference reads and validates the addressed boundary set as well +as the header; it models the old dependency on all boundary data, not the exact +old monolithic encoding. Both return the same plan identity. Each sample averages +128 read transactions on the default durable LSM. Production lease checks, +topology identity and iterative planning use only the control; initial planning +still materializes boundaries once. The generation-fenced census seals its +existing boundary slots rather than writing a second large final blob. + +Serverless fixtures contain 16,384 nodes, 262,144 unrelated edges and 256 selected +edges. Both paths prepare the same authenticated v5 artifact and calculate the +same selected semantic identity; timers include that hashing but no numerical +kernel. The narrow path retains 256 node IDs instead of 16,384. All-types reads +use the same ordinal type runs, a dense endpoint map and coalesced dictionary +pages. Sparse selection instead sorts only its selected endpoints. + +Warm cases verify source identity before timing; cold cases create a new local +verifier per sample, not a cold OS page cache. Cold authentication bytes remain +charged. Narrow reads still avoid full decode and retained topology, but cold +all-types reads increase I/O and show only a modest timing improvement. These +local measurements do not establish a cold cloud latency win. Provider/cache +allocations are outside the phase allocator. v5 adds eight bytes per local edge +plus a page/type directory to the existing traversal body (about 2.10 MB for +this fixture). This is an explicit secondary-index storage tradeoff, not free +compression. Page reads merge nearby ranges into at most 1 MiB windows, except +that one oversized dictionary page can be admitted on its own. + +Ingestion parity remains exact between reference and ordinal encoders. For +65,536 edges, ordinal JSON-to-artifact medians were 16.791 ms with 16-byte IDs +and 28.507 ms with 256-byte IDs (reference: 21.682 and 61.939 ms). The streaming directory builder uses compact +node offsets and a bounded 65,536-entry digest cache, not an adjacency view or +graph-wide digest array. A million-node graph no longer loses its directory +because of the former 64 MiB scratch estimate. + +The unchanged-topology republish measured 34 µs versus 2.871 ms recomputation, +with exact score/artifact identity checks. Stateful coalesced membership +maintenance measured 114.003 ms versus 528.457 ms per-edge maintenance; this +remains an abort-based maintenance benchmark, excluding WAL/commit. These +results do not establish the benefit of migrating stateful graph strings to +persistent numeric IDs: that separate indexing decision needs committed-write, +compaction and traversal measurements including dictionary maintenance. + +The same run measured stateful filtered discovery at 0.573 ms using type +postings versus 12.636 ms scanning all 65,536 edges (4,096 selected), with +identical selected-edge checksums. Cold selected census planning took 66.969 ms +and three durable checkpoints versus 212.916 ms and seventeen checkpoints for +the global census. Census timings include plan reset and committed checkpoints, +but exclude fixture writes and the numerical kernel. + +## Earlier v4 transactional indexing and directory-first reuse (2026-09-09) + +Run `zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --indexing-only`. +Apple M4 Max / Zig 0.16.0, shared development host, six samples with the first +discarded. These are phase measurements, not HTTP or cloud latency guarantees. + +| Phase | Reference | Current | Checked work / memory | +| --- | ---: | ---: | --- | +| Stateful membership maintenance, 65,536 edges / 16 types | 547.572 ms | 118.944 ms | 131,072 → 16,384 endpoint reads | +| Weight-only PageRank republish, 65,536 edges | 4.071 ms | 0.035 ms | Exact scores and prior artifact ID; 2,755,143 → 1,270 tracked peak bytes | +| JSON → graph artifact, 16-byte IDs | 27.550 ms | 16.489 ms | Exact current-wire SHA-256; 11,367,906 → 3,723,374 peak bytes | +| JSON → graph artifact, 256-byte IDs | 71.197 ms | 30.294 ms | Exact current-wire SHA-256; 43,562,466 → 4,214,894 peak bytes | + +Membership maintenance uses the default durable LSM and aborts each all-edge +removal to restore the identical fixture. Timing includes posting maintenance, +but excludes fixture construction and WAL/commit. The reference is immediate +per-edge incidence maintenance; the current path coalesces endpoint deltas, +bulk-reads counts, caches encoded type prefixes, and uses already-known edge +existence only when the covering index is ready in the same transaction. +Scratch allocations fall 589,824 → 147,554; scratch peak rises 361 → 1,870,804 +bytes. Backend-owned allocations are not included in that scratch measurement. +This is not an 8× reduction in WAL records: the LSM already coalesces repeated +updates to the same mutable key before commit. + +Type postings and endpoint memberships are now demand-driven. Ordinary graphs +and unfiltered metrics do not create them. The first filtered plan activates a +bounded backfill; a durable activation marker keeps concurrent ingestion covered +across pauses, filter removal and reopen. Existing partial indexes are detected +before an inactive marker is written. A regression checks zero auxiliary posting +and membership records before demand, then exact edge/node parity after activation, +interleaved mutations and reopen. The timed membership fixture explicitly enables +a filtered metric; these timings measure active-index maintenance. + +Private graph stores now serialize complete write transactions, including their +initial reads, across foreground ingestion, backfill, leases and checkpoint CAS. +Snapshots and numerical work remain outside that gate. Concurrent tests cover +memory, in-memory LSM and durable LSM; native LMDB retains its native single +writer. Commit failures retain the gate until abort, and failed opens release it. + +Census progress is a checksummed `GPC2` control record plus generation-bound, +addressed boundary slots. A resume reads and writes no previous boundary bytes; +only the new page's boundaries are persisted. Slots are reused after a generation +restart and reclaimed on completion or filter removal. The long-key regression +uses 256 boundaries of 128 KiB: control size falls from 33,686,596 to 131,140 +bytes, with zero saved-boundary reads/writes during an ordinary resume. Final +plan assembly still materializes the bounded boundary set once, outside the +write gate. Ordinary source/node scans also stop at their byte allowance. + +Graph wire v4 adds a bounded semantic type directory and an authenticated-range +trailer. Both ingestion encoders produce identical bytes. Publication checks +this directory before source-wide preparation; unchanged selected connectivity +can reuse an authenticated metric even when weights, unrelated types, isolated +documents or qualified endpoints change. Source cardinality limits, policy +identity, cancellation and shared read/hash budgets still apply. Directory size +is capped at 1 MiB and construction scratch at 64 MiB; an explicitly unavailable +accelerator uses the same current wire, not a legacy decoder. Large dictionaries +and incomplete local topology use the normal preparation path. + +The republish fixture uses a warm, content-verified local artifact store. Its +timer includes range reads, identity selection, prior control authentication and +publication; graph construction and score checking are outside it. Cold stores +may need full-content authentication, charged to the shared reuse-read budget. +Artifact-store-owned cache memory is outside the allocation tracker. First builds +and changed selected topology still prepare the source graph; the 116× ratio is +for this unchanged-connectivity republish, not all materializations. The ingestion +rows include constructing the new directory, making its extra encoding cost +visible rather than excluding it from the benchmark. + +## Earlier benchmark series + +Measured 2026-09-07 on Apple M4 Max, 36 GiB RAM, macOS 26.3.1, +Zig 0.16.0, ReleaseFast, using the system SMP allocator. One warmup and five +measured samples per case; tables report medians. This was a shared development +host, not an isolated benchmark machine. +The compact-query comparison below uses 21 measured samples instead of five. + +## Ordinal-only numerical cursors + +Measured 2026-09-08 on the same host/toolchain with: + +```sh +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -- --ordinal-cursors-only +``` + +Each fixture is a 256-node cycle in the default durable storage backend. Both +paths read the same sealed initialization and verify identical ordinal checksums. +The reference decodes membership IDs and validates their dictionary mappings; +the numerical path reads checksummed coverage and sealed leaf bounds, then +enumerates dense slots. Initialization and final publication retain dictionary +validation. Six samples per case, first discarded, 64 repetitions per sample: + +| Node ID bytes | String/dictionary traversal | Ordinal traversal | Improvement | +| --- | ---: | ---: | ---: | +| 16 | 211.796 µs | 101.843 µs | 2.08× | +| 4,096 | 1,180.156 µs | 207.796 µs | 5.68× | + +These are cursor traversal medians, including read transactions, control-record +validation and allocation. They exclude graph setup, numerical kernels, writes +and publication; they are not whole-build speedups. The 4 KiB IDs exercise a +long-ID workload rather than representing typical IDs. Other development work +was running on this shared host. + +## Durable cross-job topology reuse + +After increasing default scheduling spans to 4,096 records, the same fixture +and command measured the following on 2026-09-08, before the ordinal-only cursor +change above: + +| Complete numerical job | Physical edge records read | Checkpoints | Median time (range) | +| --- | ---: | ---: | ---: | +| Independent topology | 32,768 | 36 | 1.612 s (1.415–1.887 s) | +| Shared topology | 0 | 22 | 0.834 s (0.811–0.892 s) | + +The shared case in that measurement is 1.93× faster. Against the earlier +small-page run below, independent/shared checkpoint counts fall 638 → 36 and +110 → 22. Those work counts are directly checked. The historical wall times +were not collected in a controlled same-run comparison: storage compaction and +other activity on this shared host can materially change elapsed time. This +benchmark calls the low-level numerical runner; it does not measure concurrent +task admission or HTTP latency. + +### Historical small-page baseline + +Measured 2026-09-08 with: + +```sh +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -- --topology-only +``` + +The real default-storage fixture has 1,024 nodes and 16,384 directed edges, +with 16 neighbors per node. A first PageRank job prepares topology; a differently +configured PageRank job then executes one numerical iteration. The independent +reference forces a reuse-directory miss; the shared case adopts the sealed +owner. Both use the same implementation, numerical seed, and graph, and verify +every published score equals `1/1024`. Six samples per case, first discarded: + +| Complete numerical job | Physical edge records read | Worker/coordinator checkpoints | Median time | +| --- | ---: | ---: | ---: | +| Independent topology | 32,768 | 638 | 79.346 s | +| Shared topology | 0 | 110 | 18.573 s | + +This is a 4.27× median improvement on this fixture, with 82.8% fewer checkpoints. +Times include planning, initialization, numerical reduction, publication and +job cleanup. They exclude fixture writes, score verification, and maintenance +of retired score generations and topology owners between samples. The reference +also includes the constant-size transaction forcing a directory miss. Sample +ranges were 58.172–105.075 s and 17.436–19.115 s. Compilers were running on the +shared development host; a profile of an independent build showed substantial +native LSM compaction work. These times are not an isolated-host throughput +claim, and longer numerical runs amortize preparation over more iterations. + +Lifecycle tests additionally cover HITS-to-PageRank/eigenvector adoption, +producer cleanup and reopen, filter-set canonicalization, independent concurrent +producers, generation pins, deleted filters/metrics, bounded crash-resumable +reclamation, and rejection of retirement tasks targeting winning packed tiles. + +## Adaptive decoded-score joins + +Measured 2026-09-08 with +`zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -- --score-join-only`. +The fixture borrows one decoded 1,024-row block and verifies exact results for +each candidate count. Six samples, first discarded, 4,096 repetitions per sample: + +| Candidate rows | Binary reference | Adaptive join | Improvement | +| --- | ---: | ---: | ---: | +| 1 | 75 ns | 75 ns | unchanged | +| 16 | 431 ns | 434 ns | within 1% | +| 256 | 16.425 µs | 10.959 µs | 1.50× | +| 1,024 | 84.884 µs | 21.394 µs | 3.97× | + +Sparse requests retain binary search; dense candidates merge against sorted +scores. These timings exclude decoding, allocation, authentication and I/O. +Separate parity tests include duplicate, missing and invalid row ordinals. + +## Shared admission, sparse work, and publication checkpoints + +Measured 2026-09-08 on the same host/toolchain with +`zig build -Doptimize=ReleaseFast graph-metric-preparation-bench --summary all`. +One warmup and five measured samples; medians below. Development tests were +running on this shared host. These are bounded phase measurements, not promises +about whole-build, HTTP, or cloud-network latency. + +| Fixture | Reference | Current | Durable work / admission | +| --- | ---: | ---: | --- | +| Publish 8,192 scores, real default storage | 423.880 ms | 96.506 ms | 128 → 2 score/staging/cursor commits | +| 100 sequential 80-entry routing working sets | 0.968 ms | 0.226 ms | 8,000 → 80 cache fills | + +Publication compares 64-node and 4,096-node batches using the same current +producer helper. IDs are short, so the 1 MiB node-ID budget does not truncate +either case. Each sample opens fresh storage. Timing includes ordered staging, +primary scores, checkpoint cursor commits, and full primary-score verification; +it excludes numerical computation, worker page fencing, and final top-K merging. +Ranges were 419.238–430.928 ms and 95.571–97.751 ms. Actual production batches +also stop at their partition boundary or byte allowance; the fixture is not an +end-to-end 64× speedup claim. + +The routing fixture uses the production cache and equal-size 4 KiB payloads. +A 64-entry-equivalent byte limit models the previous residency ceiling; the +current case allows 1 MiB. Leases are released sequentially. It allocates/fills +owned cache entries but excludes codec decoding, network I/O, and query planning. +Allocation count falls from 16,000 to 160, cumulative allocated bytes from +34,688,000 to 346,880. Tracked peak rises from 281,840 to 346,880 bytes because the +whole working set is retained, still below the allowance. Fixed inline cache +buckets are not heap allocations and are excluded from those byte figures. +Ranges were 0.944–0.987 ms and 0.222–0.227 ms. Separate regression tests retain all +80 leases simultaneously and exercise page-vs-metadata eviction under pressure. + +The sparse stateful regression has 4,097 dictionary nodes but only three metric +members in two original leaves. Each later node phase now schedules two data +pages instead of 65; normalization schedules two leaves instead of 65. Empty +iteration-zero node pages receive no worker attempt. The test reopens at iteration +one and compares PageRank, eigenvector, and paired HITS output against the numeric +oracle. These are asserted work counts, not a timing benchmark. Original ordinal +identities remain unchanged. + +The sealed-vector gather fixture still fetches only 128 storage chunks across +256 checkpoints (reference: 32,768). Its median is 16.888 ms, with 604,180 tracked +peak bytes. Cache entries and bucket allocations now draw from a shared 64 MiB +process pool, rather than multiplying a full allowance by every populated index. +Failure-injection and retirement tests verify that optional admission failures +and retired metrics release their charged bytes. + +## Shared point planning and checkpoint-local folds + +Same host/toolchain, one warmup and five samples, using the production helpers: + +| Phase / scenario | Allocating reference median | Current median | Tracked heap peak, before → after | +| --- | ---: | ---: | ---: | +| Warm ordinal fold, 4,096 tiles / 1,048,576 edge visits | 7.496 ms | 6.230 ms | 12,288 → 0 bytes | +| Point row planning, 100,000 common-prefix IDs / 1 column | 17.537 ms | 3.622 ms | 1,600,000 → 1,200,000 bytes | +| Point row planning, 100,000 common-prefix IDs / 16 columns | 290.126 ms | 8.552 ms | 25,600,000 → 1,200,000 bytes | +| Point row planning, 100,000 hashed IDs / 1 column | 4.178 ms | 2.517 ms | 1,600,000 → 1,200,000 bytes | +| Point row planning, 100,000 hashed IDs / 16 columns | 70.889 ms | 4.672 ms | 25,600,000 → 1,200,000 bytes | + +The fold fixture repeats a 256-edge tile with a warm source-vector chunk and +one target accumulator. Both paths perform the same compensated addition order +and return exactly equal sums. The reference owns decoded edges, source slots, +gathered ranks and contribution rows, using the same topology validation as the +borrowed path. Current execution also replaces per-edge target hash lookups with +a bounded chunk-local slot table. Across these edge visits, 16,384 allocations +and 50,331,648 cumulative allocated bytes become zero. Fixed stack scratch and +fixture/cache residency are **not** zero memory: fixture allocations are excluded +from tracking, while constant fixture setup is included in wall time. Storage +reads, cold cache fills, checkpoint commits and whole-build execution are excluded. +Measured ranges were 7.360–8.155 ms versus 6.011–6.528 ms. + +Point fixtures use 391 routing blocks, deterministic permuted row order, and +either 30-byte collection-prefixed IDs or 16-byte hashed hexadecimal IDs. Each +path verifies the same row/block checksum. The reference retains every column's +16-byte row map. The new path admits 8-byte transient comparison keys plus one +4-byte shared permutation; the keys are freed before column preparation, leaving +only 400,000 bytes of row-mapping ownership regardless of column count. It uses +two allocations versus one per reference column. Integer prefix keys improve +both tested single-column cases; sorting full strings alone regressed hashed +single-column IDs and was not retained. + +These are **sequential row-planning phase** measurements, not parallel column +execution or end-to-end query latency. They exclude output cells, control/routing +ownership, materialized block spans, score decoding, cache and network work. +Common-prefix single-column ranges were 17.273–18.224 ms versus 3.580–4.352 ms; +16-column ranges were 282.258–295.582 ms versus 8.319–8.814 ms. Hashed-ID ranges +were 4.078–4.624 ms versus 2.495–2.569 ms and 70.246–73.654 ms versus +4.133–4.898 ms. Sparse candidates, duplicate IDs, key lengths and shared-prefix +collisions change the balance; no universal speedup is claimed. + +## Initialization and query ownership follow-up + +Same host/toolchain, one warmup and five measured samples: + +| Phase / scenario | Reference or cold median | New or warm median | Allocations, before → after | +| --- | ---: | ---: | ---: | +| Membership discovery, 64 nodes / 16,384 producer partials | 2.063 ms | 0.069 ms | 16,398 → 90 | +| 64 authenticated 64-KiB disk hits versus warm memory leases | 5.838 ms | 0.009 ms | 64 → 0 request-payload allocations | +| Top-K response conversion, 10,000 IDs of 4,096 bytes | 3.169 ms | 0.021 ms | 10,001 → 1 | + +The membership reader is now used by vector initialization as well as iterations, +convergence and publication. This measures discovery and dictionary validation, +not a whole initializer, vector writes or a complete build. Maximum fan-in is a +deliberate stress case; fewer producer duplicates reduce the benefit. + +Cache measurements use warm filesystem data in both cases. The cold-memory case +includes verification and promotion; clearing memory between lookups is excluded. +It is a cache-state comparison, **not an exact pre-change implementation**. +Allocation tracking covers request payloads, excluding cache-owned allocations. +Disk-hit times ranged 5.817–5.887 ms; warm leases ranged 0.006–0.010 ms across 64 +lookups. Network and score decoding are excluded. A regression also verifies that +a warm leased hit succeeds with an allocator that rejects every allocation. + +Response-conversion peak includes the still-resident input: 82,400,000 bytes for +copying versus 41,440,000 for ownership transfer. New cumulative allocation falls +from 41,200,000 to 240,000 bytes. Copy times ranged 1.227–3.237 ms, transfer times +0.019–0.022 ms. Input construction, cleanup, fetching and JSON serialization are +excluded. Long IDs stress the ownership boundary; ordinary shorter IDs benefit +less. These are phase measurements, not end-to-end latency guarantees. + +## Sealed membership and output admission (initial measurements) + +Additional measurements on the same host/toolchain (one warmup, five samples): + +| Phase | Former path median | Current median | Allocations, before → after | +| --- | ---: | ---: | ---: | +| Canonical membership read, 64 nodes / 16,384 producer partials | 1.962 ms | 0.061 ms | 16,398 → 90 | +| Exhausted output quota, 50,000 nodes / 400,000 edges | 10.302 ms | 6.448 ms | 34 → 20 | + +Membership uses real default storage and the same ordered dictionary validation +in both paths. It includes transaction and output ownership, but excludes fixture +writes and numerical folds. The fixture deliberately exercises maximum producer +fan-in: speedups will be smaller with fewer duplicate producer rows. Cumulative +allocation fell from 216,797 to 6,259 bytes; tracked peak increased slightly from +2,774 to 3,192 bytes. Times ranged 1.958–2.031 ms versus 0.060–0.063 ms. + +Output rejection includes one source and projection preparation in both paths. +The reference computes and encodes PageRank before rejecting an exhausted output +quota; production rejects before numerical allocation or encoding. The symmetric +degree-eight ring can converge early (maximum three iterations); this does not +claim savings for three complete iterations. Cumulative allocation fell from +19,553,871 to 16,306,596 bytes, while peak remained 12,906,304 bytes because shared +preparation dominates. Times ranged 10.264–10.342 ms versus 6.437–6.481 ms. +Fetch, upload, rejection-sidecar encoding and cloud latency are excluded. + +Metadata-tail skipping is checked as an operation-count regression: encountering +the metadata namespace issues one range seek regardless of the number of metric +records. No wall-clock speedup is claimed for that regression. + +Run from `zig/`: + +```sh +zig build -Doptimize=ReleaseFast graph-metric-preparation-bench --summary all > /tmp/graph-metric-bench.jsonl 2> /tmp/graph-metric-bench-build.log +``` + +The executable emits JSONL including min/max time, allocation count, cumulative +allocated bytes, tracked peak bytes, and workload dimensions. Separate stdout +and stderr preserve the machine-readable measurements. + +## Serverless topology preparation + +Fixtures are directed degree-eight rings with 48-byte document IDs and one edge +type. Both paths consume the **same current v3 payload**. The reference decodes +owned adjacency strings, discards inbound edges, then compiles through string +hash maps. Production reads borrowed ordinal views directly. Fixture memory, +input payload residency, fetch, encoding, projection and numerical kernels are +excluded from this timed/tracked phase. + +| Nodes / outbound edges | Reference median | Packed median | Reference peak | Packed peak | +| --- | ---: | ---: | ---: | ---: | +| 2,000 / 16,000 | 1.580 ms | 0.163 ms | 3,668,016 B | 380,051 B | +| 20,000 / 160,000 | 41.258 ms | 1.484 ms | 36,680,016 B | 3,800,051 B | +| 50,000 / 400,000 | 121.782 ms | 3.498 ms | 91,700,016 B | 9,500,051 B | + +At the largest size, preparation was about 35x faster and tracked peak +allocation was 9.65x smaller. Allocations fell from 1,750,046 to 11. Reference +time ranged 98.056–133.335 ms, packed time 3.474–3.617 ms. + +The v3 payload was 16,100,033 bytes versus 61,500,014 bytes for the exact size of +the former string-repeating v2 layout: 73.8% smaller on this fixture. The +benchmark computes the old size formula; it does not retain a legacy codec. +Short identifiers, low-degree graphs, or many distinct edge types will have +different compression ratios. + +## Non-serverless snapshot score reader + +Fixtures request four distinct metrics over 20,000 rows. The reference recreates +the former bounded sorted-key reader with a per-batch key arena and complete +key construction per logical score. Production reuses encoded prefixes and a +key slab, and deduplicates physical rows. A synchronous mock transaction checks +key ordering and hashes keys; it does **not** model LSM/LMDB latency. Every output +cell is checked outside timing. Input fixtures and output arrays are excluded +from allocation tracking. + +| Rows | Reference median | Physical reader median | Storage keys, before → after | Allocations, before → after | +| --- | ---: | ---: | ---: | ---: | +| All unique | 6.593 ms | 5.358 ms | 80,000 → 80,000 | 4,625 → 6 | +| Each node repeated twice | 7.180 ms | 3.916 ms | 80,000 → 40,000 | 4,625 → 7 | + +The unique-row case reduced median reader CPU time by 18.7%, cumulative allocations +from 13,421,150 to 715,200 bytes, and tracked peak from 838,608 to 715,200 bytes. +The repeated-row case reduced median time by 45.5%, halved storage keys, and used +795,200 peak bytes. Distinct logical aliases also share physical reads, covered +by regression tests rather than included in this timing comparison. +The unique-row production run included a 26.419 ms outlier (minimum 5.300 ms); +the duplicate-row production range was 3.898–3.948 ms. Use repeated runs on an +isolated host for latency guarantees, not these development-host samples. + +These are phase microbenchmarks, **not end-to-end query or PageRank speedups**. +Production additionally pays for I/O, authentication, snapshot/status handling, +output ownership and numerical work. Tests cover default non-serverless storage, +durable ordinal jobs, serverless publication/query integration, cancellation, +malformed input, allocation failures and alias ownership. + +## Admitted preparation and ordinal execution + +The following measurements were added with the bounded census, admission and +compact-query changes, on the same host and toolchain. They exercise production +functions against explicit former-path oracles, not alternate numerical kernels. + +| Phase | Former path median | Current path median | Scope | +| --- | ---: | ---: | --- | +| Exhausted serverless projection preparation | 26.469 ms | 3.204 ms | 50,000 nodes, 400,000 edges; 16 rejected group attempts | +| Durable vector writer | 3.637 ms | 0.101 ms | 20,000 rows; synchronous mock storage | +| One-node score snapshot during rebuild | 194 µs | 103 µs | Real default storage; 256 active scan pages | +| 64-node score snapshot during rebuild | 638 µs | 553 µs | Same real-storage fixture | + +The rejected-preparation case includes one packed source preparation, then 16 +independent projection attempts against an exhausted work budget. The former +oracle constructs each projection before rejecting; production rejects before +projection allocations or census scans. This isolates admission ordering and +does not include the publication grouping/cache, fetch, rejection encoding, or +numerical kernel. Median time fell 87.9%; allocations fell from 123 to 11 and +cumulative allocated bytes from 35,600,691 to 9,900,323. Peak stayed 9,500,051 bytes +because the shared source preparation dominates it. This is not a claim that +rejection can avoid preparing the source itself. + +The vector writer compares node-ID rows with rows already carrying their +job-local ordinals. Both execute the production writer and validate output +scores. Storage reads fell from 20,079 to 79, eliminating 20,000 dictionary point +lookups; writes remained 79. Allocations fell from 137 to 8 and tracked peak from +4,775,772 to 806,756 bytes. The roughly 36x writer CPU improvement excludes ordinal +discovery, real storage latency, adjacency reads, and numerical iteration. The +production reducer discovers ordinals with a canonical-node/dictionary range +join; the benchmark does not claim an equivalent whole-PageRank speedup. + +The query fixture publishes degree scores over 4,096 nodes and 16,384 edges, +then opens a real 256-page rebuild. Both paths include a read transaction and the +same score reader; the reference builds operator status, while production reads +only publication/freshness metadata. Validation and result freeing are outside +timing. One-node median latency fell 46.9% (p95: 198 → 117 µs); 64-node median +fell 13.3% (p95: 662 → 640 µs). These are storage-level snapshots, not HTTP latency. +Runs overlapped development/test activity; rerun on an isolated host before +setting latency guarantees. + +See [execution and resource ownership](../../docs/GRAPH_METRICS_EXECUTION.md) +for the associated admission, checkpoint, and integrity contracts. + +## Staged stateful metric queries + +Run just this case with: + +```sh +zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -- --staged-only +``` + +The fixture seeds 16 published score columns in the default storage backend, +with 100,000 distinct node IDs. One metric orders the top ten rows; all sixteen +are projected. The eager reference loads every dependency before selection. +The staged path uses the production snapshot reader and stage workspace. Both +must return exactly the same selected ordinals and every projected score. + +Final local ReleaseFast rerun (six samples, first discarded): + +| Metric | Eager reference | Staged reads | +| --- | ---: | ---: | +| Logical score keys | 1,600,000 | 100,150 | +| Median execution | 17.575 s | 1.101 s | +| Tracked peak allocations | 29,233,056 B | 5,232,576 B | + +The timer includes snapshot acquisition, score reads, selection, validation and +scratch cleanup. Fixture writes, traversal, response encoding and backend-owned +allocations are excluded. These are warm-cache results on a shared development +host with concurrent builds, not end-to-end latency guarantees. The reference +and current paths use the same byte-bounded physical score reader. The measured +work-count reduction is independent of host contention. + +The preceding full benchmark run measured 21.108 s and 1.085 s respectively; +the difference between runs illustrates why the timing is a local measurement, +not a service-level guarantee. Both runs reported the same key counts and peak +allocation sizes. + +## Sparse projection and resource-bounded iterations + +ReleaseFast measurements on the same host/toolchain, with a prepared +1,000,000-entry dictionary and two selected edges (two active endpoints). +Each sample contains 256 repetitions; one warmup sample is discarded and the +median of five samples is reported. The inactive dictionary entries are fixture +placeholders; preparation, storage I/O, kernels and upload are excluded. + +| Projection | Source-wide scratch reference | Active-endpoint path | Peak scratch, reference → current | +| --- | ---: | ---: | ---: | +| Degree | 795.582 µs | 0.179 µs | 8,125,065 → 73 bytes | +| PageRank | 830.437 µs | 0.218 µs | 4,125,097 → 89 bytes | + +Node-ID and CSR checksums must match in every repetition. The dense degree +reference is the previous direct-count path; PageRank's reference retains the +former projected-edge copy (only 16 bytes in this fixture). These are deliberately +sparse phase measurements, not end-to-end speedups. Dense projections continue +to use linear-time maps/counts rather than sorting every edge endpoint. +Concurrent development builds were active; use isolated runs for latency +guarantees. The allocation and output-parity checks do not depend on timing. + +Stateful iteration planning now eliminates later adjacency-producer pages +entirely. With 256 edge partitions and 100 iterations this removes 25,344 +PageRank/eigenvector no-op page executions and at least 50,688 claim/completion +commits; HITS removes twice those counts. This is a deterministic work-count +comparison, not a measured storage-latency claim. Regression tests verify +absence of later producer pages, immutable adjacency reuse, recovery, retries, +publication barriers and numerical parity. + +Serverless regressions also enforce two resource oracles: an 8,192-score prior +larger than 64 KiB can seed a sparse selection within a 64 KiB read/memory budget, +while dense seeds read less than the whole artifact by omitting ranked payloads; +and two concurrent requests for two cold routing pages perform exactly two +decoded fills even with 63 of the 64 fill slots occupied. The latter checks +shared lease identity and completion under fill-table saturation. + +## Ordinal ingestion and selected-type discovery + +Run `zig build graph-metric-preparation-bench -Doptimize=ReleaseFast -j1 -- --indexing-only`. +On Apple M4 Max / Zig 0.16.0, the following medians discard one warmup and +retain five samples. Each ingestion fixture has 1,024 nodes and 65,536 edges; +JSON parsing, construction and encoding are timed, input residency is excluded. +Every output has identical encoded SHA-256 to the old string-expansion oracle. + +| Node ID bytes | String builder → ordinal builder | Peak allocated bytes | Allocation calls | +| --- | ---: | ---: | ---: | +| 16 | 23.250 → 17.474 ms | 11,276,518 → 6,228,850 | 438,315 → 162,863 | +| 256 | 73.502 → 30.934 ms | 43,471,078 → 6,720,370 | 442,411 → 166,959 | + +The stateful fixture uses the default durable LSM, 65,536 edges and 16 +relationship types. Selecting one type visits 4,096 postings instead of 65,536 +reverse edges: 0.640 ms versus 14.110 ms. Selected identity checksums match. +This measures discovery only, excluding writes, migration, and numerical work. +The covering index adds one empty-value identity key per edge and its write/ +storage cost; existing indexes pay one bounded backfill. These are local phase +measurements on a shared development host, not end-to-end latency guarantees. + +### Direct packing, filter-local census and semantic metric reuse + +The same `--indexing-only` command now also measures cold durable partition +planning and a weight-only PageRank republish. The following ReleaseFast rerun +uses Apple M4 Max / Zig 0.16.0, six samples with the first discarded. Other +development builds were active; timings are observations, not latency guarantees. + +| Case | Reference | Current | Deterministic check | +| --- | ---: | ---: | --- | +| Parse/build/encode, 16-byte IDs | 24.675 ms | 15.639 ms | Identical encoded SHA-256 | +| Parse/build/encode, 256-byte IDs | 81.795 ms | 28.786 ms | Identical encoded SHA-256 | +| Selected edge discovery | 14.362 ms | 0.642 ms | Same 4,096 edge identities; 65,536 → 4,096 visits | +| Cold partition census | 254.022 ms | 87.298 ms | 17 → 3 checkpoint steps; exact selected edge/node totals | +| Weight-only PageRank republish | 4.418 ms | 3.067 ms | Exactly equal scores; 2,362,369 → 0 projection/kernel work units | + +Packing uses the old string builder as the reference. Current peak allocations +are 3,631,986 and 4,123,506 bytes, versus the reference's 11,276,518 and +43,471,078 bytes. Compared with the preceding ordinal builder's measured +6,228,850 and 6,720,370 bytes, direct count/scatter packing removes 2,596,864 +bytes in either fixture. That allocation comparison is deterministic; the +preceding run's CPU timings are not a controlled incremental comparison. + +The census reference is the former whole-graph planning prerequisite, not a +different algorithm computing a selected plan. Both use the default durable LSM; +timing includes clearing the old plan and committing all counts, boundaries and +checkpoints, but excludes fixture ingestion and numerical execution. The selected +plan has 4,096 edges and 1,024 distinct endpoints, independent of unrelated types. +The v2 index adds one reference-counted membership record per (type, endpoint), +with two incidence updates per changed edge in addition to the edge posting. +This fixture has 16,384 such records. Write/storage amplification is a deliberate +tradeoff, shared across all metric filters; this benchmark does not measure its +ingestion cost or the initial backfill. + +The republish fixture has 1,024 nodes, 65,536 edges, a high-degree hub and repeated +edges, with a 30-iteration PageRank cap. Only weights change. Its timer includes +authenticated cached-source reads, topology preparation, semantic hashing and +publication; graph construction and post-run score validation are outside it. +Peak tracked allocations remain 2,755,027 bytes in both cases because source +preparation dominates the peak. Artifact-store-owned memory is outside that +tracker. Semantic reuse removes projection/kernel/output work, but still reads +and prepares a changed source graph; it does not claim a cold object-store I/O +reduction. The reuse assertion also requires the exact prior metric artifact ID. diff --git a/zig/bench/graph/metric_preparation_bench.zig b/zig/bench/graph/metric_preparation_bench.zig new file mode 100644 index 0000000000..50819eac98 --- /dev/null +++ b/zig/bench/graph/metric_preparation_bench.zig @@ -0,0 +1,1737 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const std = @import("std"); +const antfly = @import("antfly_zig"); +const graph = antfly.serverless.graph_segment; +const metric = antfly.serverless.build.lake_graph_metric; + +const PhaseAllocStats = struct { + current_bytes: usize = 0, + peak_bytes: usize = 0, + total_alloc_bytes: usize = 0, + total_free_bytes: usize = 0, + alloc_count: usize = 0, + free_count: usize = 0, + + fn noteAlloc(self: *PhaseAllocStats, len: usize) void { + self.current_bytes +|= len; + self.total_alloc_bytes +|= len; + self.alloc_count +|= 1; + self.peak_bytes = @max(self.peak_bytes, self.current_bytes); + } + + fn noteFree(self: *PhaseAllocStats, len: usize) void { + self.current_bytes -|= len; + self.total_free_bytes +|= len; + self.free_count +|= 1; + } + + fn noteResize(self: *PhaseAllocStats, old_len: usize, new_len: usize) void { + if (new_len > old_len) { + self.noteAlloc(new_len - old_len); + } else if (old_len > new_len) { + self.noteFree(old_len - new_len); + } + } +}; + +const PhaseTrackingAllocator = struct { + backing: std.mem.Allocator, + stats: *PhaseAllocStats, + + fn allocator(self: *PhaseTrackingAllocator) std.mem.Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *PhaseTrackingAllocator = @ptrCast(@alignCast(ctx)); + const ptr = self.backing.rawAlloc(len, alignment, ret_addr) orelse return null; + self.stats.noteAlloc(len); + return ptr; + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + const self: *PhaseTrackingAllocator = @ptrCast(@alignCast(ctx)); + if (!self.backing.rawResize(memory, alignment, new_len, ret_addr)) return false; + self.stats.noteResize(memory.len, new_len); + return true; + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self: *PhaseTrackingAllocator = @ptrCast(@alignCast(ctx)); + const ptr = self.backing.rawRemap(memory, alignment, new_len, ret_addr) orelse return null; + self.stats.noteResize(memory.len, new_len); + return ptr; + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *PhaseTrackingAllocator = @ptrCast(@alignCast(ctx)); + self.backing.rawFree(memory, alignment, ret_addr); + self.stats.noteFree(memory.len); + } +}; + +fn benchmarkScoreJoin(output: anytype) !void { + const codec = antfly.serverless.graph_metric_segment.codec; + var block = codec.DecodedScoreBlock{ .len = 1024, .node_prefix = "collection/" }; + var suffixes: [1024][8]u8 = undefined; + var names: [1024][19]u8 = undefined; + var ids: [1024][]const u8 = undefined; + var rows: [1024]u32 = undefined; + var values: [1024]?f64 = @splat(null); + for (&suffixes, &names, &ids, &rows, 0..) |*suffix, *name, *id, *row, i| { + const text = try std.fmt.bufPrint(suffix, "{d:0>8}", .{i}); + id.* = try std.fmt.bufPrint(name, "collection/{s}", .{text}); + row.* = @intCast(i); + block.scores[i] = .{ .node_suffix = text, .value = @floatFromInt(i) }; + } + for ([_]usize{ 1, 16, 256, 1024 }) |count| { + for (0..count) |i| rows[i] = @intCast(i * 1024 / count); + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + for (0..6) |sample| { + const start = antfly.platform_time.monotonicNs(); + for (0..4096) |_| { + if (reference) { + for (rows[0..count]) |row| values[row] = block.score(ids[row]); + } else try block.populateSorted(&ids, rows[0..count], &values, .none); + std.mem.doNotOptimizeAway(&values); + } + const elapsed = (antfly.platform_time.monotonicNs() - start) / 4096; + for (rows[0..count]) |row| if (values[row] != @as(f64, @floatFromInt(row))) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(std.heap.smp_allocator, .{ + .mode = if (reference) "score_join_binary_reference" else "score_join_adaptive", + .rows = count, + .block_rows = 1024, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .note = "borrowed decoded block; excludes decode and I/O; six samples, first discarded; 4096 repetitions", + }, .{}); + defer std.heap.smp_allocator.free(json); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + } + } +} + +fn benchmarkSparseProjections(output: anytype) !void { + const alloc = std.heap.smp_allocator; + const ids = try alloc.alloc([]const u8, 1_000_000); + defer alloc.free(ids); + @memset(ids, "unused"); + ids[0] = "a"; + ids[ids.len - 1] = "z"; + inline for (.{ .degree, .pagerank }) |kind| { + var expected: ?u64 = null; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + for (0..256) |_| { + const digest = try metric.benchmarkSparseProjection(tracking.allocator(), ids, kind, reference); + if (expected) |value| { + if (value != digest) return error.InvalidBenchmarkResult; + } else expected = digest; + } + const elapsed = (antfly.platform_time.monotonicNs() - start) / 256; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "sparse_source_wide_reference" else "sparse_active_endpoints", + .kind = @tagName(kind), + .source_nodes = ids.len, + .active_nodes = 2, + .edges = 2, + .median_ns = times[2], + .peak_bytes = last.peak_bytes, + .allocation_count = last.alloc_count / 256, + .note = "prepared dictionary excluded; exact node/CSR checksum parity; 256 repetitions per sample; six samples, first discarded", + }, .{}); + defer alloc.free(json); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + } + } +} + +pub fn main(init: std.process.Init) !void { + var output_buf: [4096]u8 = undefined; + var output = std.Io.File.stdout().writer(init.io, &output_buf); + var args = try std.process.Args.Iterator.initAllocator(init.minimal.args, std.heap.smp_allocator); + defer args.deinit(); + _ = args.next(); + var staged_only = false; + var topology_only = false; + var score_join_only = false; + var ordinal_cursors_only = false; + var indexing_only = false; + while (args.next()) |arg| { + if (std.mem.eql(u8, arg, "--paged-only")) return @import("paged_read_bench.zig").run(init.io, &output); + if (std.mem.eql(u8, arg, "--presence-only")) return benchmarkPresence(&output); + if (std.mem.eql(u8, arg, "--indexing-only")) { + indexing_only = true; + continue; + } + if (std.mem.eql(u8, arg, "--ordinal-cursors-only")) ordinal_cursors_only = true else if (std.mem.eql(u8, arg, "--staged-only")) staged_only = true else if (std.mem.eql(u8, arg, "--topology-only")) topology_only = true else if (std.mem.eql(u8, arg, "--score-join-only")) score_join_only = true else return error.InvalidArgument; + } + if (indexing_only) { + try benchmarkGraphIndexConstruction(&output); + try benchmarkTypedEdgeScans(init.io, &output); + try benchmarkCommittedCounters(init.io, &output); + try benchmarkSelectedTopologyReads(init.io, &output); + return benchmarkSemanticMetricReuse(init.io, &output); + } + if (score_join_only) return benchmarkScoreJoin(&output); + if (ordinal_cursors_only) return benchmarkOrdinalCursors(init.io, &output); + if (topology_only) return benchmarkSharedTopology(init.io, &output); + try benchmarkStagedQueries(init.io, &output); + if (staged_only) return; + try benchmarkStateful(&output); + try benchmarkVectorWrites(&output); + try benchmarkQuerySnapshots(init.io, &output); + try benchmarkMembership(init.io, &output); + try benchmarkOrdinalFold(&output); + try benchmarkSealedVectors(init.io, &output); + try benchmarkPublication(init.io, &output); + try benchmarkRoutingWorkingSet(&output); + try benchmarkSparseProjections(&output); + try benchmarkCandidatePlanning(&output); + try benchmarkScoreJoin(&output); + try benchmarkAuthenticatedCache(init.io, &output); + try benchmarkTopOwnership(&output); + for ([_]usize{ 2_000, 20_000, 50_000 }) |nodes| { + var fixture = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer fixture.deinit(); + const alloc = fixture.allocator(); + const degree = 8; + const ids = try alloc.alloc([]u8, nodes); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(alloc, "source/snapshot/file-0001/customer-record-{d:0>8}", .{i}); + const adjacencies = try alloc.alloc(graph.Adjacency, nodes); + var old_wire_bytes: usize = 14; + for (adjacencies, 0..) |*adjacency, i| { + const out = try alloc.alloc(graph.Edge, degree); + const in = try alloc.alloc(graph.Edge, degree); + for (out, in, 0..) |*forward, *reverse, j| { + forward.* = .{ .neighbor_id = ids[(i + j + 1) % nodes], .edge_type = @constCast("follows"), .weight = 1 }; + reverse.* = .{ .neighbor_id = ids[(i + nodes - j - 1) % nodes], .edge_type = @constCast("follows"), .weight = 1 }; + old_wire_bytes += 2 * (16 + ids[i].len + "follows".len); + } + const less = struct { + fn less(_: void, a: graph.Edge, b: graph.Edge) bool { + return graph.edgeLookupOrder(a.edge_type, a.neighbor_id, b.edge_type, b.neighbor_id) == .lt; + } + }.less; + std.mem.sort(graph.Edge, out, {}, less); + std.mem.sort(graph.Edge, in, {}, less); + adjacency.* = .{ .node_id = ids[i], .out_edges = out, .in_edges = in }; + old_wire_bytes += 12 + ids[i].len; + } + const segment = graph.Segment{ .adjacencies = adjacencies }; + const payload = try graph.encodeAlloc(alloc, segment); + if (nodes == 50_000) { + var expected: ?usize = null; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + const checksum = try metric.benchmarkProjection(tracking.allocator(), payload, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (expected) |value| { + if (value != checksum) return error.InvalidBenchmarkResult; + } else expected = checksum; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "projection_edge_copy_reference" else "projection_direct_csr", + .nodes = nodes, + .edges = nodes * degree, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "source preparation and PageRank projection; exact CSR checksum equality; excludes fetch, kernels and upload", + }, .{}); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + } + } + if (nodes == 50_000) for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + const count = try metric.benchmarkRejectedOutput(tracking.allocator(), payload, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (count != nodes * degree or stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "output_reject_after_kernel_reference" else "output_reject_before_kernel", + .nodes = nodes, + .edges = nodes * degree, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "includes source and projection preparation; reference computes and encodes PageRank before quota rejection; excludes fetch and upload", + }, .{}); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + }; + if (nodes == 50_000) for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + const edge_count = try metric.benchmarkRejectedPreparation(tracking.allocator(), payload, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (edge_count != nodes * degree or stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "rejection_after_projection_reference" else "rejection_before_projection", + .nodes = nodes, + .edges = nodes * degree, + .projection_groups = 16, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "includes one source preparation and sixteen exhausted projection attempts; excludes fetch and rejection encoding", + }, .{}); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + }; + for ([_]bool{ true, false }) |reference| { + _ = try metric.benchmarkPreparation(std.heap.smp_allocator, payload, reference); + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (×) |*elapsed| { + var stats = PhaseAllocStats{}; + var allocator = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + const edge_count = try metric.benchmarkPreparation(allocator.allocator(), payload, reference); + elapsed.* = antfly.platform_time.monotonicNs() - start; + if (edge_count != nodes * degree or stats.current_bytes != 0) return error.InvalidBenchmarkResult; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "unpack_hash_reference" else "packed_ordinals", + .nodes = nodes, + .edges = nodes * degree, + .v2_wire_bytes = old_wire_bytes, + .v3_wire_bytes = payload.len, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .samples = times.len, + .note = "same current-wire input; excludes fetch, encoding and numeric kernel; one warmup", + }, .{}); + try output.interface.writeAll(json); + try output.interface.writeByte('\n'); + try output.flush(); + } + } +} + +fn benchmarkGraphIndexConstruction(out: anytype) !void { + const alloc = std.heap.smp_allocator; + const builder = antfly.serverless.build.builder; + for ([_]usize{ 16, 256 }) |id_len| { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const ids = try fixture.alloc([]u8, 1024); + for (ids, 0..) |*id, i| { + id.* = try fixture.alloc(u8, id_len); + @memset(id.*, 'x'); + _ = try std.fmt.bufPrint(id.*[id_len - 8 ..], "{d:0>8}", .{i}); + } + const docs = try fixture.alloc(antfly.serverless.query.QueryMaterializedDocument, ids.len); + const InputEdge = struct { target: []const u8, edge_type: []const u8 = "link" }; + var edges: [64]InputEdge = undefined; + for (docs, 0..) |*doc, i| { + for (&edges, 0..) |*edge, j| edge.* = .{ .target = ids[(i + j + 1) % ids.len] }; + doc.* = .{ .doc_id = ids[i], .body = try std.json.Stringify.valueAlloc(fixture, .{ .graph_edges = &edges }, .{}), .last_lsn = 0, .last_timestamp_ns = 0 }; + } + var expected: ?[32]u8 = null; + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var measured: PhaseAllocStats = undefined; + var payload_len: usize = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const tracked = tracking.allocator(); + const started = antfly.platform_time.monotonicNs(); + const payload = if (reference) + (try builder.benchmarkReferenceGraphSegmentAlloc(tracked, "docs", docs, true)).payload orelse return error.InvalidBenchmarkResult + else + (try builder.buildGraphSegmentAlloc(tracked, "docs", docs, true)).payload orelse return error.InvalidBenchmarkResult; + const elapsed = antfly.platform_time.monotonicNs() - started; + payload_len = payload.len; + var checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &checksum, .{}); + if (expected) |prior| { + if (!std.mem.eql(u8, &prior, &checksum)) return error.InvalidBenchmarkResult; + } else expected = checksum; + tracked.free(payload); + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + measured = stats; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "graph_index_string_reference" else "graph_index_ordinal_builder", + .nodes = ids.len, + .edges = ids.len * edges.len, + .id_bytes = id_len, + .median_ns = samples[2], + .peak_bytes = measured.peak_bytes, + .allocation_count = measured.alloc_count, + .payload_bytes = payload_len, + .note = "same JSON input; exact encoded SHA256 parity; input residency excluded; six samples, first discarded; parse, construction and encoding included", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn benchmarkCommittedCounters(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const ids = try fixture.alloc([]const u8, 1024); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + const writes = try fixture.alloc(antfly.graph.BatchWrite, ids.len * 64); + const deletes = try fixture.alloc(antfly.graph.BatchDelete, writes.len); + for (writes, deletes, 0..) |*write, *delete, i| { + write.* = .{ .source = ids[i / 64], .target = ids[(i / 64 + i % 64 + 1) % ids.len], .edge_type = "link" }; + delete.* = .{ .source = write.source, .target = write.target, .edge_type = write.edge_type }; + } + for ([_]bool{ true, false }) |reference| { + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-global-counter-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "links", .{}); + defer index.close(); + var samples: [5]u64 = undefined; + for (0..6) |sample| { + const started = antfly.platform_time.monotonicNs(); + try index.benchmarkBatchApply(writes, &.{}, reference); + if (index.edge_count != writes.len or index.node_count != ids.len) return error.InvalidBenchmarkResult; + try index.benchmarkBatchApply(&.{}, deletes, reference); + const elapsed = antfly.platform_time.monotonicNs() - started; + if (index.edge_count != 0 or index.node_count != 0) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_global_counters_per_edge_committed" else "stateful_global_counters_coalesced_committed", + .edges = writes.len, + .nodes = ids.len, + .median_ns = samples[2], + .endpoint_counter_reads = if (reference) writes.len * 4 else ids.len * 2, + .note = "default durable LSM; scalar presence/per-edge counters versus sorted presence/coalesced counters; six insert+delete cycles, first discarded; identical original/final topology; includes both directional commits and WAL, excludes fixture; no forced compaction or reopen", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkPresence(out: anytype) !void { + const alloc = std.heap.smp_allocator; + const lsm = antfly.lsm_backend; + for ([_]usize{ 256, 16384 }) |value_bytes| { + var fixture = std.heap.ArenaAllocator.init(alloc); + defer fixture.deinit(); + const a = fixture.allocator(); + const keys = try a.alloc([]const u8, 1024); + for (keys, 0..) |*key, i| key.* = try std.fmt.allocPrint(a, "reverse/link/source-{d:0>8}", .{i}); + const value = try a.alloc(u8, value_bytes); + @memset(value, 'm'); + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const measured_alloc = tracking.allocator(); + var storage = lsm.MemoryStorage.init(alloc); + defer storage.deinit(); + var cache = lsm.Cache.init(measured_alloc, lsm.DefaultCacheSizeBytes); + defer cache.deinit(); + var backend = try lsm.Backend.open(measured_alloc, "/graph-presence-bench", .{ .flush_threshold = 1, .storage = storage.storage(), .cache = &cache }); + defer backend.close(); + var runtime = try backend.runtimeStore(measured_alloc, .{ .name = "graph" }); + defer runtime.deinit(); + { + var write = try runtime.beginWrite(); + errdefer write.abort(); + for (keys) |key| try write.put(key, value); + try write.commit(); + } + while (try backend.runMaintenanceStep()) {} + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var peaks: [5]usize = undefined; + var copies: u64 = 0; + for (0..6) |sample| { + const baseline = stats.current_bytes; + stats.peak_bytes = baseline; + const before = backend.snapshotReadStats(); + const started = antfly.platform_time.monotonicNs(); + var batch = try runtime.beginBatch(); + errdefer batch.abort(); + if (reference) { + for (keys) |key| if ((try batch.get(key)).len != value.len) return error.InvalidBenchmarkResult; + } else { + var present: [1024]bool = undefined; + try batch.containsManySorted(keys, &present); + for (present) |exists| if (!exists) return error.InvalidBenchmarkResult; + } + batch.abort(); + const elapsed = antfly.platform_time.monotonicNs() - started; + copies = backend.snapshotReadStats().point_value_copies - before.point_value_copies; + if (copies != (if (reference) @as(u64, keys.len) else 0)) return error.InvalidBenchmarkResult; + if (sample != 0) { + samples[sample - 1] = elapsed; + peaks[sample - 1] = stats.peak_bytes -| baseline; + } + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + std.mem.sort(usize, &peaks, {}, std.sort.asc(usize)); + const json = try std.json.Stringify.valueAlloc(a, .{ .mode = if (reference) "scalar_presence" else "sorted_presence", .keys = keys.len, .value_bytes = value_bytes, .median_ns = samples[2], .extra_peak_bytes = peaks[2], .value_copies = copies, .note = "warm immutable LSM runs and block cache on modeled storage; identical existing-key results; six samples, first discarded; includes batch lifetime; excludes fixture and disk latency" }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn benchmarkTypedEdgeScans(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-typed-edge-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "links", .{ .metric_configs = &.{.{ .name = "selected", .kind = .degree, .edge_filter = .{ .mode = .types, .types = &.{"type-00"} } }} }); + defer index.close(); + const ids = try fixture.alloc([]const u8, 1024); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + const types = try fixture.alloc([]const u8, 16); + for (types, 0..) |*kind, i| kind.* = try std.fmt.allocPrint(fixture, "type-{d:0>2}", .{i}); + const writes = try fixture.alloc(antfly.graph.BatchWrite, ids.len * 64); + for (writes, 0..) |*write, i| write.* = .{ .source = ids[i / 64], .target = ids[(i / 64 + i % 64 + 1) % ids.len], .edge_type = types[i % types.len] }; + try index.batchApply(writes, &.{}); + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var measured: PhaseAllocStats = undefined; + var endpoint_reads: usize = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const started = antfly.platform_time.monotonicNs(); + endpoint_reads = try index.benchmarkTypedMembershipUpdates(tracking.allocator(), writes, reference); + const elapsed = antfly.platform_time.monotonicNs() - started; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + measured = stats; + } + if (endpoint_reads != (if (reference) writes.len * 2 else ids.len * types.len)) return error.InvalidBenchmarkResult; + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_membership_per_edge" else "stateful_membership_coalesced", + .edges = writes.len, + .endpoint_reads = endpoint_reads, + .median_ns = samples[2], + .scratch_peak_bytes = measured.peak_bytes, + .scratch_allocations = measured.alloc_count, + .note = "default durable LSM; identical all-edge removal followed by abort; six samples, first discarded; includes posting maintenance, excludes fixture and WAL commit; allocator measures update scratch only", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + const filter = antfly.graph.GraphMetricEdgeFilter{ .mode = .types, .types = types[0..1] }; + const expected = try index.benchmarkMetricEdgeScan(filter, true); + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var measured: antfly.graph.GraphIndex.MetricEdgeScanBenchmark = undefined; + for (0..6) |sample| { + const started = antfly.platform_time.monotonicNs(); + measured = try index.benchmarkMetricEdgeScan(filter, reference); + const elapsed = antfly.platform_time.monotonicNs() - started; + if (measured.matched != expected.matched or measured.checksum != expected.checksum) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + } + if (measured.visited != (if (reference) writes.len else writes.len / types.len)) return error.InvalidBenchmarkResult; + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_filtered_full_scan" else "stateful_filtered_type_postings", + .edges = writes.len, + .visited = measured.visited, + .matched = measured.matched, + .median_ns = samples[2], + .note = "default durable LSM; same selected edge identity checksum; six samples, first discarded; discovery only, excludes fixture writes and numerical execution", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var steps: usize = 0; + for (0..6) |sample| { + const started = antfly.platform_time.monotonicNs(); + const result = try index.benchmarkPartitionCensus(if (reference) .{} else filter); + const elapsed = antfly.platform_time.monotonicNs() - started; + if (result.edges != (if (reference) writes.len else writes.len / types.len) or result.nodes != ids.len) return error.InvalidBenchmarkResult; + steps = result.steps; + if (sample != 0) samples[sample - 1] = elapsed; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_global_census" else "stateful_selected_census", + .source_edges = writes.len, + .selected_edges = writes.len / types.len, + .checkpoint_steps = steps, + .median_ns = samples[2], + .note = "cold plans on default durable LSM; includes clearing prior plan, durable checkpoints, counts and boundaries; excludes fixture writes and numerical execution", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + const control_oracle = try index.benchmarkPartitionPlanControl(true); + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + for (0..6) |sample| { + const started = antfly.platform_time.monotonicNs(); + for (0..128) |_| if (try index.benchmarkPartitionPlanControl(reference) != control_oracle) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = (antfly.platform_time.monotonicNs() - started) / 128; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_plan_with_boundaries" else "stateful_plan_control_only", + .median_ns = samples[2], + .control_bytes = 76, + .note = "default durable LSM; same validated plan identity; 128 reads per sample; reference includes addressed boundary loading and validation; no writes", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkSelectedTopologyReads(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-selected-topology-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + var fs = try antfly.serverless.artifacts.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + var builder = graph.Builder{ .alloc = alloc }; + defer builder.deinit(); + const ids = try fixture.alloc([]const u8, 16384); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .{i}); + for (ids, 0..) |id, i| { + for (0..16) |j| try builder.addEdge(id, ids[(i + j + 1) % ids.len], "noise", 1, null); + if (i < 256) try builder.addEdge(id, ids[(i + 1) % 256], "selected", 1, null); + } + const payload = try builder.encodeAlloc(256 * 1024 * 1024, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + var source = antfly.serverless.manifest.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = metadata.artifact_id, .checksum = metadata.checksum, .byte_len = metadata.byte_len }; + try graph.codec.compact.bindTopologyControl(&source, payload); + try artifacts.verifyContentWithCancellationUsingAllocator(alloc, source.artifact_id, source.byte_len, source.checksum, .none); + for ([_]bool{ true, false }) |sparse| { + const config = antfly.graph.GraphMetricConfig{ .name = "degree", .kind = .degree, .edge_filter = if (sparse) .{ .mode = .types, .types = &.{"selected"} } else .{} }; + const oracle = try metric.benchmarkSelectedArtifactPreparation(alloc, &artifacts, source, config, true); + for ([_][2]bool{ .{ true, true }, .{ false, true }, .{ true, false }, .{ false, false } }) |mode| { + const reference = mode[0]; + const warm = mode[1]; + var samples: [5]u64 = undefined; + var measured: PhaseAllocStats = undefined; + var result: metric.SelectedPreparationBenchmark = undefined; + for (0..6) |sample| { + var cold_fs = try antfly.serverless.artifacts.FsStore.init(alloc, root); + var cold_store = cold_fs.artifactStore(); + defer cold_store.deinit(); + var stats = PhaseAllocStats{}; + var tracker = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const started = antfly.platform_time.monotonicNs(); + result = try metric.benchmarkSelectedArtifactPreparation(tracker.allocator(), if (warm) &artifacts else &cold_store, source, config, reference); + const elapsed = antfly.platform_time.monotonicNs() - started; + if (stats.current_bytes != 0 or result.edges != oracle.edges or !std.mem.eql(u8, &result.digest, &oracle.digest)) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + measured = stats; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "serverless_sourcewide_preparation" else "serverless_addressed_preparation", + .sparse = sparse, + .warm_identity = warm, + .source_bytes = source.byte_len, + .selected_edges = result.edges, + .retained_nodes = result.retained_nodes, + .read_bytes = result.read_bytes, + .median_ns = samples[2], + .peak_bytes = measured.peak_bytes, + .allocations = measured.alloc_count, + .note = "local source; cold identity uses a new verifier per sample, not a cold OS page cache; includes preparation and semantic identity, excludes fixture and numerical kernel; six samples, first discarded; digest parity", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn benchmarkSemanticMetricReuse(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + const manifest = antfly.serverless.manifest; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-semantic-reuse-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + var fs = try antfly.serverless.artifacts.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + const ids = try fixture.alloc([]const u8, 1024); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + var sources: [2]manifest.ArtifactRef = undefined; + for (&sources, 0..) |*source, variant| { + var builder = graph.Builder{ .alloc = alloc }; + defer builder.deinit(); + for (ids, 0..) |id, i| for (0..64) |j| { + const target = if (j % 4 == 0) 0 else (i + j * j + 1) % ids.len; + try builder.addEdge(id, ids[target], "link", @floatFromInt(variant + 1), null); + }; + const payload = try builder.encodeAlloc(16 * 1024 * 1024, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + source.* = .{ .kind = .graph_segment, .name = "graph", .artifact_id = try fixture.dupe(u8, metadata.artifact_id), .checksum = try fixture.dupe(u8, metadata.checksum), .byte_len = metadata.byte_len }; + try graph.codec.compact.bindTopologyControl(source, payload); + } + const config = antfly.graph.GraphMetricConfig{ .name = "rank", .kind = .pagerank, .max_iterations = 30, .tolerance = 1e-15 }; + const first_request = metric.PublicationRequest{ .graph_index_name = "graph", .source_graph = sources[0], .config = config, .provenance = .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 } }; + var first_budget = antfly.serverless.build.graph_metric_policy.Budget{ .limits = .{} }; + const first = try metric.publishRequestsAlloc(alloc, &artifacts, &.{first_request}, .none, .{}, &first_budget, .{}); + defer { + manifest.types.freeArtifactRefs(alloc, first); + alloc.free(first); + } + const first_bytes = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, first[0].artifact_id, first[0].byte_len, first[0].checksum, .none); + defer alloc.free(first_bytes); + var expected = try antfly.serverless.graph_metric_segment.decodeAlloc(alloc, first_bytes); + defer expected.deinit(alloc); + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + var measured = PhaseAllocStats{}; + var work: u64 = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracker = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const tracked = tracker.allocator(); + const request = metric.PublicationRequest{ .graph_index_name = "graph", .source_graph = sources[1], .config = config, .prior_artifact = if (reference) null else first[0], .provenance = .{ .published_generation = 2, .edge_generation = 2, .computed_at_ms = 2 } }; + var budget = antfly.serverless.build.graph_metric_policy.Budget{ .limits = .{} }; + const started = antfly.platform_time.monotonicNs(); + const refs = try metric.publishRequestsWithPriorAlloc(tracked, &artifacts, &.{request}, if (reference) &.{} else first, .none, .{}, &budget, .{}); + const elapsed = antfly.platform_time.monotonicNs() - started; + const bytes = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, refs[0].artifact_id, refs[0].byte_len, refs[0].checksum, .none); + defer alloc.free(bytes); + var decoded = try antfly.serverless.graph_metric_segment.decodeAlloc(alloc, bytes); + defer decoded.deinit(alloc); + if (decoded.scores.len != expected.scores.len) return error.InvalidBenchmarkResult; + for (decoded.scores, expected.scores) |actual, wanted| { + if (!std.mem.eql(u8, actual.node_id, wanted.node_id) or actual.value != wanted.value) return error.InvalidBenchmarkResult; + } + if (!reference and (!std.mem.eql(u8, refs[0].artifact_id, first[0].artifact_id) or budget.work_items != 0)) return error.InvalidBenchmarkResult; + work = budget.work_items; + manifest.types.freeArtifactRefs(tracked, refs); + tracked.free(refs); + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + measured = stats; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "serverless_weight_change_recompute" else "serverless_weight_change_semantic_reuse", + .nodes = ids.len, + .edges = ids.len * 64, + .median_ns = samples[2], + .peak_bytes = measured.peak_bytes, + .projection_kernel_work = work, + .note = "same weighted source change; exact score parity; includes authenticated cached-source reads, preparation, identity hashing and publication; excludes graph construction and post-run score validation; six samples, first discarded", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkOrdinalCursors(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + for ([_]usize{ 16, 4096 }) |id_len| { + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-ordinal-cursor-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + const configs = [_]antfly.graph.GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }}; + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "links", .{ .metric_configs = &configs }); + defer index.close(); + var ids: [256][]const u8 = undefined; + for (&ids, 0..) |*id, i| { + const bytes = try fixture.alloc(u8, id_len); + @memset(bytes, 'x'); + _ = try std.fmt.bufPrint(bytes[0..8], "{d:0>8}", .{i}); + id.* = bytes; + } + for (ids, 0..) |id, i| try index.addEdge(id, ids[(i + 1) % ids.len], "cites", 1, 0, 0, ""); + try index.benchmarkPrepareOrdinalCursor("rank"); + const expected = try index.benchmarkOrdinalCursorRead("rank", true); + for ([_]bool{ true, false }) |reference| { + var samples: [5]u64 = undefined; + for (0..6) |sample| { + const started = antfly.platform_time.monotonicNs(); + for (0..64) |_| if (try index.benchmarkOrdinalCursorRead("rank", reference) != expected) return error.InvalidBenchmarkResult; + const elapsed = (antfly.platform_time.monotonicNs() - started) / 64; + if (sample != 0) samples[sample - 1] = elapsed; + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const encoded = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_string_cursor" else "stateful_ordinal_cursor", + .nodes = ids.len, + .node_id_bytes = id_len, + .median_ns = samples[2], + .note = "same sealed topology; exact ordinal checksum parity; cursor traversal only, excludes numeric kernel and writes; 64 repetitions; six samples, first discarded", + }, .{}); + try out.interface.writeAll(encoded); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn benchmarkSharedTopology(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-shared-topology-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + const configs = [_]antfly.graph.GraphMetricConfig{ + .{ .name = "seed", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }, + .{ .name = "candidate", .kind = .pagerank, .refresh = .manual, .max_iterations = 1, .damping = 0.5 }, + }; + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "links", .{ .metric_configs = &configs }); + defer index.close(); + const ids = try fixture.alloc([]const u8, 1024); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + for (ids, 0..) |source, i| for (1..17) |offset| try index.addEdge(source, ids[(i + offset) % ids.len], "cites", 1, 0, 0, ""); + var seed = try index.runPageRankMetricPlanned("seed"); + seed.deinit(alloc); + for ([_]bool{ false, true }) |reuse| { + var samples: [5]u64 = undefined; + var measured: antfly.graph.GraphIndex.TopologyBuildBenchmark = undefined; + for (0..6) |sample| { + const start = antfly.platform_time.monotonicNs(); + measured = try index.benchmarkTopologyBuild("candidate", reuse); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (measured.adopted != reuse or measured.physical_edge_units != (if (reuse) @as(u64, 0) else 2 * 16 * ids.len)) return error.InvalidBenchmarkResult; + if (sample != 0) samples[sample - 1] = elapsed; + for (ids) |id| if (@abs((try index.graphMetricScore("candidate", id)).? - 1.0 / @as(f64, @floatFromInt(ids.len))) > 1e-12) return error.InvalidBenchmarkResult; + while (try index.cleanupRetiredGraphMetricScoreGenerationPage("candidate")) {} + for (0..16) |_| _ = try index.cleanupGraphMetricTopologyPage(); + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const encoded = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reuse) "stateful_shared_topology" else "stateful_independent_topology", + .nodes = ids.len, + .edges = ids.len * 16, + .physical_edge_units = measured.physical_edge_units, + .checkpoints = measured.checkpoints, + .median_ns = samples[2], + .min_ns = samples[0], + .max_ns = samples[4], + .note = "default storage; complete one-iteration numerical job including publication and job cleanup; six samples, first discarded; excludes fixture writes, verification and topology GC; verifies every score and physical edge work", + }, .{}); + try out.interface.writeAll(encoded); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +const ScoreTxn = struct { + raw: [8]u8 = .{ 0, 0, 0, 0, 0, 0, 0xf0, 0x3f }, + key_count: usize = 0, + key_hash: u64 = 0, + pub fn getManySorted(self: *@This(), keys: []const []const u8, values: []?[]const u8) !void { + for (keys, values, 0..) |key, *value, i| { + if (i > 0 and std.mem.order(u8, keys[i - 1], key) == .gt) return error.UnsortedBenchmarkKeys; + self.key_hash +%= std.hash.Wyhash.hash(0, key); + value.* = &self.raw; + } + self.key_count += keys.len; + } +}; + +// Reference to the former sorted bounded reader: a key arena per batch and a +// freshly formatted complete metric/generation/node key per logical score. +fn referenceScores(alloc: std.mem.Allocator, txn: *ScoreTxn, names: []const []const u8, nodes: []const []const u8, columns: []const []?f64) !void { + const rows = try alloc.alloc(usize, nodes.len); + defer alloc.free(rows); + for (rows, 0..) |*row, i| row.* = i; + const Order = struct { + nodes: []const []const u8, + fn less(self: @This(), a: usize, b: usize) bool { + const order = std.mem.order(u8, self.nodes[a], self.nodes[b]); + return order == .lt or (order == .eq and a < b); + } + }; + std.mem.sort(usize, rows, Order{ .nodes = nodes }, Order.less); + var offset: usize = 0; + const total = names.len * nodes.len; + const Pending = struct { column: usize, row: usize, key: []const u8 }; + while (offset < total) { + const len = @min(4096, total - offset); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const ka = arena.allocator(); + const pending = try alloc.alloc(Pending, len); + defer alloc.free(pending); + for (pending, 0..) |*item, i| { + const flat = offset + i; + const column = flat / nodes.len; + const row = rows[flat % nodes.len]; + var generation_buf: [20]u8 = undefined; + const generation = try std.fmt.bufPrint(&generation_buf, "{d}", .{@as(u64, 12345)}); + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(ka); + try key.appendSlice(ka, "meta:metric:"); + for ([_][]const u8{ names[column], "score", generation, nodes[row] }) |part| try antfly.internal_keys.appendEncodedComponent(&key, ka, part); + item.* = .{ .column = column, .row = row, .key = try key.toOwnedSlice(ka) }; + } + const keys = try alloc.alloc([]const u8, len); + defer alloc.free(keys); + const values = try alloc.alloc(?[]const u8, len); + defer alloc.free(values); + for (pending, keys) |item, *key| key.* = item.key; + @memset(values, null); + try txn.getManySorted(keys, values); + for (pending, values) |item, value| columns[item.column][item.row] = if (value) |raw| @bitCast(std.mem.readInt(u64, raw[0..8], .little)) else null; + offset += len; + } +} + +const VectorWriteTxn = struct { + slots: []const [8]u8, + reads: usize = 0, + writes: usize = 0, + sum: f64 = 0, + pub fn get(self: *@This(), key: []const u8) ![]const u8 { + self.reads += 1; + const pos = std.mem.indexOf(u8, key, "node-") orelse return error.NotFound; + const i = try std.fmt.parseInt(usize, key[pos + 5 ..][0..8], 10); + return &self.slots[i]; + } + pub fn getManySorted(self: *@This(), keys: []const []const u8, values: []?[]const u8) !void { + for (keys, values) |key, *value| value.* = try self.get(key); + } + pub fn put(self: *@This(), _: []const u8, bytes: []const u8) !void { + const chunk = antfly.graph.vector_chunk; + self.writes += 1; + for (0..chunk.entries) |i| self.sum += try chunk.get(bytes, i, false); + } + pub fn delete(_: *@This(), _: []const u8) anyerror!void { + return error.NotFound; + } +}; + +fn benchmarkVectorWrites(out: anytype) !void { + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer arena.deinit(); + const fixture = arena.allocator(); + const nodes = try fixture.alloc([]const u8, 20_000); + const slots = try fixture.alloc([8]u8, nodes.len); + for (nodes, slots, 0..) |*node, *slot, i| { + node.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + std.mem.writeInt(u64, slot, i + 1, .little); + } + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + var last_txn = VectorWriteTxn{ .slots = slots }; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + var index: antfly.graph.GraphIndex = undefined; + index.alloc = tracking.allocator(); + var txn = VectorWriteTxn{ .slots = slots }; + const start = antfly.platform_time.monotonicNs(); + try index.benchmarkVectorRowsAlloc(&txn, nodes, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (stats.current_bytes != 0 or txn.sum != @as(f64, @floatFromInt(nodes.len)) * 0.5) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + last_txn = txn; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "vector_write_node_ids_reference" else "vector_write_ordinal_rows", + .rows = nodes.len, + .storage_reads = last_txn.reads, + .storage_writes = last_txn.writes, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "one production vector write; mock storage; all output scores checked; excludes caller fixture and numerical iteration", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkStagedQueries(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + const query_mod = antfly.graph_query; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-metric-staged-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + var configs: [16]antfly.graph.GraphMetricConfig = undefined; + var reads: [16]query_mod.GraphMetricRead = undefined; + var names: [16][]const u8 = undefined; + for (&configs, &reads, &names, 0..) |*config, *read, *name, i| { + name.* = try std.fmt.allocPrint(fixture, "metric-{d:0>2}", .{i}); + config.* = .{ .name = name.*, .kind = .degree, .refresh = .manual }; + read.* = .{ .name = name.* }; + } + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "graph", .{ .metric_configs = &configs }); + defer index.close(); + const ids = try fixture.alloc([]const u8, 100_000); + const nodes = try fixture.alloc(query_mod.GraphResultNode, ids.len); + for (ids, nodes, 0..) |*id, *node, i| { + id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + node.* = .{ .key = id.*, .depth = 0, .distance = 0 }; + } + try index.benchmarkSeedScoreColumns(&names, ids); + const query = query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph", + .start_nodes = .{ .keys = &.{} }, + .metrics = &reads, + .order_by = &.{.{ .name = names[0] }}, + .params = .{ .max_results = 10 }, + }; + const plan = try query_mod.MetricReadPlan.init(query); + const policies: [16]antfly.graph.GraphIndex.GraphMetricColumnReadPolicy = @splat(.{ .require_published = true }); + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + var keys: usize = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const tracked = tracking.allocator(); + const start = antfly.platform_time.monotonicNs(); + { + var session = try index.openGraphMetricReadSessionAlloc(tracked, &names, &policies); + defer session.deinit(); + var work = try query_mod.GraphQueryEngine.MetricStageWorkspace.init(tracked, plan, nodes.len); + defer work.deinit(); + try work.ensure(&session, if (reference) plan.dependencies.slice() else plan.orders.slice(), nodes); + try work.select(query, true, plan.orders.slice(), plan.projections.slice(), &.{}); + try work.ensure(&session, plan.projections.slice(), nodes); + keys = session.reads.keys; + for (work.rows, 0..) |row, i| if (row != nodes.len - i - 1) return error.InvalidBenchmarkResult; + for (work.columns) |column| for (column.?, 0..) |value, i| { + if (value != @as(f64, @floatFromInt(nodes.len - i - 1))) return error.InvalidBenchmarkResult; + }; + } + const elapsed = antfly.platform_time.monotonicNs() - start; + if (stats.current_bytes != 0 or keys != (if (reference) @as(usize, 1_600_000) else 100_150)) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_eager_metric_columns" else "stateful_staged_metric_columns", + .candidates = nodes.len, + .metrics = names.len, + .selected = 10, + .score_keys = keys, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .peak_bytes = last.peak_bytes, + .allocation_count = last.alloc_count, + .note = "real default storage; six warm-cache samples, first discarded; exact selected row and score parity; includes snapshot, score reads, selection and scratch frees; excludes fixture writes, traversal, backend-owned allocations and response encoding", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkQuerySnapshots(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-metric-query-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + const configs = [_]antfly.graph.GraphMetricConfig{.{ .name = "degree", .kind = .degree, .refresh = .manual }}; + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "graph", .{ .metric_configs = &configs }); + defer index.close(); + const ids = try fixture.alloc([]const u8, 4096); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + const writes = try fixture.alloc(antfly.graph.BatchWrite, ids.len * 4); + for (writes, 0..) |*write, i| write.* = .{ .source = ids[i / 4], .target = ids[(i / 4 + i % 4 + 1) % ids.len], .edge_type = "follows" }; + try index.batchApply(writes, &.{}); + var published = try index.runGraphMetric("degree"); + defer published.deinit(alloc); + var started = try index.ensureGraphMetricPlannedBuild("degree", index.edge_generation); + defer started.deinit(alloc); + for (0..8) |_| { + var status = try index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.phase == .scan_edges_and_out_degree) break; + _ = try index.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + _ = try index.runGraphMetricPlannedWorkerPageStepForMetric("degree", "benchmark"); + } + var active = try index.graphMetricStatus("degree"); + defer active.deinit(alloc); + if (active.phase != .scan_edges_and_out_degree) return error.InvalidBenchmarkResult; + for ([_]usize{ 1, 64 }) |rows| for ([_]bool{ true, false }) |reference| { + var times: [21]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..22) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + index.alloc = tracking.allocator(); + defer index.alloc = alloc; + const start = antfly.platform_time.monotonicNs(); + var result = try index.benchmarkScoreSnapshotAlloc("degree", ids[0..rows], reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + for (result.scores) |score| if (score != 8.0) return error.InvalidBenchmarkResult; + result.deinit(index.alloc); + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "query_operator_status_reference" else "query_compact_snapshot", + .rows = rows, + .nodes = ids.len, + .edges = writes.len, + .active_scan_pages = 256, + .median_ns = times[10], + .p95_ns = times[19], + .min_ns = times[0], + .max_ns = times[20], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "real default storage; active rebuild; includes transaction, metadata and scores; validation and result free outside timer", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + }; +} + +fn benchmarkAuthenticatedCache(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + const cache_mod = antfly.serverless.query.cache; + const root = try std.fmt.allocPrint(alloc, "/tmp/antfly-cache-promotion-bench-{d}", .{antfly.platform_time.monotonicNs()}); + defer alloc.free(root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + var cache = try cache_mod.QueryCache.init(alloc, root); + defer cache.deinit(); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + const payload = try alloc.alloc(u8, 64 * 1024); + defer alloc.free(payload); + for (payload, 0..) |*byte, i| byte.* = @truncate(i); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &digest, .{}); + const block_id = "graph-metric-score-0-exact"; + try cache.publishAuthenticatedBlocks(artifact_id, payload.len, checksum, &.{.{ .block_id = block_id, .offset = 0, .contents = payload, .checksum = digest }}, .none); + for ([_]bool{ false, true }) |warm| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + var elapsed: u64 = 0; + for (0..64) |_| { + if (!warm) { + cache.graph_metric_blocks.deinit(); + cache.graph_metric_blocks = .{}; + } + const start = antfly.platform_time.monotonicNs(); + var hit = (try cache.readAuthenticatedBlockIfPresentLease(tracking.allocator(), artifact_id, block_id, payload.len, checksum, &digest, 0, payload.len, .none)).?; + elapsed += antfly.platform_time.monotonicNs() - start; + if (!std.mem.eql(u8, hit.bytes(), payload)) return error.InvalidBenchmarkResult; + hit.deinit(); + } + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (warm) "authenticated_warm_memory_lease" else "authenticated_cold_memory_disk_hit", + .lookups = 64, + .block_bytes = payload.len, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "warm disk in both cases; cold memory includes authentication and promotion; reset excluded; request payload allocations only, cache-owned allocations excluded; no network", + }, .{}); + defer alloc.free(json); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkTopOwnership(out: anytype) !void { + const alloc = std.heap.smp_allocator; + const reader = antfly.serverless.query.graph_metric_reader; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const a = tracking.allocator(); + const scores = try a.alloc(reader.Score, 10_000); + for (scores) |*score| { + const node = try a.alloc(u8, 4096); + @memset(node, 'x'); + score.* = .{ .node_id = node, .value = 1 }; + } + var result = reader.Result{ .scores = scores, .config_fingerprint = 1, .converged = true, .iterations_completed = 1, .delta = 0, .edge_filter = .{}, .metadata_version = 9, .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }; + const resident = stats.current_bytes; + stats = .{ .current_bytes = resident, .peak_bytes = resident }; + var session = antfly.serverless.query.QuerySession{ .alloc = a, .artifacts = undefined, .manifest = undefined }; + const start = antfly.platform_time.monotonicNs(); + const output: []reader.PublicScore = if (reference) blk: { + const cloned = try a.alloc(reader.PublicScore, scores.len); + for (scores, cloned) |score, *copy| copy.* = .{ .node = try a.dupe(u8, score.node_id), .score = score.value }; + break :blk cloned; + } else try result.takePublicScoresAlloc(a, &session); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (output.len != 10_000 or output[0].node.len != 4096 or output[0].score != 1) return error.InvalidBenchmarkResult; + last = stats; + for (output) |*score| score.deinit(a); + a.free(output); + result.deinit(a); + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "top_public_response_copy_reference" else "top_public_response_ownership_transfer", + .nodes = 10_000, + .node_id_bytes = 4096, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "conversion only; peak includes retained input; input construction, result destruction, fetch and JSON encoding excluded", + }, .{}); + defer alloc.free(json); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkMembership(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-membership-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const store_path = try std.fmt.allocPrint(fixture, "{s}/store\x00", .{root}); + const reverse_path = try std.fmt.allocPrint(fixture, "{s}/reverse\x00", .{root}); + var store = try antfly.docstore.DocStore.open(alloc, @ptrCast(store_path.ptr), .{}); + defer store.close(); + var index = try antfly.graph.GraphIndex.open(alloc, &store, @ptrCast(reverse_path.ptr), "graph", .{}); + defer index.close(); + const ids = try fixture.alloc([]const u8, 64); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}); + try index.benchmarkMembershipFixture(ids); + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + index.alloc = tracking.allocator(); + defer index.alloc = alloc; + const start = antfly.platform_time.monotonicNs(); + const count = try index.benchmarkMembershipRead(reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (count != ids.len or stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "membership_partials_reference" else "membership_sealed_blocks", + .nodes = ids.len, + .producer_partials = ids.len * 256, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "real default storage; includes transaction, canonical membership and dictionary validation; excludes fixture writes and numeric fold", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkPublication(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const scores = try fixture.alloc(antfly.graph.GraphIndex.GraphMetricScore, 8192); + for (scores, 0..) |*score, i| score.* = .{ .node = try std.fmt.allocPrint(fixture, "node-{d:0>8}", .{i}), .score = @as(f64, @floatFromInt(i)) / 8192 }; + for ([_]usize{ 64, 4096 }) |limit| { + var times: [5]u64 = undefined; + var commits: usize = 0; + for (0..6) |sample| { + const root = try std.fmt.allocPrint(fixture, "/tmp/antfly-publication-bench-{d}", .{antfly.platform_time.monotonicNs()}); + try std.Io.Dir.cwd().createDirPath(io, root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const path = try std.fmt.allocPrintSentinel(fixture, "{s}/reverse", .{root}, 0); + var index = try antfly.graph.GraphIndex.open(alloc, {}, path, "graph", .{}); + defer index.close(); + const start = antfly.platform_time.monotonicNs(); + commits = try index.benchmarkScorePublication(scores, limit, 1); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (commits != scores.len / limit) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (limit == 64) "publication_64_node_reference" else "publication_bounded_4096_nodes", + .nodes = scores.len, + .checkpoint_commits = commits, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .note = "real default storage; atomic score/staging/cursor commits and full primary-score validation; excludes graph computation, page fencing, and final top-k merge", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkRoutingWorkingSet(out: anytype) !void { + const routing = antfly.serverless.query.graph_metric_routing_cache; + const alloc = std.heap.smp_allocator; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + var fills: usize = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const tracked = tracking.allocator(); + var cache = routing.Cache{}; + const entry_bytes = @sizeOf(routing.Entry) + 4096; + const budget: usize = if (reference) entry_bytes * 64 else 1024 * 1024; + fills = 0; + const start = antfly.platform_time.monotonicNs(); + for (0..100) |_| { + for (0..80) |i| { + var key: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(std.mem.asBytes(&i), &key, .{}); + var lease = cache.acquire(key) orelse blk: { + const entry = try tracked.create(routing.Entry); + entry.* = .{ .key = key, .alloc = tracked, .footer = try tracked.alloc(u8, 4096), .routing = .{ .entries = &.{}, .ranked_entries = &.{}, .footer_offset = 1, .top_score_count = 0 } }; + @memset(entry.footer, @intCast(i)); + fills += 1; + break :blk cache.adopt(entry, budget); + }; + if (lease.entry.footer[0] != i) return error.InvalidBenchmarkResult; + lease.deinit(); + } + } + cache.deinit(); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (fills != (if (reference) @as(usize, 8000) else 80) or stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "routing_64_entry_capacity_model" else "routing_byte_admission", + .queries = 100, + .entries_per_query = 80, + .fills = fills, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "production cache; equal 4 KiB entries model former 64-slot capacity with a byte limit; sequential released leases; excludes codec decoding, object I/O, and query execution", + }, .{}); + defer alloc.free(json); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkOrdinalFold(out: anytype) !void { + const alloc = std.heap.smp_allocator; + const tiles = 4096; + var expected: ?f64 = null; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + const start = antfly.platform_time.monotonicNs(); + const sum = try antfly.graph.GraphIndex.benchmarkOrdinalFold(tracking.allocator(), reference, tiles); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (expected) |value| { + if (sum != value) return error.InvalidBenchmarkResult; + } else expected = sum; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "ordinal_fold_owned_reference" else "ordinal_fold_borrowed_scratch", + .tiles = tiles, + .edge_visits = tiles * 256, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .sum = expected.?, + .note = "warm vector cache; validates and folds the same tile repeatedly; includes constant fixture setup in time but excludes fixture allocations; no storage I/O or checkpoint commit", + }, .{}); + defer alloc.free(json); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkSealedVectors(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + const root = try std.fmt.allocPrint(alloc, "/tmp/antfly-sealed-vector-bench-{d}", .{antfly.platform_time.monotonicNs()}); + defer alloc.free(root); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const primary = try std.fmt.allocPrintSentinel(alloc, "{s}/primary", .{root}, 0); + defer alloc.free(primary); + const reverse = try std.fmt.allocPrintSentinel(alloc, "{s}/reverse", .{root}, 0); + defer alloc.free(reverse); + var store = try antfly.docstore.DocStore.open(alloc, primary, .{}); + defer store.close(); + var index = try antfly.graph.GraphIndex.open(alloc, &store, reverse, "links", .{}); + defer index.close(); + try index.prepareSealedVectorBenchmark(); + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + var reads: usize = 0; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + index.alloc = tracking.allocator(); + defer index.alloc = alloc; + const start = antfly.platform_time.monotonicNs(); + reads = try index.benchmarkSealedVectorGather(256, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(alloc, .{ + .mode = if (reference) "vector_checkpoint_local_reference" else "vector_sealed_cross_checkpoint", + .nodes = 32768, + .gathers = 256 * 2048, + .checkpoints = 256, + .storage_chunks = reads, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .note = "real default storage; uniform-source gathers and new read transactions; sealed cache starts empty; excludes fixture writes and fold/checkpoint commits; tracking excludes backend-owned allocations", + }, .{}); + defer alloc.free(json); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } +} + +fn benchmarkCandidatePlanning(out: anytype) !void { + for ([_]bool{ true, false }) |common_prefix| try benchmarkCandidatePlanningIds(out, common_prefix); +} + +fn benchmarkCandidatePlanningIds(out: anytype, common_prefix: bool) !void { + const alloc = std.heap.smp_allocator; + const reader = antfly.serverless.query.graph_metric_reader; + const codec = antfly.serverless.graph_metric_segment.codec; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const fixture = arena.allocator(); + const count = 100_000; + const canonical = try fixture.alloc([]const u8, count); + for (canonical, 0..) |*id, i| id.* = if (common_prefix) + try std.fmt.allocPrint(fixture, "graph/customer-record-{d:0>8}", .{i}) + else + try std.fmt.allocPrint(fixture, "{x:0>16}", .{std.hash.Wyhash.hash(0, std.mem.asBytes(&i))}); + std.mem.sort([]const u8, canonical, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + const ids = try fixture.alloc([]const u8, count); + for (ids, 0..) |*id, i| id.* = canonical[(i * 7919) % count]; + const entries = try fixture.alloc(codec.RoutingEntry, (count + 255) / 256); + for (entries, 0..) |*entry, i| entry.* = .{ + .block_index = i, + .first_node_id = canonical[i * 256], + .offset = i * 4096, + .len = 4096, + }; + const routing = codec.RoutingIndex{ .entries = entries, .top_score_count = 0, .ranked_entries = &.{}, .footer_offset = entries.len * 4096 }; + for ([_]usize{ 1, 16 }) |columns| { + var expected: ?u64 = null; + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = alloc, .stats = &stats }; + var session = antfly.serverless.query.QuerySession{ .alloc = tracking.allocator(), .artifacts = undefined, .manifest = undefined }; + const start = antfly.platform_time.monotonicNs(); + const sum = try reader.benchmarkCandidatePlanningAlloc(tracking.allocator(), &session, ids, routing, columns, reference); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (expected) |value| { + if (sum != value) return error.InvalidBenchmarkResult; + } else expected = sum; + if (stats.current_bytes != 0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "point_row_maps_reference" else "point_shared_candidate_order", + .rows = count, + .columns = columns, + .blocks = entries.len, + .id_shape = if (common_prefix) "common_prefix" else "hashed_hex", + .node_id_bytes = ids[0].len, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .checksum = expected.?, + .note = "row mapping only; permuted IDs; all legacy column maps retained; excludes output, routing/control ownership, span materialization, fetch and score decoding", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn benchmarkStateful(out: anytype) !void { + for ([_]bool{ false, true }) |duplicates| { + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer arena.deinit(); + const fixture = arena.allocator(); + const count = 20_000; + const nodes = try fixture.alloc([]const u8, count); + for (nodes, 0..) |*node, i| node.* = try std.fmt.allocPrint(fixture, "snapshot/customer-record-{d:0>8}", .{(count - i - 1) / @as(usize, if (duplicates) 2 else 1)}); + const names: []const []const u8 = &.{ "a-rank", "b-rank", "c-rank", "d-rank" }; + var prefixes: [4]?[]const u8 = undefined; + for (names, &prefixes) |name, *prefix| { + var key = std.ArrayListUnmanaged(u8).empty; + try key.appendSlice(fixture, "meta:metric:"); + for ([_][]const u8{ name, "score", "12345" }) |part| try antfly.internal_keys.appendEncodedComponent(&key, fixture, part); + prefix.* = try key.toOwnedSlice(fixture); + } + var columns: [4][]?f64 = undefined; + for (&columns) |*column| column.* = try fixture.alloc(?f64, count); + for ([_]bool{ true, false }) |reference| { + var times: [5]u64 = undefined; + var last = PhaseAllocStats{}; + var last_txn = ScoreTxn{}; + // One warmup plus five measured runs. Storage is a synchronous + // in-memory sink; all output cells are verified outside timing. + for (0..6) |sample| { + var stats = PhaseAllocStats{}; + var tracking = PhaseTrackingAllocator{ .backing = std.heap.smp_allocator, .stats = &stats }; + var txn = ScoreTxn{}; + const start = antfly.platform_time.monotonicNs(); + if (reference) try referenceScores(tracking.allocator(), &txn, names, nodes, &columns) else _ = try antfly.graph.score_read.populate(tracking.allocator(), &txn, &prefixes, nodes, &columns); + const elapsed = antfly.platform_time.monotonicNs() - start; + if (stats.current_bytes != 0 or txn.key_hash == 0) return error.InvalidBenchmarkResult; + for (columns) |column| for (column) |score| if (score != 1.0) return error.InvalidBenchmarkResult; + if (sample != 0) times[sample - 1] = elapsed; + last = stats; + last_txn = txn; + } + std.mem.sort(u64, ×, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(fixture, .{ + .mode = if (reference) "stateful_arena_reference" else "stateful_physical_slab", + .rows = count, + .columns = names.len, + .duplicates = duplicates, + .storage_keys = last_txn.key_count, + .median_ns = times[2], + .min_ns = times[0], + .max_ns = times[4], + .allocation_count = last.alloc_count, + .allocated_bytes = last.total_alloc_bytes, + .peak_bytes = last.peak_bytes, + .samples = times.len, + .note = "reader only; mock storage; excludes output arrays and input fixture; one warmup", + }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} diff --git a/zig/bench/graph/paged_read_bench.zig b/zig/bench/graph/paged_read_bench.zig new file mode 100644 index 0000000000..beaba04ce3 --- /dev/null +++ b/zig/bench/graph/paged_read_bench.zig @@ -0,0 +1,145 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const std = @import("std"); +const antfly = @import("antfly_zig"); +const graph = antfly.serverless.graph_segment; +const artifacts = antfly.serverless.artifacts; +const Allocator = std.mem.Allocator; + +const Memory = struct { + payload: []const u8, + calls: usize = 0, + bytes: usize = 0, + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(_: *anyopaque, _: Allocator, _: []const u8) !artifacts.ArtifactMetadata { + return error.Unsupported; + } + fn get(ptr: *anyopaque, alloc: Allocator, _: []const u8) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.calls += 1; + self.bytes += self.payload.len; + return alloc.dupe(u8, self.payload); + } + fn range(ptr: *anyopaque, alloc: Allocator, _: []const u8, offset: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.calls += 1; + self.bytes += len; + return alloc.dupe(u8, self.payload[@intCast(offset)..][0..len]); + } + fn stat(_: *anyopaque, _: Allocator, _: []const u8) !artifacts.ArtifactMetadata { + return error.Unsupported; + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.Unsupported; + } + const vtable = artifacts.ArtifactStore.VTable{ .deinit = deinit, .put = put, .get_alloc = get, .get_range_alloc = range, .stat = stat, .delete = delete }; +}; + +pub fn main(init: std.process.Init) !void { + var buffer: [4096]u8 = undefined; + var output = std.Io.File.stdout().writer(init.io, &buffer); + try run(init.io, &output); +} + +pub fn run(io: std.Io, out: anytype) !void { + const alloc = std.heap.smp_allocator; + for ([_]usize{ 64, 1024, 10000 }) |count| { + var fixture = std.heap.ArenaAllocator.init(alloc); + defer fixture.deinit(); + const a = fixture.allocator(); + var builder = graph.Builder{ .alloc = a }; + defer builder.deinit(); + for (0..count) |i| try builder.addEdge("a", "b", try std.fmt.allocPrint(a, "kind{d:0>5}", .{i}), 1, null); + const payload = try builder.encodeAlloc(256 * 1024 * 1024, .none); + const checksum = try digestAlloc(a, payload); + var source = antfly.serverless.ArtifactRef{ .kind = .graph_segment, .name = "g", .artifact_id = try std.fmt.allocPrint(a, "sha256:{s}", .{checksum}), .checksum = checksum, .byte_len = payload.len }; + try graph.codec.compact.bindTopologyControl(&source, payload); + var memory = Memory{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &Memory.vtable }; + var samples: [5]u64 = undefined; + for (0..6) |sample| { + memory.calls = 0; + memory.bytes = 0; + const start = std.Io.Clock.awake.now(io); + const result = try antfly.serverless.build.lake_graph_metric.benchmarkSelectedArtifactPreparation(alloc, &store, source, .{ .name = "degree", .kind = .degree }, false); + if (result.edges != count) return error.InvalidBenchmarkResult; + const elapsed: u64 = @intCast(start.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds()); + if (sample != 0) samples[sample - 1] = elapsed; + } + if (memory.bytes > payload.len or memory.calls > 8) return error.RangeAmplificationRegression; + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(a, .{ .mode = "many_type_topology", .types = count, .artifact_bytes = payload.len, .range_calls = memory.calls, .read_bytes = memory.bytes, .median_ns = samples[2] }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + + for ([_]usize{ 16384, 100000 }) |count| { + var fixture = std.heap.ArenaAllocator.init(alloc); + defer fixture.deinit(); + const a = fixture.allocator(); + const ids = try a.alloc([]const u8, count); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(a, "collection/customer-record-{d:0>8}", .{i}); + var builder = graph.Builder{ .alloc = a }; + defer builder.deinit(); + for (ids, 0..) |id, i| try builder.addEdge(id, ids[(i + 1) % count], "link", 1, null); + const payload = try builder.encodeAlloc(256 * 1024 * 1024, .none); + const checksum = try digestAlloc(a, payload); + var source = antfly.serverless.ArtifactRef{ .kind = .graph_segment, .name = "g", .artifact_id = try std.fmt.allocPrint(a, "sha256:{s}", .{checksum}), .checksum = checksum, .byte_len = payload.len }; + try graph.codec.compact.bindTopologyControl(&source, payload); + for ([_]bool{ true, false }) |reference| { + var memory = Memory{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &Memory.vtable }; + var samples: [5]u64 = undefined; + for (0..6) |sample| { + memory.calls = 0; + memory.bytes = 0; + const start = std.Io.Clock.awake.now(io); + if (reference) { + const bytes = try store.getAlloc(source.artifact_id); + defer alloc.free(bytes); + var segment = try graph.decodeAlloc(alloc, bytes); + defer segment.deinit(alloc); + var index = try graph.AdjacencyIndex.init(alloc, segment); + defer index.deinit(alloc); + const row = index.find(segment, ids[count / 2]).?; + if (row.out_edges.len != 1 or !std.mem.eql(u8, row.out_edges[0].neighbor_id, ids[count / 2 + 1])) return error.InvalidBenchmarkResult; + } else { + var remaining: u64 = 512 * 1024 * 1024; + var reader = (try graph.AdjacencyReader.init(alloc, &store, source, .none, &remaining)).?; + defer reader.deinit(); + var work: usize = 100; + var row = (try reader.adjacency(ids[count / 2], &.{}, enum { out, in, both }.out, 1, &work)).?; + defer row.deinit(alloc); + if (row.out_edges.len != 1 or !std.mem.eql(u8, row.out_edges[0].neighbor_id, ids[count / 2 + 1])) return error.InvalidBenchmarkResult; + } + const elapsed: u64 = @intCast(start.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds()); + if (sample != 0) samples[sample - 1] = elapsed; + } + if (!reference and memory.bytes >= payload.len) return error.RangeAmplificationRegression; + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + const json = try std.json.Stringify.valueAlloc(a, .{ .mode = if (reference) "whole_graph_adjacency" else "paged_graph_adjacency", .nodes = count, .artifact_bytes = payload.len, .range_calls = memory.calls, .read_bytes = memory.bytes, .median_ns = samples[2], .note = "fresh reader each sample; in-memory transport counts exact bytes; includes decoding/authentication and cleanup; no network latency model; full reference omits transport SHA verification" }, .{}); + try out.interface.writeAll(json); + try out.interface.writeByte('\n'); + try out.flush(); + } + } +} + +fn digestAlloc(alloc: Allocator, payload: []const u8) ![]u8 { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &digest, .{}); + return alloc.dupe(u8, &std.fmt.bytesToHex(digest, .lower)); +} diff --git a/zig/build.zig b/zig/build.zig index 220785ff9c..2709e49431 100644 --- a/zig/build.zig +++ b/zig/build.zig @@ -37,6 +37,7 @@ const selectTestFilters = antfly_tests_build.selectTestFilters; const RuntimeArtifactRole = enum { cli, data, + graph_metric_maintenance, inference, metadata, standalone, @@ -1478,7 +1479,7 @@ pub fn build(b: *std.Build) void { const with_tla = b.option(bool, "with_tla", "Enable TLA+ trace instrumentation (ndjson event logging)") orelse false; const link_libc = b.option(bool, "link-libc", "Link Antfly runtime modules against libc") orelse true; const sanitize_thread = b.option(bool, "sanitize-thread", "Enable ThreadSanitizer for the Antfly runtime") orelse false; - const runtime_artifact_role = b.option(RuntimeArtifactRole, "runtime-artifact-role", "Build one focused runtime artifact: cli, data, inference, metadata, or standalone"); + const runtime_artifact_role = b.option(RuntimeArtifactRole, "runtime-artifact-role", "Build one focused runtime artifact: cli, data, graph_metric_maintenance, inference, metadata, or standalone"); const antfly_bin_name = b.option([]const u8, "antfly-bin-name", "Installed filename for the top-level Antfly CLI") orelse "antfly"; if (antfly_bin_name.len == 0 or std.mem.indexOfAny(u8, antfly_bin_name, "/\\") != null) { @panic("-Dantfly-bin-name must be a non-empty filename, not a path"); @@ -2253,6 +2254,13 @@ pub fn build(b: *std.Build) void { }); antfly_imports.configure(b, api_http_runtime_test_mod, true, true); + const api_graph_metric_test_mod = b.createModule(.{ + .root_source_file = b.path("pkg/antfly/src/api_graph_metric_test_root.zig"), + .target = target, + .optimize = optimize, + }); + antfly_imports.configure(b, api_graph_metric_test_mod, true, true); + const metadata_unit_baseline_root_paths = [_][]const u8{ "pkg/antfly/src/metadata_reconciler_test_root.zig", "pkg/antfly/src/metadata_service_http_test_root.zig", @@ -3953,6 +3961,7 @@ pub fn build(b: *std.Build) void { "public index config encoders retain credential-free provider urls", "public index config encoders omit root write-only producer documents", "created graph index response projects closed nested schemas", + "created graph metric configuration projects closed nested schemas", "enrichment index status encodes worker lifecycle diagnostics", "compact index repair status keeps corrupt terminal state actionable", "data runtime report preserves compact managed repair admission state", @@ -4023,6 +4032,7 @@ pub fn build(b: *std.Build) void { "table contract rejects graph configs the runtime cannot materialize", "table graph validation rejects runtime-invalid configs before catalog admission", "table contract preserves typed artifact-backed graph configuration", + "table contract preserves graph metric configuration and rejects malformed nested values", "table contract rejects unknown fields in closed nested index objects", "table contract treats nullable nested index fields as omitted", "table contract preserves artifact-backed public full text indexes", @@ -4353,6 +4363,12 @@ pub fn build(b: *std.Build) void { }); const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests); addRuntimeTestFilters(b, run_lib_unit_tests, lib_unit_filters); + // The broad root discovery anchor may retain API tests after a merge adds + // imports. Keep this stateful error-path test in its dedicated API shards. + run_lib_unit_tests.addArgs(&.{ + "--skip-test-filter", + "cluster backup retains its fenced attempt after an ambiguous table outcome", + }); for (root_test_skip_filters) |filter| { run_lib_unit_tests.addArgs(&.{ "--skip-test-filter", filter }); } @@ -4457,6 +4473,7 @@ pub fn build(b: *std.Build) void { "local inference connection ABI retains C layout and validates capabilities", "local inference response validation contains malformed ownership", "inference connection invocation requires inference write permission", + "graph metric operational actions require table admin permission", "httpx inference connection preserves upstream retry guidance", "api http client preserves exact-group join unavailability and absence", "distributed join translates native and borrowed deadline boundaries", @@ -4464,6 +4481,15 @@ pub fn build(b: *std.Build) void { "distributed graph translates native worker and catalog deadline boundaries", "query embedding cache translates native query deadlines", "typed internal HTTP errors preserve conflict semantics", + "api http index generation retry refreshes once and preserves readiness cancellation and deadlines", + "api http retries identity generation and topology churn from a fresh query snapshot", + "derived enrichment visibility guard observes cancellation and deadline", + "db reverse graph probe rejects a deleted or replaced index incarnation", + "api http client preserves group doc identity conflicts", + "typed internal group reads preserve retryable resident storage failures", + "boundary dispatcher preserves local calls and maps cross-unit calls", + "stable status preserves public boundary semantics", + "db graph search filters result nodes and hidden traversal intermediates", "internal transaction HTTP responses prove not-proposed only before decision", "internal transaction ingress establishes and validates pre-decision deadline", "request admission bounds positive capacity and preserves unlimited mode", @@ -4478,6 +4504,10 @@ pub fn build(b: *std.Build) void { const api_http_runtime_tests = b.addTest(.{ .root_module = api_http_runtime_test_mod, .filters = api_http_runtime_filters, + // The native-generation merge raised this linked API/DB harness to + // 16.01 GB in macOS ReleaseFast codegen. Reserve measured usage plus + // headroom; the shared runner still caps aggregate compilation. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 17 else 7) * 1024 * 1024 * 1024, .test_runner = .{ .path = b.path("pkg/antfly/src/test_runner.zig"), .mode = .simple, @@ -4566,6 +4596,31 @@ pub fn build(b: *std.Build) void { const cmd_test_step = b.step("cmd-test", "Run Antfly command and client CLI tests"); cmd_test_step.dependOn(&run_cmd_tests.step); + const graph_metric_command_test_mod = b.createModule(.{ + .root_source_file = b.path("pkg/antfly/src/cmd_graph_metric_maintenance_test_root.zig"), + .target = target, + .optimize = optimize, + }); + antfly_imports.configure(b, graph_metric_command_test_mod, true, true); + graph_metric_command_test_mod.addImport("antfly-zig", antfly_mod); + graph_metric_command_test_mod.addImport("antfly-client", antfly_client_pkg_mod); + const graph_metric_command_tests = b.addTest(.{ + .root_module = graph_metric_command_test_mod, + .filters = &.{"cmd.graph_metric_maintenance.test.graph metric maintenance"}, + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + // This root includes the in-process HTTP service and storage runtime. + // Mach-O optimized codegen measured 8.4 GiB; allow the same debug + // headroom as the broad command root without raising Linux admission. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 13 else 7) * 1024 * 1024 * 1024, + }); + const run_graph_metric_command_tests = addFilteredTestRunArtifact(b, graph_metric_command_tests); + const graph_metric_command_test_step = b.step("graph-metric-command-test", "Run graph metric maintenance CLI and supervisor tests"); + graph_metric_command_test_step.dependOn(&run_graph_metric_command_tests.step); + cmd_test_step.dependOn(&run_graph_metric_command_tests.step); + const lite_cmd_test_mod = b.createModule(.{ .root_source_file = b.path("pkg/antfly/src/lite_cmd_test.zig"), .target = target, @@ -4902,6 +4957,9 @@ pub fn build(b: *std.Build) void { const serverless_default_filters = [_][]const u8{"serverless"}; const serverless_tests = b.addTest(.{ + // macOS ReleaseFast measured 10.74 GB for this root. Reserve realistic + // compiler headroom for aggregate scheduling; Linux CI stays bounded. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 11 else 7) * 1024 * 1024 * 1024, .root_module = antfly_test_mod, .filters = &serverless_default_filters, .test_runner = .{ @@ -10172,6 +10230,282 @@ pub fn build(b: *std.Build) void { vopr_test_step.dependOn(storage_vopr_step); chaos_test_step.dependOn(storage_vopr_step); + const graph_metric_unit_filters = [_][]const u8{ + "graph metric sorted batch presence", + "db reverse graph probe rejects a deleted or replaced index incarnation", + "graph pagerank planned scan page writes durable out-degree intermediates", + "graph pagerank contribution and reduce pages resume", + "graph pagerank later iteration pages resume", + "graph pagerank convergence page reclaim", + "graph eigenvector contribution and reduce pages resume", + "graph eigenvector convergence page reclaim", + "graph hits contribution and reduce pages resume", + "graph hits hub contribution and hub reduce pages resume", + "graph hits convergence page reclaim", + "graph metric runtime config rejects", + "graph metric runtime role gates apply", + "graph metric runtime worker pool identity", + "graph metric runtime boundary tick", + "graph metric runtime retirement", + "ownership state tracks lease takeover and loss", + "ownership state renews only at the cached renewal deadline", + "lease release preserves tenure fencing across owner ID reuse", + "graph metric query shape bounds clauses and unique dependencies", + "graph metric staged", + "borrowed graph metric names do not allocate per node", + "graph metric column selection retains deterministic bounded top k", + "graph metric shared column application is allocation-failure safe", + "graph metric stable row materialization moves nodes once and is allocation-failure safe", + "graph metric order and filter dependencies attach status without projection", + "graph metric order and filter apply max results after metric processing", + "shortest path metric filtering evaluates the complete bounded candidate set", + "pattern metric filtering evaluates matches beyond the response limit", + "graph both direction emits one physical self loop and preserves reciprocal edges", + "graph durable writes reject invalid edge types before mutation", + "graph bounded adjacency pages preserve order and fail before budget overflow", + "graph edge encoding round-trip", + "graph metric reverse edge parser borrows ordinary keys and owns escaped components", + "graph storage rejects non-finite edge weights", + "graph metric metadata preserves score epoch input and decodes v3", + "graph metric edge filter equality and fingerprint treat types as set", + "graph metric rebuild at unchanged edge generation publishes an isolated score epoch", + "graph metric native rank index retains only the supported top-k prefix", + "graph degree scan attempt adoption resumes in bounded pages", + "graph degree scan page reclaim recomputes without double counting partials", + "graph degree large-build summary counts filtered materialization without coordinator scan", + "graph metric large-build summary", + "graph metric vector chunks", + "graph metric ordinal", + "graph metric membership", + "graph metric shared topology", + "topology receipts", + "graph metric edge scan", + "graph metric consumer barrier", + "ordinal blocks", + "graph degree planned build honors edge filter during scan page execution", + "graph metric filtered", + "graph metric coalesced global counters", + "graph metric partition spans remain balanced at production cardinality", + "graph metric partition census", + "partition census owns bounded checkpoints", + "runtime store erases concrete single-namespace store handles", + "failed commit keeps erased write handle abortable", + "graph metric floating page aggregates are deterministic across adoption order", + "graph metric column snapshots preserve order across chunks and reject stale reads before scores", + "graph metric physical score reads", + "graph metric status exposes queued and active local build lease", + "graph metric coordinator reports expired exhausted page lease", + "graph planned metric build retires a superseded generation without poisoning newer work", + "graph pagerank planned build publishes scores matching local runner", + "graph pagerank warm rebuild normalizes changed node sets across summary pages", + "graph metric execution epoch fences old jobs without hiding published scores", + "graph pagerank reclaimed contribution and reduce pages overwrite partial output", + "graph pagerank scan adoption maintains one idempotent out-degree total", + "graph eigenvector reclaimed contribution and reduce pages overwrite stale output", + "graph hits reclaimed contribution and reduce pages overwrite stale output", + "graph hits planned build drains partitioned paired pages across workers", + "graph pagerank coordinator publish failure preserves prior published generation after reopen", + "graph pagerank exhausted publish page preserves root cause and prior generation", + "graph hits coordinator publish failure preserves prior published pair after reopen", + }; + const graph_metric_unit_tests = b.addTest(.{ + // The merged storage root measured 7.61 GB on macOS ReleaseFast. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 8 else 7) * 1024 * 1024 * 1024, + .root_module = db_test_mod, + .filters = compileFiltersWithAnchors( + b, + &.{"db default primary backend survives reopen"}, + &graph_metric_unit_filters, + ), + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_metric_unit_tests = addFilteredTestRunArtifactWithRuntimeFilters( + b, + graph_metric_unit_tests, + &graph_metric_unit_filters, + ); + const graph_metric_unit_test_step = b.step("graph-metric-unit-test", "Run cheap graph metric runtime, ownership, and query tests"); + graph_metric_unit_test_step.dependOn(&run_graph_metric_unit_tests.step); + b.step("graph-metric-core-test", "Run graph metric core unit tests without rebuilding API/wire harnesses").dependOn(&run_graph_metric_unit_tests.step); + const graph_metric_topology_filters = [_][]const u8{"graph metric shared topology"}; + const graph_metric_topology_tests = b.addTest(.{ + .root_module = db_test_mod, + .filters = compileFiltersWithAnchors(b, &.{"db default primary backend survives reopen"}, &graph_metric_topology_filters), + .test_runner = .{ .path = b.path("pkg/antfly/src/test_runner.zig"), .mode = .simple }, + }); + const run_graph_metric_topology_tests = addFilteredTestRunArtifactWithRuntimeFilters(b, graph_metric_topology_tests, &graph_metric_topology_filters); + const graph_metric_topology_test_step = b.step("graph-metric-topology-test", "Run durable topology preparation, sharing, recovery, and reclamation tests"); + graph_metric_topology_test_step.dependOn(&run_graph_metric_topology_tests.step); + unit_test_step.dependOn(&run_graph_metric_unit_tests.step); + unit_test_step.dependOn(&run_graph_metric_command_tests.step); + + const graph_metric_fan_in_filters = [_][]const u8{ + "query parser accepts direct graph metric reads", + "query parser accepts graph metric rerank", + "api query contract bounds graph metric top k", + "api query contract uses portable graph metric filter operators", + "api query contract rejects oversized and duplicate graph metric clauses", + "query encoder emits graph metric results", + "query profile reports failed graph metric status across read surfaces", + "query encoder emits graph metric rerank score details", + "query merge applies deterministic graph metric top-k across shards", + "query merge rejects missing or unpublished graph metric shard results", + "query merge rejects duplicate direct graph metric score nodes", + "query merge rejects non-finite direct graph metric scores", + "query merge rejects duplicate direct graph metric shard results", + "query merge rejects mismatched direct graph metric shard identity", + "query merge rejects inconsistent graph metric fan-in status state", + "query merge rejects non-finite graph metric fan-in status numbers", + "query merge rejects out-of-range graph metric fan-in progress", + "query merge rejects incompatible graph metric fan-in metadata", + "query merge rejects unsolicited graph score surfaces", + "query merge rejects unsolicited graph search metric status", + "query merge validates included graph search metric status list", + "query merge rejects malformed graph search metric payloads", + "query merge rejects malformed graph search traversal payloads", + "query merge rejects unqualified graph search identity collisions without collapsing qualified identities", + "query merge rejects malformed graph search hit payloads", + "query merge preserves failed graph metric status across shard fan-in", + "query merge requires comparable graph search metric generations across shards", + "query merge allows unpublished projected graph search metric status", + "query merge rejects ambiguous graph search fan-in metric status", + "query merge preserves failed graph search metric status across shards", + "query merge enforces graph search order and filter metric generations across shards", + "query profile reports merged graph search metric generation", + "query merge requires comparable graph metric rerank generations across shards", + "query merge rejects malformed graph metric rerank score details", + "query merge rejects missing or unpublished graph metric rerank shard status", + "distributed graph result accounting includes shared metric storage and status details", + "distributed graph expand request bounds deferred worker metric candidates", + "distributed graph metric status merge validates metadata compatibility", + "distributed graph metric post processing applies max results after filter and order", + "public index contract exposes runtime status metadata", + "indexes openapi parses graph metric runtime summary", + "client openapi parses graph metric runtime summary", + "metadata openapi module generates extractor surface for routed endpoints", + "index encoders expose graph metric runtime ownership summary", + "index encoders expose mixed graph metric runtime roles without aggregate role", + "graph metric status encoder exposes active build pages", + "public table graph metric action handler returns status response", + "db query result shape executeSingleNonPatternQueryWithSets hides metric status unless requested", + "graph metric status clone owns active build worker id", + "graph metric index stats cleanup owns nested status payloads", + "graph metric cached index stats clone retains owned progress and survives allocation failures", + }; + const graph_metric_fan_in_tests = b.addTest(.{ + .root_module = api_graph_metric_test_mod, + .filters = &graph_metric_fan_in_filters, + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_metric_fan_in_tests = addFilteredTestRunArtifact(b, graph_metric_fan_in_tests); + const graph_metric_fan_in_test_step = b.step("graph-metric-fan-in-test", "Run graph metric API contract and fail-closed distributed fan-in tests"); + graph_metric_fan_in_test_step.dependOn(&run_graph_metric_fan_in_tests.step); + graph_metric_unit_test_step.dependOn(&run_graph_metric_fan_in_tests.step); + unit_test_step.dependOn(&run_graph_metric_fan_in_tests.step); + + const graph_metric_remote_wire_filters = [_][]const u8{ + "api http client authenticates only the internal API namespace", + "multi-shard reads fail closed for shard-local graph metric scores", + "graph metric shard request carries internal status without mutating public request", + "encode query request includes graph metric read rerank and traversal status", + "remote query parser preserves graph metric fan-in provenance and durable status", + "remote query parser rejects invalid graph metric status and duplicate rerank profiles", + "graph metric queries use general table read preparation and search path", + "hosted cross-range graph metric fan-in merges compatible published shard generations", + "hosted cross-range graph metric fan-in merges active stale shard for published", + "hosted cross-range graph metric fan-in merges nonuniform promotion shard layout", + "hosted cross-range graph metric fan-in merges compatible hits pair", + "hosted cross-range graph metric fan-in rejects incompatible remote hits pair", + "hosted cross-range graph metric fan-in rejects missing remote hits status", + "hosted cross-range graph metric fan-in rejects unpublished or incompatible shard generations", + }; + const graph_metric_remote_wire_tests = b.addTest(.{ + // macOS ReleaseFast measured 13.45 GB after the native-generation + // merge. Admit this indivisible compiler job with headroom; the shared + // runner's 22 GiB cap still bounds aggregate concurrent compilation. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 14 else 7) * 1024 * 1024 * 1024, + .root_module = api_table_reads_docid_test_mod, + .filters = &graph_metric_remote_wire_filters, + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_metric_remote_wire_tests = addFilteredTestRunArtifact(b, graph_metric_remote_wire_tests); + const graph_metric_remote_wire_test_step = b.step("graph-metric-remote-wire-test", "Run hosted graph metric shard request and remote-wire tests"); + graph_metric_remote_wire_test_step.dependOn(&run_graph_metric_remote_wire_tests.step); + graph_metric_unit_test_step.dependOn(&run_graph_metric_remote_wire_tests.step); + unit_test_step.dependOn(&run_graph_metric_remote_wire_tests.step); + + const graph_metric_smoke_filters = [_][]const u8{ + "db graph metric runtime background coordinator and worker pool loops publish pagerank", + }; + const graph_metric_smoke_tests = b.addTest(.{ + .root_module = db_test_mod, + .filters = compileFiltersWithAnchors( + b, + &.{"db default primary backend survives reopen"}, + &graph_metric_smoke_filters, + ), + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_metric_smoke_tests = addFilteredTestRunArtifactWithRuntimeFilters( + b, + graph_metric_smoke_tests, + &graph_metric_smoke_filters, + ); + const graph_metric_smoke_test_step = b.step("graph-metric-smoke-test", "Run a small end-to-end PageRank scheduler and worker smoke test"); + graph_metric_smoke_test_step.dependOn(&run_graph_metric_smoke_tests.step); + unit_test_step.dependOn(&run_graph_metric_smoke_tests.step); + + const graph_metric_integration_filters = [_][]const u8{ + "graph.graph.test.graph pagerank ", + "graph.graph.test.graph eigenvector ", + "graph.graph.test.graph hits ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime background ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime planned ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime query ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime role ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime lease ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime operations ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime degree canary ", + "storage.db.maintenance.graph_metric_runtime.test.db graph metric runtime default gate ", + "graph metric failed planned build", + "graph metric repeated failed", + "graph metric build job cleanup", + }; + const graph_metric_integration_tests = b.addTest(.{ + // macOS ReleaseFast measured 7.74 GB for the lifecycle root. + .max_rss = @as(usize, if (target.result.os.tag == .macos) 10 else 7) * 1024 * 1024 * 1024, + .root_module = db_test_mod, + .filters = compileFiltersWithAnchors( + b, + &.{"db default primary backend survives reopen"}, + &graph_metric_integration_filters, + ), + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_metric_integration_tests = addFilteredTestRunArtifactWithRuntimeFilters( + b, + graph_metric_integration_tests, + &graph_metric_integration_filters, + ); + const graph_metric_integration_test_step = b.step("graph-metric-integration-test", "Run graph metric scheduler, worker, query, cleanup, and runtime lifecycle tests"); + graph_metric_integration_test_step.dependOn(&run_graph_metric_integration_tests.step); + integration_test_step.dependOn(&run_graph_metric_integration_tests.step); + const db_unit_tests = b.addTest(.{ .root_module = db_test_mod, .filters = selectTestFilters(b, &.{}), @@ -10187,6 +10521,25 @@ pub fn build(b: *std.Build) void { const db_test_step = b.step("antfly-storage-db-test", "Run storage/db tests"); db_test_step.dependOn(&run_db_unit_tests.step); + const graph_runtime_filters = [_][]const u8{ + "storage.db.graph_runtime.test.", + }; + const graph_runtime_tests = b.addTest(.{ + .root_module = db_test_mod, + .filters = &graph_runtime_filters, + .test_runner = .{ + .path = b.path("pkg/antfly/src/test_runner.zig"), + .mode = .simple, + }, + }); + const run_graph_runtime_tests = addFilteredTestRunArtifactWithRuntimeFilters( + b, + graph_runtime_tests, + &graph_runtime_filters, + ); + const graph_runtime_test_step = b.step("graph-runtime-test", "Run graph artifact replay, repair, and traversal integration tests"); + graph_runtime_test_step.dependOn(&run_graph_runtime_tests.step); + // Keep the small, deterministic release-blocker primitives in the PR/base // unit gate. The corpus-scale fixtures below protect thresholds that only // appear at thousands of documents and run in the zig-full gate instead. @@ -10411,6 +10764,7 @@ pub fn build(b: *std.Build) void { "storage.db.document_mapper.", "storage.db.document_query.", "storage.db.generation_lifecycle.", + "storage.db.graph_runtime.", "storage.db.graph_asset_state.", "storage.db.graph_edge_contender.", "storage.db.graph_state_name.", @@ -11149,6 +11503,20 @@ pub fn build(b: *std.Build) void { const backend_bench_step = b.step("backend-bench", "Build and install backend_bench"); backend_bench_step.dependOn(&b.addInstallArtifact(backend_bench, .{}).step); + const graph_metric_prepare_bench_mod = b.createModule(.{ + .root_source_file = b.path("bench/graph/metric_preparation_bench.zig"), + .target = target, + .optimize = .ReleaseFast, + }); + graph_metric_prepare_bench_mod.addImport("antfly_zig", antfly_mod); + const graph_metric_prepare_bench = b.addExecutable(.{ + .name = "graph_metric_preparation_bench", + .root_module = graph_metric_prepare_bench_mod, + }); + const run_graph_metric_prepare_bench = b.addRunArtifact(graph_metric_prepare_bench); + if (b.args) |args| run_graph_metric_prepare_bench.addArgs(args); + b.step("graph-metric-preparation-bench", "Compare unpack/hash and packed ordinal graph-metric preparation").dependOn(&run_graph_metric_prepare_bench.step); + const graph_pattern_bench_mod = b.createModule(.{ .root_source_file = b.path("bench/graph/pattern_query_bench.zig"), .target = target, @@ -12193,11 +12561,12 @@ pub fn build(b: *std.Build) void { // 19.51 GB (18.17 GiB) with the platform frameworks enabled. // A clean native aarch64-linux-musl production container build // reached 19.89 GB (18.52 GiB) for the current production - // graph. Reserve 20 GiB on both targets so Zig's scheduler does + // graph. The merged graph/inference work reached 21.59 GB on + // aarch64-macOS. Reserve 22 GiB on both targets so Zig's scheduler does // not discard a successfully compiled production artifact. // Use the same Linux-target claim for native and cross builds; // the target artifact determines the dominant codegen shape. - .distributed => 20 * 1024 * 1024 * 1024, + .distributed => 22 * 1024 * 1024 * 1024, // This is deliberately a separate non-PIC product unit. The // cold aarch64-macOS ReleaseFast build peaks near 2 GiB; // the 10 GiB reservation keeps it serialized with the macOS @@ -12299,7 +12668,7 @@ pub fn build(b: *std.Build) void { role_exe.root_module.linkLibrary(runtime_library_artifacts[@intFromEnum(RuntimeLibraryUnit.cli)].?); role_exe.root_module.linkLibrary(runtime_library_artifacts[@intFromEnum(RuntimeLibraryUnit.distributed)].?); }, - .data, .metadata => { + .data, .graph_metric_maintenance, .metadata => { role_exe.root_module.linkLibrary(runtime_library_artifacts[@intFromEnum(RuntimeLibraryUnit.distributed)].?); role_exe.root_module.linkLibrary(runtime_library_artifacts[@intFromEnum(RuntimeLibraryUnit.api_kernel)].?); }, @@ -12340,6 +12709,30 @@ pub fn build(b: *std.Build) void { antfly_main_test_step.dependOn(&run_antfly_main_tests.step); unit_test_step.dependOn(&run_antfly_main_tests.step); + const graph_metric_process_harness_mod = b.createModule(.{ + .root_source_file = b.path("pkg/antfly/src/cmd/graph_metric_process_harness.zig"), + .target = target, + .optimize = optimize, + }); + graph_metric_process_harness_mod.addImport("antfly-zig", antfly_mod); + graph_metric_process_harness_mod.addImport("antfly_platform", platform_mod); + graph_metric_process_harness_mod.addImport("httpx", httpx_mod); + const graph_metric_process_harness = b.addExecutable(.{ + .name = "graph-metric-process-harness", + .root_module = graph_metric_process_harness_mod, + }); + graph_metric_process_harness.root_module.linkLibrary( + runtime_library_artifacts[@intFromEnum(RuntimeLibraryUnit.api_kernel)].?, + ); + graph_metric_process_harness.step.dependOn(&antfly_main.step); + const run_graph_metric_process_harness = b.addRunArtifact(graph_metric_process_harness); + run_graph_metric_process_harness.addArtifactArg(antfly_main); + run_graph_metric_process_harness.addArgs(&.{ "--profile", "promotion" }); + run_graph_metric_process_harness.has_side_effects = true; + const graph_metric_process_test_step = b.step("graph-metric-process-test", "Run process-level graph metric promotion and failover gates"); + graph_metric_process_test_step.dependOn(&run_graph_metric_process_harness.step); + integration_test_step.dependOn(&run_graph_metric_process_harness.step); + // The aggregate intentionally runs with normal CPU concurrency. Give every // compile step a conservative scheduler claim unless it already has a // measured, domain-specific claim above; CI supplies the cgroup-aware diff --git a/zig/build_test_filters.zig b/zig/build_test_filters.zig index 035d27a42f..680d3dc991 100644 --- a/zig/build_test_filters.zig +++ b/zig/build_test_filters.zig @@ -18,6 +18,13 @@ fn validateSkipTestFilter(value: []const u8) error{EmptySkipTestFilter}!void { if (value.len == 0) return error.EmptySkipTestFilter; } +fn validateExpectedErrorLogs(filter: []const u8, count_value: []const u8) error{ InvalidExpectedErrorLogFilter, InvalidExpectedErrorLogCount }!void { + if (filter.len == 0) return error.InvalidExpectedErrorLogFilter; + const count = std.fmt.parseUnsigned(usize, count_value, 10) catch + return error.InvalidExpectedErrorLogCount; + if (count == 0) return error.InvalidExpectedErrorLogCount; +} + fn isTestControl(arg: []const u8) bool { return std.mem.eql(u8, arg, "--list-tests") or std.mem.eql(u8, arg, "--allow-empty-test-filter") or @@ -25,6 +32,7 @@ fn isTestControl(arg: []const u8) bool { std.mem.startsWith(u8, arg, "--test-filter=") or std.mem.eql(u8, arg, "--skip-test-filter") or std.mem.startsWith(u8, arg, "--skip-test-filter=") or + std.mem.eql(u8, arg, "--expect-error-logs") or std.mem.startsWith(u8, arg, "--seed=") or std.mem.startsWith(u8, arg, "--cache-dir=") or std.mem.eql(u8, arg, "--listen=-"); @@ -74,6 +82,11 @@ pub fn select( } else if (std.mem.startsWith(u8, arg, "--skip-test-filter=")) { validateSkipTestFilter(arg["--skip-test-filter=".len..]) catch @panic("missing value after --skip-test-filter="); + } else if (std.mem.eql(u8, arg, "--expect-error-logs")) { + if (i + 2 >= args.len) @panic("--expect-error-logs requires a test filter and exact count"); + validateExpectedErrorLogs(args[i + 1], args[i + 2]) catch + @panic("invalid --expect-error-logs test filter or count"); + i += 2; } else if (std.mem.eql(u8, arg, "--list-tests") or std.mem.eql(u8, arg, "--allow-empty-test-filter") or std.mem.startsWith(u8, arg, "--seed=") or @@ -127,6 +140,12 @@ pub fn addRuntimeControls( validateSkipTestFilter(arg["--skip-test-filter=".len..]) catch @panic("missing value after --skip-test-filter="); run.addArg(arg); + } else if (std.mem.eql(u8, arg, "--expect-error-logs")) { + if (i + 2 >= args.len) @panic("--expect-error-logs requires a test filter and exact count"); + validateExpectedErrorLogs(args[i + 1], args[i + 2]) catch + @panic("invalid --expect-error-logs test filter or count"); + run.addArgs(args[i .. i + 3]); + i += 2; } else if (std.mem.eql(u8, arg, "--list-tests") or std.mem.eql(u8, arg, "--allow-empty-test-filter") or std.mem.startsWith(u8, arg, "--seed=") or @@ -146,6 +165,9 @@ test "select accepts repeated and equals-form test filters" { "--test-filter=table manager", "--skip-test-filter", "metadata VOPR", + "--expect-error-logs", + "expected failure path", + "2", "--seed=0x1234", }; const filters = select(std.testing.allocator, &args, &.{"default"}); @@ -211,6 +233,9 @@ test "foreign option detection is generic and preserves test-only arguments" { "--test-filter=metadata", "--skip-test-filter", "slow", + "--expect-error-logs", + "expected failure path", + "2", "--seed=1234", "--cache-dir=/tmp/cache", "--listen=-", @@ -222,6 +247,22 @@ test "empty skip filters are rejected before compiling a zero-test selection" { try validateSkipTestFilter("known flaky test"); } +test "expected error log controls require a named test and positive exact count" { + try validateExpectedErrorLogs("expected failure path", "2"); + try std.testing.expectError( + error.InvalidExpectedErrorLogFilter, + validateExpectedErrorLogs("", "2"), + ); + try std.testing.expectError( + error.InvalidExpectedErrorLogCount, + validateExpectedErrorLogs("expected failure path", "0"), + ); + try std.testing.expectError( + error.InvalidExpectedErrorLogCount, + validateExpectedErrorLogs("expected failure path", "many"), + ); +} + test "list mode preserves an independently requested runtime selection" { const filters = select(std.testing.allocator, &.{ "--list-tests", "--test-filter", "cutover" }, &.{"suite"}); defer std.testing.allocator.free(filters); diff --git a/zig/docs/GRAPH_METRICS_EXECUTION.md b/zig/docs/GRAPH_METRICS_EXECUTION.md new file mode 100644 index 0000000000..054995843b --- /dev/null +++ b/zig/docs/GRAPH_METRICS_EXECUTION.md @@ -0,0 +1,426 @@ +# Graph metric execution and resource ownership + +Graph metrics use shared numerical semantics with backend-specific persistence. +The production boundary is admitted, generation-fenced work—not a synchronous +full-graph calculation hidden inside a query or maintenance tick. + +### Durable cross-job stateful topology + +Membership blocks, ordinal dictionaries, exact out-degree totals, and packed +adjacency have an independent durable owner. Its SHA-256 identity binds the +topology format epoch, canonical edge-filter set, and complete checksummed +generation partition plan. Metric names, damping, tolerance, and iteration +limits do not enter that identity. Numeric vectors (including the PageRank +degree-vector accelerator), folds, seeds, page attempts, and publication remain +job-local. + +The first complete forward reduction seals the forward topology; HITS seals +both orientations after its first reverse reduction. Only complete phase +barriers can publish an owner. An adopting job validates the sealed owner and +writes its binding and lifetime pin in one transaction. It skips physical edge +discovery and adjacency production, reads the shared canonical membership for +its own seed/initialization, and uses shared packed tiles for every iteration. +HITS topology can serve PageRank or eigenvector; a forward-only owner cannot +satisfy HITS. Published scores keep their existing format; intermediate jobs +from execution schemas before v20 restart. + +Cold scheduled builds first enqueue an index-scoped preparation task keyed by +generation, filter and required orientation. Concurrent PageRank/eigenvector +requests share it; queued HITS requirements select a bidirectional task. The +task has its own control namespace, page leases and recovery checkpoints, and +one independently admitted execution slot per index. At most 16 distinct cold +tasks are admitted per index; compatible consumers join an existing task without +using another slot. Admission precedes the numerical active-build cap, so ready +topology can be prepared while numerical slots are occupied. Each bounded +checkpoint rotates to the next task. The rotation cursor is an in-memory fairness +hint, while task incarnations and leases provide durable recovery. It builds membership and +packed adjacency without rank vectors or score publication. Waiting metrics +retain durable requests, but hold no numerical lease or admission slot. +Only after the owner seals does the coordinator admit numerical jobs. Explicit +low-level planned execution retains an independent-producer path for isolated +maintenance and parity benchmarks; the first sealed owner wins its directory. + +Task failures preserve their root cause on dependent metrics. Retirement is +durably marked before bounded control-key deletion, so a crash cannot resurrect +a partially deleted task. A durable monotonic incarnation gives each retry a +separate control namespace. Admission, failure delivery and retirement validate +that incarnation, including across reopened handles. Failure delivery is +idempotent per task/canonical lifecycle owner so a delayed reporter cannot consume +a new manual retry request. Paired HITS failures use the authority owner even when +the hub alias reports first; both lanes receive the same root cause atomically. +Generation changes and loss of all eligible consumers +retire preparation; independent numerical/publication lifetimes are unchanged. +Inline numerical drains propagate coordinator terminal failures as failed status, +preserving the durable root cause instead of replacing it with an idle-page error. +Intermediate partition-plan v9 uses 4,096-unit scheduling ranges (capped at 256 +partitions), with byte/work-bounded checkpoints within each range. Canonical +256-entry membership/vector chunks remain separate from scheduling page size. +Tests can inject smaller ranges to exercise takeover and partition boundaries. +The sealed plan is a 76-byte counts/identity/checksum record. Census checkpoints +write separately addressed, generation-checked boundary slots; completion hashes +the boundary set outside the writer and publishes the small header in the same +checkpoint CAS. Only initial manifest planning materializes those slots. Lease +checks, topology ownership and subsequent iteration planning read the header; +later numerical and summary pages reuse their iteration-zero range templates. +Slots are bounded to 256 per direction and reused on generation changes. Filter +removal reclaims both controls and slots through the bounded filter-plan GC. + +Reclamation is index-scoped, including indexes with zero configured metrics. +Each transaction examines at most 64 pins and deletes at most 512 topology +records. Current configured filters retain reusable owners; active job pins +protect older generations. Removed filters, removed metrics, failed producers, +obsolete format epochs, and unreferenced concurrent owners become reclaimable. +A durable deleting tombstone atomically unpublishes the owner and fences late +writes/adoption; deletion resumes after crashes by removing the next key page. +Superseded packing attempts have a separate bounded retirement queue so a +retained owner does not retain abandoned tiles indefinitely. +Census position is an in-memory fairness hint: retained-owner and end-of-catalog +scans write no durable cursor or WAL record. Idle inspections have a per-sweep +budget without reporting eligible worker work. Actual reclamation consumes the +normal worker-page budget; durable tombstones/deleted keys provide recovery. + +## Non-serverless + +- Generation-transition contention retains a stable retryable error across + runtime archives and internal HTTP. Public queries retry a fresh snapshot + within the existing cancellation/deadline budget; persistent contention is + reported as temporary read unavailability instead of an opaque internal failure. +- Derived visibility waits carry their cancellation, absolute deadline and clock + together through manual and Io-backed executors. A borrowed backend clock's + timestamp is never reinterpreted in the native process clock domain. +- Global incidence counts use the same original/final mutation set as topology + invalidation. Duplicate operations and delete/reinsert replacements do not + perform intermediate counter writes. Endpoint deltas borrow input IDs, encode + each distinct changed node once, and bulk-read sorted counts in 256-key pages. + Both ends of a self-loop contribute; all borrowed values are decoded before + batch mutation. This also applies to graphs without configured metrics. +- Connectivity epochs advance only when a batch changes the final edge identity + set. Identical upserts, attribute-only updates, missing deletes, and delete/ + reinsert replacements do not restart unweighted metric jobs. Each selected + relationship type has a durable epoch; a filtered metric depends on their + maximum, not unrelated writes. Status generations describe that dependency. + All-edge metrics still depend on the global connectivity epoch. Old stores + acquire a conservative migration floor without a writer-side full scan. +- Type-addressable, empty-value covering postings retain reverse-key ordering + within each type. Filtered discovery seeks only selected type ranges; weights + and metadata need not be decoded. Existing stores backfill in bounded, + checkpointed steps (record and key-memory limits) while connectivity + mutations maintain postings transactionally; attribute updates do not rewrite + these postings. The v2 covering index also keeps an incidence reference count + per (type, endpoint). Insert/delete and idempotent backfill update these counts + with edge postings in the same transaction; self-loops count twice. This adds + storage and mutation work, shared across all filters, instead of rebuilding a + source-wide endpoint set for each cold metric. + A filter-epoch partition snapshot freezes scheduling boundaries so unrelated + writes cannot invalidate in-flight discovery or shared topology adoption. + Removed-filter snapshots are reclaimed in bounded 64-record maintenance + sweeps, including indexes with no remaining metrics. +- Metric queries share a storage-independent read plan with serverless: load + filters, restrict stable source-row ordinals, load ordering columns, select + top-K, then load display-only columns. Reusable columns follow the selection + and public nodes move only once. Qualified nodes never probe local score keys. + A stateful read session validates every dependency policy up front and holds + one transaction across all stages, including empty selections. Publication or + cleanup between stages cannot mix generations or turn scores into misses. +- Query scratch, score columns, owned status metadata and replacement output + allocations reserve bytes from the request's shared graph budget before + allocation. Scratch frees release reservations; escaping output retains its + request charge without retaining a pointer to a stack-owned budget allocator. + Budget denial reports `GraphWorkBudgetExceeded`, not allocator exhaustion. + Sorted score reads stop at 4,096 keys or 1 MiB of encoded keys; one oversized + key may progress only if its allocation fits the caller's budget. +- Automatic and planned maintenance use resumable coordinator/worker pages. + Standalone HITS authority/hub definitions are eligible independently; compatible + pairs share a lifecycle as an optimization. Admission caps leave work queued + and `runUntilIdle` returns `RunUntilIdleDidNotConverge`, rather than selecting + unlimited local computation. Explicit legacy/oracle helpers remain opt-in. +- Planned idle maintenance uses the same catalog lifetime protection and + transactional generation/attempt fences as background workers. It does not + hold the DB apply lock while draining graph computation. +- A cold partition census visits at most 4,096 records per coordinator planning + step. It skips the metric metadata namespace by range seek, checkpoints its + cursor/counts/boundaries, and resumes after reopen. The checkpoint and completed + plan are shared by metrics on one graph generation. Compare-and-swap checkpoint + publication prevents competing coordinators from regressing progress. A graph + mutation invalidates the obsolete census; it cannot publish mixed-generation + boundaries. Memory is bounded by the maximum 256 partitions, not graph size. + Filtered plans do not depend on that global census: they merge only selected + edge and endpoint posting ranges, count exact selected cardinalities, then + choose balanced boundaries. Endpoint streams deduplicate nodes shared by + selected types using a fanout-bounded heap. Their checkpoint and CAS bind the + filter epoch, so unrelated graph churn cannot reset cold planning. Each step + also stops at 1 MiB of visited suffix bytes (one oversized record may progress). +- All-edge scans range-seek past metadata. Filtered scans charge only selected + postings against their checkpoint limit and persist a type-qualified resume + key, validated against the filter and scheduling range. Intermediate progress + counts visited edges; completion seals the entire scheduling range. An + unbounded final partition cannot walk all metric state. + Ordinal topology extraction retains one reusable full resume-key buffer, not + one per visited edge. A conservative 1 MiB input-scratch admission limit also + bounds decoded endpoints and pending ordinal lookups, allowing one oversized + edge to make progress. Long type names therefore cannot multiply a 4,096-record + page into hundreds of MiB of retained cursor copies. +- Initialization writes canonical membership once in checksummed 256-row blocks, + alongside ordinal assignments. Completed initialization leaves seal exact row + counts. Vector initialization, iteration, convergence and publication read these addressed blocks, + not up to 256 producer partials per node on every checkpoint. Readers bind + block ordinals and node ranges to the leaf, validate resume-node identity, and + reject missing/truncated/misplaced blocks. Replayed writes accept only identical + rows even when checkpoint boundaries change. +- Reducers join sealed canonical nodes with the ordinal dictionary using + ordered cursors, then carry ordinals through numeric reads and writes. The + canonical membership check is essential: a missing dictionary row is an error, + not permission to omit a node. PageRank stores immutable out-degrees in exact + `u64` chunks, avoiding per-node string-key lookups on every iteration. +- After the initialization-summary barrier, every initializer consumes the sealed + membership and carries its validated slots into all rank/factor/HITS lane + writes. It does not rediscover producers or resolve the same node dictionary + separately for each output lane. Numerical seeds still use the global summary. +- The initialization phase barrier seals a bounded metric-specific active-node + plan. Empty node partitions in iteration zero are completed without worker + claims; later iterations omit their data pages and scalar leaves. Original + leaf IDs and membership blocks remain unchanged because vector slots encode + those identities. Plan totals must match the sealed initialization root, and + missing active leaves fail closed. Reopen and retries reuse the same plan. +- Adjacency producer phases exist only in iteration zero. Subsequent PageRank + and eigenvector iterations start at reduction; HITS moves from authority + reduction directly to hub reduction. Publication still verifies the sealed + iteration-zero producer barriers. No metadata-only producer pages, claims, + or completion transactions are scheduled for later iterations, and progress + fractions use only the phases that actually run. +- Numerical folds, normalization and convergence enumerate dense ordinal slots + from a checksummed active-node plan and sealed membership-leaf counts. Their + durable completed-unit count is the resume cursor: they do not decode node IDs + or join the node dictionary on each iteration. Range boundaries are reloaded + in the current transaction. Initialization and publication still validate + membership and the dictionary; missing required vector values fail closed. +- Numerical folds validate and borrow each immutable 256-edge tile from the read + transaction. One checkpoint-local scratch buffer gathers vector values and + maps chunk-local target slots directly to compensated accumulators. Warm vector + gathers allocate no per-tile arrays; cold gathers reuse arena capacity. Chunk + changes clear target mappings, and framing, receipt counts, ordinal validity, + generation/attempt fences and accumulation order remain enforced. Transaction + scratch is bounded by checkpoint limits, not total graph size. +- Sealed source-vector chunks may be reused across checkpoints. Each index has + a lazy 4,096-entry LRU, but all indexes share a 64 MiB admission pool by default, + charging entries and hash buckets. Hosts may inject a different shared pool + through `GraphIndexOptions.sealed_vector_budget`; it must outlive its indexes. + A full pool causes local recycling or storage-read fallback, never build + failure. Metric retirement releases cached chunks and empty bucket storage; + admission tickets prevent already-running checkpoints from repopulating + retired data. Worker handles observe retirement independently of coordinators. +- Final numeric-score publication admits at most 4,096 nodes or 1 MiB of node + IDs per checkpoint (one oversized ID is allowed to guarantee progress). + This is independent of scheduling range size. Prior scores for both + HITS lanes are bulk-read before either lane stages mutations. Primary scores, + ordered staging keys, and the attempt-fenced page cursor commit atomically. + The coordinator checkpoints the bounded top-K prefix before pointer publication. +- Execution schema 19 fences older intermediate jobs. Published score epochs + retain their existing read contract; an execution-format change does not hide + previously published results. + +## Serverless + +- Normal and lake ingestion share one ordinal graph builder. Distinct node IDs, + relationship types, and target tables are interned once; retained edges carry + numeric ordinals. Encoding counts adjacency sizes and scatters directly into + the final wire allocation, then sorts each node/direction in place. It does not + retain separate forward and reverse edge arrays. Scratch is 24 bytes per + dictionary node instead of up to 40 bytes per edge; sparse graphs can have a + different tradeoff than dense graphs. Canonical ordering, qualified endpoints + and isolated local nodes are preserved. + Both JSON adapters propagate allocator exhaustion and unwind partial edge + ownership; allocation failure cannot silently produce an empty graph. +- Graph wire v7 and manifest v22 bind an 80-byte topology trailer to the + manifest. Its SHA-256-authenticated directory contains canonical per-type + semantic digests, dictionary page offsets, bounded first-key fence prefixes, + and SHA-256 checksums for 64 KiB + data blocks. Cold readers fetch only the trailer, directory and blocks covering + selected ranges. They verify every fetched block before decoding; actual + aligned/overfetched bytes count against the shared read allowance. Directory + size remains capped at 1 MiB. An explicitly unavailable directory retains the + current-wire full-preparation path, not a legacy decoder. Low-level callers + without a manifest control binding must authenticate the complete artifact + before trusting its directory. +- An authenticated eight-byte-per-dictionary-node routing array addresses + adjacency rows. Public MATCH, traversal, path, and dedicated graph-query + readers resolve dictionary pages through the small fence directory, then read + only the requested row/type intervals. Exact relationship probes binary-search + canonical `(type, neighbor)` ordinals and preserve the minimum-weight match. + Fence prefixes are capped at 64 bytes per page; long common prefixes widen a + bounded binary search rather than making the control object unbounded. + Query allocators admit directory, cache, page, and decoded-row memory before + allocation. Traversal parent identities borrow only visited retained rows. + All reads share one byte allowance and authenticate every fetched block. + Adjacent metric type runs share one authenticated boundary block, preventing + thousands of small runs from repeatedly downloading the same 64 KiB range. + Point/traversal readers retain up to eight blocks (512 KiB) per source so + dictionary, routing, and row reads do not evict each other on every hop. +- Filesystem object GET pins one file handle for metadata and body reads. + Concurrent atomic HEAD replacement cannot turn an unconditional read into a + failed precondition; explicit ETag conditions still bind that pinned object. +- One bounded source-control object is retained while all pending filters on + that source drain, independent of request/alias ordering. Filters prepare + separately, smallest selected edge count first; one oversized filter cannot + make a small filter retain its topology. Equivalent filters still share + preparation, and compatible metric requirements share their projection. + An unavailable directory falls back to one shared full-source preparation. + Returned topology carries verified type digests, avoiding a second graph-wide + hashing pass and per-node digest array. Fallback preparation computes the same + digests under the separate identity-work allowance. + Compatible projection requirements share preparation when the combined work + and memory fit. Otherwise, cheaper exact requirements are admitted first. + Admission bounds active nodes by `min(source_nodes, 2 * selected_edges)` and + uses the same sparse/dense census work model as construction. It still charges + source-wide maps on the dense path. Identical computation aliases count once, + as do compatible HITS pairs. This admission pass needs no edge scan or scratch + allocation; exact construction admission and the live-allocation limiter + remain authoritative. Serverless additionally reserves local-ID adapters, + selection permutations and replacement-node buffers before allocation. + Materializer epoch 23 binds the current addressed graph layout and preparation admission. +- Preparation has two admission phases. The projection census is charged before + allocations or edge scans; exact projection construction is charged after the + census and before CSR allocation. Reserved census work remains charged when + construction is rejected. Exhausted publications cannot repeatedly construct + unaffordable projections. A live-allocation limiter also covers scratch buffers + and failure paths before a post-census size estimate is available. +- If selected edges are at most 1/64 of the source node count, projection sorts + and deduplicates their endpoints and uses binary ordinal lookup while replaying + the original edge order. Scratch and census work then depend on the selection, + not the source dictionary. Dense projections retain linear-time source-wide + maps/counts. Both paths preserve canonical node order and numerical summation + order; degree projections do not retain neighbors. +- Output has two admission phases too: a framing/row lower bound rejects + impossible output before kernels or warm-start reads; a prepared encoding plan + then reserves exact payload bytes before allocation. Compatible HITS lanes + reserve both outputs atomically before either upload, while encoding one at a + time. Allocation, cancellation and integrity failures refund reservations. +- Reuse uses a single authenticated, provider-pinned range read. A table-wide + `max_total_reuse_read_bytes` allowance (512 MiB by default) covers requested + headers and cold full-content authentication. This allowance is separate from + optional warm-start reads and numerical work. Native stores charge full-content + bytes only on a verification miss; a cached object pin needs no redundant HEAD. + SHA-256 provider metadata can authenticate a cold object without downloading it. + Custom stores use conservative full-object admission. Exhausting reuse admission + skips that optimization and leaves materialization subject to its own budgets. +- Immutable metric payloads carry a SHA-256 identity of selected unweighted local + topology, distinct from the current publication's full graph checksum. Hashing + canonical endpoint identities (not ordinals) makes weights, unrelated types, + qualified edges and isolated documents irrelevant to this identity. A matching + authenticated metric header, configuration and materializer policy allow reuse + without projection, kernels or score encoding. The manifest rebinds it to the + current graph checksum/generations while preserving the real computation time. + Readers validate both semantic binding and current source integrity. Original + source strings in the immutable payload need not equal the new publication's + strings; authenticated control lengths come from the artifact manifest. + Changed indexed graphs authenticate their directory and only the selected + topology blocks. Reuse can read semantic digests directly from that verified + directory; full-content authentication is unnecessary when the manifest + already binds the control root. + Optional hashing has a separate 1 GiB byte-work allowance and live-allocation + admission including any retained projection. Per-type digest scratch is freed + before numerical work. If admission is exhausted, a zero identity disables + cross-source reuse and retains exact-source validation; cold work keeps its + independent budget. +- Optional PageRank warm starts authenticate control, root, directory, selected + routing pages and primary score windows. Sparse selections skip unrelated + blocks; consecutive selected blocks share windows up to 1 MiB, narrowed to + available memory headroom (one larger block is allowed if it fits). Only one + page/window is retained alongside the seed and + bounded metadata. Ranked score payloads are not read. A live allocator bounds + preparation memory, and requested bytes plus any cold provider verification + are charged to the seed budget before I/O. Budget/integrity failures discard + the partial seed and fall back to cold computation; cancellation and genuine + allocation failures propagate. +- Cold providers without comparable SHA-256 metadata still require a bounded + full-content hash. Identity caches are process-local; no untrusted durable + “verified” flag bypasses authentication. Persisting verification evidence would + require a defined trust and provider-generation contract, not just caching a + boolean in a manifest. +- Manifest v19 and metric segment v10 are current-version-only. Missing graph + provenance starts at the current publication generation; there is no inference + from pre-release sidecars and no obsolete wire decoder or migration path. + +## Query and operator views + +Score, top-K, and column snapshots read compact publication/freshness metadata +and scores under one stable transaction. Queries do not fetch operator event or +failure histories, aggregate worker progress, or enumerate page details. Detailed +administrative status remains available through the existing operator paths. +Freshness requirements are checked before score reads, including reranking. + +Serverless authenticated disk hits promote into the same bounded canonical block +cache used by network fills. Promotion is optional and never waits on a pending +fill or pinned-capacity pressure. Point-score consumers borrow ref-counted leases +on warm blocks instead of copying payloads; leases keep entries alive during +decoding. Authentication is unchanged, and a cache failure remains a miss rather +than authority over the immutable source. + +Decoded point-routing pages, roots, and directories share the configured +`max_graph_metric_routing_bytes` allowance (16 MiB by default), without a separate +64-entry residency ceiling. Intrusive hash buckets provide keyed lookup and +separate unpinned LRUs prioritize page eviction over metadata eviction. Pinned +entries stay charged; a saturated cache safely bypasses admission. Eviction does +not scan pinned entries. The bounded 64-slot in-flight ownership table is still +independent of residency and retains its cancellation/single-flight contract. +Decoded page misses reserve this same ownership table before fetching or +decoding. A bounded group publishes and finishes every owned fill before +waiting on other producers or table capacity, preventing multi-page deadlocks. +Waiters share the decoded lease and are charged retained memory, not a duplicate +decode. Cancellation and failed producers release fill registrations. + +Point queries admit output descriptors/cells before allocating them, then admit +one `u32` candidate permutation shared by every physical metric column. IDs are +validated once and a common prefix is skipped. Admitted transient `u64` prefix +keys accelerate sorting, with full-string comparison on ties; they are freed +before any metric plan is prepared. Only the shared `u32` permutation survives. +Original row indexes preserve duplicate IDs and public result order. Routing +uses binary boundaries in that order, so dense block/page spans do not rescan +every row per column. Per-column ownership contains only unresolved block spans, +not another row map; authenticated cache hits are consumed during preparation. +Span, range, selected-page and decoded-routing capacities are charged before +allocation, including possible owned-slice replacement peaks. These reservations +share the request memory limit, but point-read scratch and routing leases release +their conservative charge when the read ends and all children have joined. +Control buffers, routing transport buffers, and decoded routing leases have +explicit live reservations that retire at their actual ownership boundaries. +Cold decode reservations transfer into leases without a release/reacquire gap. +Preparation fanout falls back to one column when a conservative two-column +memory envelope does not fit; exact reservations, not that envelope, decide +request eligibility. +Request-scoped output columns transfer move-only reservations into the staged +HTTP query cache; replacing or discarding a column releases its prior charge. +Rebasing reserves the replacement before allocation and commits ownership only +after successful scatter, preserving old data and admission on failure. +Public output APIs detach the reservation because their results may outlive the +session; those escaping results retain a conservative request charge. Network +requests/bytes, decoded blocks and work remain cumulative and cannot be refunded +by dropping a stage. The complete transport plan is admitted before score I/O. + +Live transport buffers also reserve this shared memory budget, including +authenticated cache-fill/lease bytes, the contiguous output and block descriptors. +The reservation survives until decoding releases the range. Column execution +reserves its joined group before launching workers and reduces column/range +fanout when only serial execution fits. The eight-range/32 MiB transport cap is +an additional ceiling, not a substitute for request-wide memory admission. + +Within an authenticated score block, sparse candidates use binary lookup while +dense sorted candidates merge once through the block. Original row ordinals +scatter results without reordering callers or losing duplicate/missing IDs. + +Top-K reserves descriptor storage before allocation or ranked-block reads, then +charges each decoded node ID before allocating it. Both per-result and shared +request limits apply. The HTTP response transfers those node allocations, rather +than temporarily retaining a second copy; its replacement descriptor array is +also admitted before allocation. Transfer failure leaves the original owner +intact. Retained-byte accounting remains conservative and request-cumulative. + +## Validation and measurements + +Regression coverage includes planning reopen/mutation fencing, metadata range +skipping, missing ordinal rows, exact integer degrees, standalone/incompatible +HITS definitions, admission caps, pre-allocation rejection, cold/warm artifact +authentication budgets, and point-only query metadata. See +[the benchmark report](../bench/graph/METRIC_PREPARATION.md) for measured scope, +fixtures, and limitations. Kernel or mock-storage microbenchmarks are not claims +about whole-query, whole-build, or cloud-network latency. diff --git a/zig/e2e/antfly/test_graph.py b/zig/e2e/antfly/test_graph.py index eb5e975e01..f210b20ac4 100644 --- a/zig/e2e/antfly/test_graph.py +++ b/zig/e2e/antfly/test_graph.py @@ -127,6 +127,76 @@ def _batch_write_stateful(api, table_name: str, **kwargs) -> dict: return batch +def test_serverless_filtered_pagerank_uses_selected_topology(serverless_api): + table = f"graph_metric_selected_{time.time_ns()}" + serverless_api.ensure_table(table, created_at_ns=100) + assert_created_index( + serverless_api.create_index( + table, + "graph_idx", + { + "name": "graph_idx", + "type": "graph", + "metrics": { + "rank": { + "kind": "pagerank", + "max_iterations": 20, + "edge_filter": {"types": ["selected"]}, + } + }, + }, + ), + "graph_idx", + "graph", + ) + config = serverless_api.get_index(table, "graph_idx")["config"] + assert config["metrics"]["rank"]["edge_filter"] == {"types": ["selected"]} + mutations = [ + upsert( + source, + json_doc( + text=source, + graph_edges=[{"target": target, "edge_type": "selected"}], + ), + ) + for source, target in [("a", "b"), ("b", "a")] + ] + mutations.extend( + upsert( + f"noise-{i}", + json_doc( + text="unrelated", + graph_edges=[{"target": f"noise-{i}", "edge_type": "noise"}], + ), + ) + for i in range(128) + ) + serverless_api.ingest_table(table, timestamp_ns=200, mutations=mutations) + try: + serverless_api.build_table(table) + except requests.HTTPError as error: + if error.response is None or error.response.status_code != 409: + raise + + def published_scores(): + response = serverless_api.query_table( + table, + {"graph_metric": {"index": "graph_idx", "metric": "rank", "top_k": 10}}, + ) + results = response.get("responses", []) + if not results: + return None + result = results[0].get("graph_metric_results", {}).get("rank", {}) + scores = result.get("scores", []) + return scores if len(scores) == 2 else None + + scores = wait_until(published_scores, timeout_s=30.0, interval_s=0.1) + assert scores is not None + assert {row["node"] for row in scores} == {"a", "b"} + for row in scores: + assert row["score"] == pytest.approx(0.5, abs=1e-12) + + def test_graph_neighbors_traverse_and_shortest_path(serverless_api): public_traverse_payload = { "graph_queries": { @@ -1860,6 +1930,18 @@ def doc(label: str, edges: dict | None = None) -> dict: assert batch["inserted"] == len(inserts) assert wait_until(predicates_ready, timeout_s=120.0, interval_s=0.25) is not None + # Index creation/reconciliation is asynchronous. full_index waits for + # indexed writes, not installation of a newly requested graph incarnation; + # predicate readiness alone does not establish graph readiness. + def graph_ready() -> dict | None: + return ready_index_status( + backup_api.get_index(table_name, "social"), + until="complete", + require_query_fresh=True, + ) + + assert wait_until(graph_ready, timeout_s=120.0, interval_s=0.25) is not None + def node(label: str) -> dict: return {"filter": {"term": label, "path": "/type"}} diff --git a/zig/lib/objectstore/src/client.zig b/zig/lib/objectstore/src/client.zig index ddcb2ed2e8..8175d054db 100644 --- a/zig/lib/objectstore/src/client.zig +++ b/zig/lib/objectstore/src/client.zig @@ -87,7 +87,7 @@ pub const Client = struct { if (stat.size > fallback_upload_limit_bytes) return error.StreamingUploadUnsupported; const body = try self.allocator.alloc(u8, @intCast(stat.size)); defer self.allocator.free(body); - if (try file.readPositionalAll(io, body, 0) != body.len) return error.SourceFileChanged; + try readPositionalAllWithCancellation(file, io, body, opts.cancellation); var extra: [1]u8 = undefined; if (try file.readPositionalAll(io, &extra, stat.size) != 0) return error.SourceFileChanged; if (opts.cancellation) |token| try token.check(); @@ -228,6 +228,60 @@ fn threadedIo() std.Io.Threaded { return std.Io.Threaded.init(std.heap.page_allocator, .{}); } +/// Reads a stable file snapshot with bounded cancellation latency. Provider +/// implementations share this path so their non-multipart uploads cannot hide +/// a long local read behind one cancellation checkpoint. +pub fn readPositionalAllWithCancellation( + file: std.Io.File, + io: std.Io, + body: []u8, + cancellation: ?types.CancellationToken, +) !void { + const cancellation_chunk_bytes = 1024 * 1024; + var copied: usize = 0; + while (copied < body.len) { + if (cancellation) |token| try token.check(); + const chunk_len = @min(cancellation_chunk_bytes, body.len - copied); + if (try file.readPositionalAll(io, body[copied..][0..chunk_len], copied) != chunk_len) return error.SourceFileChanged; + copied += chunk_len; + } + if (cancellation) |token| try token.check(); +} + +test "objectstore file reads observe cancellation between bounded chunks" { + const alloc = std.testing.allocator; + const io = std.testing.io; + const path = try std.fmt.allocPrint(alloc, "/tmp/antfly-objectstore-cancel-read-{d}", .{uniqueNs(io)}); + defer alloc.free(path); + defer std.Io.Dir.deleteFileAbsolute(io, path) catch {}; + const source_bytes = try alloc.alloc(u8, 3 * 1024 * 1024); + defer alloc.free(source_bytes); + @memset(source_bytes, 0x5a); + { + var output = try std.Io.Dir.createFileAbsolute(io, path, .{ .truncate = true }); + defer output.close(io); + try output.writePositionalAll(io, source_bytes, 0); + } + const source = try std.Io.Dir.openFileAbsolute(io, path, .{}); + defer source.close(io); + const body = try alloc.alloc(u8, source_bytes.len); + defer alloc.free(body); + @memset(body, 0); + const State = struct { + calls: usize = 0, + fn cancelled(ptr: *const anyopaque) bool { + const self: *@This() = @ptrCast(@alignCast(@constCast(ptr))); + self.calls += 1; + return self.calls >= 2; + } + }; + var state = State{}; + const cancellation = types.CancellationToken{ .ptr = &state, .is_cancelled_fn = State.cancelled }; + try std.testing.expectError(error.Canceled, readPositionalAllWithCancellation(source, io, body, cancellation)); + try std.testing.expectEqual(@as(u8, 0x5a), body[0]); + try std.testing.expectEqual(@as(u8, 0), body[1024 * 1024]); +} + fn ensureParentDir(io: std.Io, path: []const u8) !void { const parent = std.fs.path.dirname(path) orelse return; try std.Io.Dir.cwd().createDirPath(io, parent); diff --git a/zig/lib/objectstore/src/filesystem.zig b/zig/lib/objectstore/src/filesystem.zig index ea278b7a2c..d89c29e52b 100644 --- a/zig/lib/objectstore/src/filesystem.zig +++ b/zig/lib/objectstore/src/filesystem.zig @@ -108,7 +108,7 @@ pub const FilesystemClient = struct { } try ensureParentDir(self.io, object_path); - const etag = try sha256HexAlloc(alloc, body); + const etag = try sha256HexAllocWithCancellation(alloc, body, opts.cancellation); errdefer alloc.free(etag); const staging_path = try stagingPathAlloc(alloc, self.root_dir, bucket); defer alloc.free(staging_path); @@ -226,10 +226,15 @@ pub const FilesystemClient = struct { if (opts.range != null and opts.part_number != null) return error.AmbiguousRange; const object_path = try objectPathAlloc(alloc, self.root_dir, bucket, key); defer alloc.free(object_path); - // Publication atomically replaces the path. Keep this descriptor for - // both metadata and payload so a GET observes one object generation. const file = try openFilePath(self.io, object_path); defer file.close(self.io); + return self.getOpenObject(alloc, bucket, key, file, opts); + } + + // Conditional writes replace the path atomically. Pin one opened object + // for metadata, range resolution and body reads so unconditional GET never + // becomes a spurious failed precondition when that path is replaced. + fn getOpenObject(self: *FilesystemClient, alloc: Allocator, bucket: []const u8, key: []const u8, file: std.Io.File, opts: types.GetOptions) !types.GetResult { const file_stat = try file.stat(self.io); var header = try readObjectHeader(alloc, self.io, file, file_stat.size); defer header.deinit(alloc); @@ -307,7 +312,6 @@ pub const FilesystemClient = struct { const file_stat = try file.stat(self.io); var header = try readObjectHeader(alloc, self.io, file, file_stat.size); defer header.deinit(alloc); - return objectMetadataAlloc(alloc, bucket, key, header, file_stat.mtime.toMilliseconds()); } @@ -794,6 +798,7 @@ fn writeObjectFileAtomically( encodeObjectHeader(&placeholder, source_stat.size, content_type.len, etag); try output.writePositionalAll(io, &placeholder, 0); try output.sync(io); + if (cancellation) |token| try token.check(); output.unlock(io); output_locked = false; output.close(io); @@ -984,8 +989,21 @@ fn partCount(content_length: u64) usize { } fn sha256HexAlloc(alloc: Allocator, body: []const u8) ![]u8 { + return sha256HexAllocWithCancellation(alloc, body, null); +} + +fn sha256HexAllocWithCancellation(alloc: Allocator, body: []const u8, cancellation: ?types.CancellationToken) ![]u8 { var digest: [32]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(body, &digest, .{}); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes = 1024 * 1024; + var offset: usize = 0; + while (offset < body.len) { + if (cancellation) |token| try token.check(); + const len = @min(chunk_bytes, body.len - offset); + hasher.update(body[offset..][0..len]); + offset += len; + } + hasher.final(&digest); return try digestHexAlloc(alloc, &digest); } @@ -1112,6 +1130,37 @@ fn cleanupTmp(path: [*:0]const u8) void { std.Io.Dir.cwd().deleteTree(io_impl.io(), std.mem.span(path)) catch {}; } +test "filesystem get pins metadata and body across atomic path replacement" { + const alloc = std.testing.allocator; + var path_buf: [256]u8 = undefined; + const path = tmpPath(&path_buf, "get-snapshot"); + defer cleanupTmp(path); + var fs = try FilesystemClient.init(alloc, std.mem.span(path)); + var client = fs.client(); + defer client.deinit(); + var first = try client.putObject("docs", "HEAD", "first", .{ .content_type = "old/type" }); + defer first.deinit(alloc); + const object_path = try objectPathAlloc(alloc, std.mem.span(path), "docs", "HEAD"); + defer alloc.free(object_path); + const pinned = try openFilePath(fs.io, object_path); + defer pinned.close(fs.io); + var second = try client.putObject("docs", "HEAD", "longer replacement", .{ .content_type = "new/type" }); + defer second.deinit(alloc); + var old = try fs.getOpenObject(alloc, "docs", "HEAD", pinned, .{}); + defer old.deinit(alloc); + try std.testing.expectEqualStrings("first", old.body); + try std.testing.expectEqualStrings("old/type", old.metadata.content_type.?); + try std.testing.expectEqualStrings(first.etag.?, old.metadata.etag.?); + var range = try fs.getOpenObject(alloc, "docs", "HEAD", pinned, .{ .range = .{ .offset = 1, .length = 3 }, .if_match_etag = first.etag }); + defer range.deinit(alloc); + try std.testing.expectEqualStrings("irs", range.body); + try std.testing.expectError(error.PreconditionFailed, fs.getOpenObject(alloc, "docs", "HEAD", pinned, .{ .if_match_etag = second.etag })); + var current = try client.getObject("docs", "HEAD", .{ .if_match_etag = second.etag }); + defer current.deinit(alloc); + try std.testing.expectEqualStrings("longer replacement", current.body); + try std.testing.expectEqualStrings("new/type", current.metadata.content_type.?); +} + test "filesystem client supports bucket/object lifecycle and file helpers" { const alloc = std.testing.allocator; var path_buf: [256]u8 = undefined; diff --git a/zig/lib/objectstore/src/gcs.zig b/zig/lib/objectstore/src/gcs.zig index fb0d32a594..818f22548f 100644 --- a/zig/lib/objectstore/src/gcs.zig +++ b/zig/lib/objectstore/src/gcs.zig @@ -352,6 +352,9 @@ pub const JsonApiClient = struct { opts: types.PutOptions, ) !types.PutResult { if (opts.cancellation) |token| try token.check(); + if (opts.checksum_sha256_hex) |checksum| { + if (body.len != 0) return try self.putObjectWithSha256Metadata(alloc, bucket, key, body, opts, checksum); + } const url = try uploadMediaUrlAlloc(alloc, self.cfg, bucket, key, opts); defer alloc.free(url); @@ -384,6 +387,75 @@ pub const JsonApiClient = struct { }; } + fn putObjectWithSha256Metadata( + self: *JsonApiClient, + alloc: Allocator, + bucket: []const u8, + key: []const u8, + body: []const u8, + opts: types.PutOptions, + checksum: []const u8, + ) !types.PutResult { + try validateSha256Hex(checksum); + const initiate_url = try uploadResumableUrlAlloc(alloc, self.cfg, bucket, key, opts); + defer alloc.free(initiate_url); + const size_text = try std.fmt.allocPrint(alloc, "{d}", .{body.len}); + defer alloc.free(size_text); + const content_type = opts.content_type orelse "application/octet-stream"; + const initiate_headers = [_]HeaderPair{ + .{ "X-Upload-Content-Length", size_text }, + .{ "X-Upload-Content-Type", content_type }, + }; + const metadata_payload = try uploadMetadataPayloadAlloc(alloc, content_type, checksum); + defer alloc.free(metadata_payload); + var initiated = try self.performWithResponseLimitAndCancellation( + .POST, + initiate_url, + &initiate_headers, + metadata_payload, + "application/json", + null, + opts.cancellation, + ); + defer initiated.deinit(alloc); + switch (initiated.status) { + 200, 201 => {}, + 304, 412 => return error.PreconditionFailed, + 404 => return error.FileNotFound, + else => return mapUnexpectedStatus(initiated.status), + } + const session_url = initiated.location orelse return error.MissingResumableUploadLocation; + var completed = false; + defer if (!completed) { + var cleanup_deadline: ?transfer.CleanupDeadline = if (self.operationIo()) |cleanup_io| + transfer.CleanupDeadline.init(cleanup_io) + else + null; + self.cancelResumableUpload( + session_url, + if (cleanup_deadline) |*deadline| deadline.token() else null, + ) catch {}; + }; + const content_range = try std.fmt.allocPrint(alloc, "bytes 0-{d}/{d}", .{ body.len - 1, body.len }); + defer alloc.free(content_range); + const upload_headers = [_]HeaderPair{.{ "Content-Range", content_range }}; + var response = try self.performWithResponseLimitAndCancellation( + .PUT, + session_url, + &upload_headers, + body, + content_type, + null, + opts.cancellation, + ); + defer response.deinit(alloc); + if (response.status != 200 and response.status != 201) return mapUnexpectedStatus(response.status); + var metadata = try parseObjectMetadataResponse(alloc, bucket, response.body); + defer metadata.deinit(alloc); + completed = true; + return .{ .etag = if (metadata.etag) |value| try alloc.dupe(u8, value) else null }; + } + fn putFile( self: *JsonApiClient, alloc: Allocator, @@ -413,7 +485,7 @@ pub const JsonApiClient = struct { if (stat.size <= resumable_threshold) { const body = try alloc.alloc(u8, @intCast(stat.size)); defer alloc.free(body); - if (try source.readPositionalAll(io, body, 0) != body.len) return error.SourceFileChanged; + try client_mod.readPositionalAllWithCancellation(source, io, body, opts.cancellation); var extra: [1]u8 = undefined; if (try source.readPositionalAll(io, &extra, stat.size) != 0) return error.SourceFileChanged; const current_stat = try source.stat(io); @@ -436,11 +508,14 @@ pub const JsonApiClient = struct { .{ "X-Upload-Content-Length", size_text }, .{ "X-Upload-Content-Type", upload_type }, }; + if (opts.checksum_sha256_hex) |checksum| try validateSha256Hex(checksum); + const metadata_payload = try uploadMetadataPayloadAlloc(alloc, upload_type, opts.checksum_sha256_hex); + defer alloc.free(metadata_payload); var initiated = try self.performWithResponseLimitAndCancellation( .POST, initiate_url, &initiate_headers, - "{}", + metadata_payload, "application/json", null, opts.cancellation, @@ -1169,6 +1244,21 @@ fn uploadResumableUrlAlloc( return url; } +fn validateSha256Hex(checksum: []const u8) !void { + var decoded: [32]u8 = undefined; + _ = std.fmt.hexToBytes(&decoded, checksum) catch return error.InvalidChecksum; +} + +fn uploadMetadataPayloadAlloc(alloc: Allocator, content_type: []const u8, checksum: ?[]const u8) ![]u8 { + if (checksum) |value| { + return try httpx.json.Json.stringify(alloc, .{ + .contentType = content_type, + .metadata = .{ .antfly_sha256 = value }, + }); + } + return try httpx.json.Json.stringify(alloc, .{ .contentType = content_type }); +} + fn openFilePath(io: std.Io, path: []const u8) !std.Io.File { return if (std.fs.path.isAbsolute(path)) try std.Io.Dir.openFileAbsolute(io, path, .{}) @@ -1226,6 +1316,9 @@ fn parseObjectMetadataResponse(alloc: Allocator, bucket: []const u8, body: []con contentType: ?[]const u8 = null, md5Hash: ?[]const u8 = null, crc32c: ?[]const u8 = null, + metadata: ?struct { + antfly_sha256: ?[]const u8 = null, + } = null, }; var parsed = try std.json.parseFromSlice(Parsed, alloc, body, .{ .ignore_unknown_fields = true }); @@ -1242,6 +1335,9 @@ fn parseObjectMetadataResponse(alloc: Allocator, bucket: []const u8, body: []con errdefer if (version_id) |value| alloc.free(value); const content_type = if (parsed.value.contentType) |value| try alloc.dupe(u8, value) else null; errdefer if (content_type) |value| alloc.free(value); + // Custom metadata is caller-controlled and is therefore not an + // authenticated checksum of the stored object. Only expose digests that + // GCS computes from the object bytes. var checksum: ?types.ObjectChecksum = if (parsed.value.md5Hash) |value| .{ .algorithm = .md5_base64, .value = try alloc.dupe(u8, value), @@ -1644,6 +1740,29 @@ test "json api metadata falls back to the always-available crc32c checksum" { try std.testing.expectEqual(types.ObjectChecksumType.full_object, meta.checksum.?.checksum_type); } +test "json api metadata never promotes caller-owned SHA-256 over provider checksum" { + const alloc = std.testing.allocator; + var meta = try parseObjectMetadataResponse( + alloc, + "bucket", + "{\"name\":\"metric\",\"generation\":\"9\",\"size\":\"4\",\"crc32c\":\"crc-body\",\"metadata\":{\"antfly_sha256\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}}", + ); + defer meta.deinit(alloc); + try std.testing.expectEqual(types.ObjectChecksumAlgorithm.crc32c_base64, meta.checksum.?.algorithm); + try std.testing.expectEqualStrings("crc-body", meta.checksum.?.value); +} + +test "json api metadata does not expose caller-owned SHA-256 as a checksum" { + const alloc = std.testing.allocator; + var meta = try parseObjectMetadataResponse( + alloc, + "bucket", + "{\"name\":\"metric\",\"generation\":\"9\",\"size\":\"4\",\"metadata\":{\"antfly_sha256\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}}", + ); + defer meta.deinit(alloc); + try std.testing.expect(meta.checksum == null); +} + test "gcs cancellation reaches active read and write requests" { const alloc = std.testing.allocator; const State = struct { @@ -1791,6 +1910,62 @@ test "json api client put object encodes upload url and returns etag" { try std.testing.expectEqualStrings("etag-2", put.etag.?); } +test "json api client persists informational caller SHA-256 metadata" { + const alloc = std.testing.allocator; + const checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const State = struct { + calls: usize = 0, + + fn request( + ptr: ?*anyopaque, + request_alloc: Allocator, + method: HttpMethod, + url: []const u8, + headers: []const HeaderPair, + body: ?[]const u8, + content_type: ?[]const u8, + _: ?usize, + _: ?types.CancellationToken, + ) !TransportResponse { + const self: *@This() = @ptrCast(@alignCast(ptr.?)); + self.calls += 1; + if (self.calls == 1) { + try std.testing.expectEqual(HttpMethod.POST, method); + try std.testing.expect(std.mem.indexOf(u8, url, "uploadType=resumable") != null); + try std.testing.expect(std.mem.indexOf(u8, body.?, "\"antfly_sha256\":\"" ++ checksum ++ "\"") != null); + try std.testing.expectEqualStrings("application/json", content_type.?); + return try transportResponseAlloc(request_alloc, 200, "", null, null, "https://upload.example/session"); + } + try std.testing.expectEqual(@as(usize, 2), self.calls); + try std.testing.expectEqual(HttpMethod.PUT, method); + try std.testing.expectEqualStrings("https://upload.example/session", url); + try expectHeader(headers, "Content-Range", "bytes 0-4/5"); + try std.testing.expectEqualStrings("hello", body.?); + return try transportResponseAlloc( + request_alloc, + 200, + "{\"bucket\":\"bucket\",\"name\":\"metric\",\"etag\":\"etag-sha\",\"size\":\"5\",\"metadata\":{\"antfly_sha256\":\"" ++ checksum ++ "\"}}", + null, + null, + null, + ); + } + }; + + var state = State{}; + const cfg = try jsonApiClientConfigAlloc(alloc); + var json_client = JsonApiClient.initWithRequestFn(alloc, cfg, &state, State.request); + var client = json_client.client(); + defer client.deinit(); + var put = try client.putObject("bucket", "metric", "hello", .{ + .content_type = "application/octet-stream", + .checksum_sha256_hex = checksum, + }); + defer put.deinit(alloc); + try std.testing.expectEqualStrings("etag-sha", put.etag.?); + try std.testing.expectEqual(@as(usize, 2), state.calls); +} + test "json api client lists objects and prefixes" { const alloc = std.testing.allocator; const State = struct { diff --git a/zig/lib/objectstore/src/memory.zig b/zig/lib/objectstore/src/memory.zig index 94fd72832a..23d927726b 100644 --- a/zig/lib/objectstore/src/memory.zig +++ b/zig/lib/objectstore/src/memory.zig @@ -107,11 +107,19 @@ pub const MemoryClient = struct { if (existing == null or !std.mem.eql(u8, existing.?.etag, expected)) return error.PreconditionFailed; } - const etag = try sha256HexAlloc(alloc, body); + const etag = try sha256HexAllocWithCancellation(alloc, body, opts.cancellation); errdefer alloc.free(etag); - const owned_body = try self.alloc.dupe(u8, body); + const owned_body = try self.alloc.alloc(u8, body.len); errdefer self.alloc.free(owned_body); + const cancellation_chunk_bytes = 1024 * 1024; + var copied: usize = 0; + while (copied < body.len) { + if (opts.cancellation) |token| try token.check(); + const chunk_len = @min(cancellation_chunk_bytes, body.len - copied); + @memcpy(owned_body[copied..][0..chunk_len], body[copied..][0..chunk_len]); + copied += chunk_len; + } const owned_etag = try self.alloc.dupe(u8, etag); errdefer self.alloc.free(owned_etag); const owned_content_type = if (opts.content_type) |value| try self.alloc.dupe(u8, value) else null; @@ -395,8 +403,21 @@ pub const MemoryClient = struct { }; fn sha256HexAlloc(alloc: Allocator, body: []const u8) ![]u8 { + return sha256HexAllocWithCancellation(alloc, body, null); +} + +fn sha256HexAllocWithCancellation(alloc: Allocator, body: []const u8, cancellation: ?types.CancellationToken) ![]u8 { var digest: [32]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(body, &digest, .{}); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes = 1024 * 1024; + var offset: usize = 0; + while (offset < body.len) { + if (cancellation) |token| try token.check(); + const len = @min(chunk_bytes, body.len - offset); + hasher.update(body[offset..][0..len]); + offset += len; + } + hasher.final(&digest); const out = try alloc.alloc(u8, 64); for (digest, 0..) |byte, idx| { out[idx * 2] = std.fmt.digitToChar(byte >> 4, .lower); @@ -484,6 +505,19 @@ test "memory get result construction cleans up every allocation failure" { try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); } +test "memory put cancellation never publishes an object" { + const alloc = std.testing.allocator; + var memory = MemoryClient.init(alloc); + var client = memory.client(); + defer client.deinit(); + try client.makeBucket("bucket"); + var canceled = std.atomic.Value(bool).init(true); + try std.testing.expectError(error.Canceled, client.putObject("bucket", "object", "payload", .{ + .cancellation = types.CancellationToken.fromAtomic(&canceled), + })); + try std.testing.expectError(error.FileNotFound, client.statObject("bucket", "object")); +} + test "memory client supports non-recursive listing with common prefixes" { const alloc = std.testing.allocator; var memory = MemoryClient.init(alloc); diff --git a/zig/lib/objectstore/src/s3.zig b/zig/lib/objectstore/src/s3.zig index 332b9375a5..23e7bcb4dd 100644 --- a/zig/lib/objectstore/src/s3.zig +++ b/zig/lib/objectstore/src/s3.zig @@ -769,6 +769,9 @@ pub const Client = struct { defer headers.deinit(alloc); const owned_if_match = try appendConditionalHeaders(alloc, &headers, opts.if_match_etag, opts.if_none_match); defer if (owned_if_match) |value| alloc.free(value); + if (opts.checksum_sha256_base64) |value| { + try headers.append(alloc, .{ "x-amz-checksum-sha256", value }); + } var response = try self.performWithResponseLimitAndCancellation( .PUT, @@ -822,7 +825,7 @@ pub const Client = struct { if (stat.size <= multipart_threshold) { const body = try alloc.alloc(u8, @intCast(stat.size)); defer alloc.free(body); - if (try source.readPositionalAll(io, body, 0) != body.len) return error.SourceFileChanged; + try client_mod.readPositionalAllWithCancellation(source, io, body, opts.cancellation); var extra: [1]u8 = undefined; if (try source.readPositionalAll(io, &extra, stat.size) != 0) return error.SourceFileChanged; const current_stat = try source.stat(io); @@ -2705,6 +2708,7 @@ test "s3 client signs and issues object operations through request fn" { checksum_value: ?[]const u8 = null, checksum_type: types.ObjectChecksumType = .unknown, expect_checksum_mode: bool = false, + expect_checksum_sha256: ?[]const u8 = null, expect_body: ?[]const u8 = null, expect_range: ?[]const u8 = null, expect_max_response_size: ?usize = null, @@ -2736,6 +2740,9 @@ test "s3 client signs and issues object operations through request fn" { if (step.expect_checksum_mode) { try expectHeaderValue(headers, "x-amz-checksum-mode", "ENABLED"); } + if (step.expect_checksum_sha256) |expected| { + try expectHeaderValue(headers, "x-amz-checksum-sha256", expected); + } try std.testing.expectEqual(step.expect_max_response_size, max_response_size); if (step.expect_body) |expected| { try std.testing.expectEqualStrings(expected, body orelse ""); @@ -2779,7 +2786,7 @@ test "s3 client signs and issues object operations through request fn" { const steps = [_]Step{ .{ .method = .HEAD, .url_contains = "/bucket", .status = 404 }, .{ .method = .PUT, .url_contains = "/bucket", .status = 200 }, - .{ .method = .PUT, .url_contains = "/bucket/docs/a.txt", .status = 200, .etag = "\"etag-put\"", .expect_body = "hello" }, + .{ .method = .PUT, .url_contains = "/bucket/docs/a.txt", .status = 200, .etag = "\"etag-put\"", .expect_checksum_sha256 = "checksum-base64", .expect_body = "hello" }, .{ .method = .HEAD, .url_contains = "versionId=v2", .status = 200, .etag = "\"etag-head\"", .content_type = "text/plain", .content_length = 5, .checksum_algorithm = .crc64nvme_base64, .checksum_value = "crc64-version", .checksum_type = .full_object, .expect_checksum_mode = true }, .{ .method = .GET, .url_contains = "partNumber=7&versionId=v2", .status = 206, .body = "ell", .etag = "\"etag-get\"", .content_type = "text/plain", .content_length = 3, .version_id = "v2", .checksum_algorithm = .sha256_base64, .checksum_value = "sha256-get", .checksum_type = .composite, .expect_checksum_mode = true, .expect_range = "bytes=1-3" }, .{ .method = .GET, .url_contains = "/bucket/docs/a.txt", .status = 206, .body = "hell", .etag = "\"etag-direct\"", .content_type = "text/plain", .content_length = 4, .expect_checksum_mode = true, .expect_max_response_size = 4 }, @@ -2808,7 +2815,10 @@ test "s3 client signs and issues object operations through request fn" { try std.testing.expect(!(try client.bucketExists("bucket"))); try client.makeBucket("bucket"); - var put = try client.putObject("bucket", "docs/a.txt", "hello", .{ .content_type = "text/plain" }); + var put = try client.putObject("bucket", "docs/a.txt", "hello", .{ + .content_type = "text/plain", + .checksum_sha256_base64 = "checksum-base64", + }); defer put.deinit(alloc); try std.testing.expectEqualStrings("etag-put", put.etag.?); diff --git a/zig/lib/objectstore/src/types.zig b/zig/lib/objectstore/src/types.zig index 1bd4d1c7e0..83d92ea554 100644 --- a/zig/lib/objectstore/src/types.zig +++ b/zig/lib/objectstore/src/types.zig @@ -128,6 +128,15 @@ pub const PutOptions = struct { content_type: ?[]const u8 = null, if_match_etag: ?[]const u8 = null, if_none_match: bool = false, + /// Optional full-object SHA-256 supplied by callers that already hashed + /// the body. S3-compatible implementations persist this as provider + /// checksum metadata so later integrity checks remain metadata-only. + checksum_sha256_base64: ?[]const u8 = null, + /// Optional caller-computed SHA-256 in lowercase hexadecimal form. GCS JSON + /// uploads persist this as informational custom metadata only. Providers do + /// not validate custom metadata against object bytes, so consumers must not + /// treat it as an authenticated `ObjectChecksum`. + checksum_sha256_hex: ?[]const u8 = null, /// Borrowed for the duration of this operation. Remote providers interrupt /// their active transport; streaming providers check between bounded /// chunks and before the final publication step. diff --git a/zig/lib/openapi/test/fixtures/antfly-metadata.json b/zig/lib/openapi/test/fixtures/antfly-metadata.json index 098f7ec4fd..92e6b4ed3d 100644 --- a/zig/lib/openapi/test/fixtures/antfly-metadata.json +++ b/zig/lib/openapi/test/fixtures/antfly-metadata.json @@ -4531,6 +4531,13 @@ }, "description": "Results from declarative graph queries." }, + "graph_metric_results": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/GraphMetricResult" + }, + "description": "Results from direct graph metric reads." + }, "profile": { "x-go-type-skip-optional-pointer": false, "allOf": [ @@ -6654,6 +6661,88 @@ } } }, + "GraphMetricStatus": { + "type": "object", + "required": [ + "state", + "published_generation", + "edge_generation", + "converged", + "iterations_completed", + "delta", + "computed_at_ms" + ], + "properties": { + "state": { + "type": "string" + }, + "published_generation": { + "type": "integer", + "format": "int64" + }, + "edge_generation": { + "type": "integer", + "format": "int64" + }, + "converged": { + "type": "boolean" + }, + "iterations_completed": { + "type": "integer", + "format": "int64" + }, + "delta": { + "type": "number", + "format": "double" + }, + "computed_at_ms": { + "type": "integer", + "format": "int64" + } + } + }, + "GraphMetricScore": { + "type": "object", + "required": [ + "node", + "score" + ], + "properties": { + "node": { + "type": "string" + }, + "score": { + "type": "number", + "format": "double" + } + } + }, + "GraphMetricResult": { + "type": "object", + "required": [ + "index_name", + "metric", + "scores", + "status" + ], + "properties": { + "index_name": { + "type": "string" + }, + "metric": { + "type": "string" + }, + "scores": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphMetricScore" + } + }, + "status": { + "$ref": "#/components/schemas/GraphMetricStatus" + } + } + }, "EvaluatorName": { "type": "string", "enum": [ diff --git a/zig/lib/platform/src/clock.zig b/zig/lib/platform/src/clock.zig index f5d929c3ec..118dcaa386 100644 --- a/zig/lib/platform/src/clock.zig +++ b/zig/lib/platform/src/clock.zig @@ -44,6 +44,14 @@ pub const Clock = struct { pub fn sleepMs(self: Clock, ms: u64) void { self.sleep_ms_fn(self.ctx, ms); } + + /// Returns whether this clock uses the process' wall clock. Runtime code + /// with an existing std.Io backend can use that backend for waits instead + /// of constructing a temporary threaded backend for every sleep. + pub fn isReal(self: Clock) bool { + return self.now_realtime_ns_fn == realNowRealtimeNs and + self.sleep_ms_fn == realSleepMs; + } }; pub const ManualClock = struct { diff --git a/zig/pkg/antfly/src/api/batch.zig b/zig/pkg/antfly/src/api/batch.zig index fbc53c4ab0..e1f47db215 100644 --- a/zig/pkg/antfly/src/api/batch.zig +++ b/zig/pkg/antfly/src/api/batch.zig @@ -13,15 +13,27 @@ // limitations. const std = @import("std"); +const ant_json = @import("antfly-json"); const db_mod = @import("../storage/db/mod.zig"); const document_mapper = @import("../storage/db/document_mapper.zig"); const public_limits = @import("public_limits.zig"); +pub const BatchFailure = struct { + code: []const u8, + message: []const u8, + reason: ?[]const u8 = null, + retryable: bool, +}; + pub const BatchResult = struct { status: []const u8 = "committed", inserted: u32, deleted: u32, transformed: u32 = 0, + /// Additive details for a durable commit that needs operator action. Keep + /// the stable status vocabulary so older strict-enum SDKs can still parse + /// the committed response and safely avoid replaying the mutation. + failure: ?BatchFailure = null, }; pub const OwnedBatchRequest = struct { @@ -85,6 +97,12 @@ pub const OwnedBatchRequest = struct { result_value.status = status; return result_value; } + + pub fn resultWithFailure(self: OwnedBatchRequest, failure: BatchFailure) BatchResult { + var result_value = self.resultWithStatus("committed_repair_required"); + result_value.failure = failure; + return result_value; + } }; pub fn parseBatchRequest(alloc: std.mem.Allocator, body: []const u8) !OwnedBatchRequest { @@ -610,12 +628,7 @@ fn parseBatchRequestWithOptions( } pub fn encodeBatchResponse(alloc: std.mem.Allocator, result: BatchResult) ![]u8 { - return try std.fmt.allocPrint(alloc, "{{\"status\":{f},\"inserted\":{d},\"deleted\":{d},\"transformed\":{d}}}", .{ - std.json.fmt(result.status, .{}), - result.inserted, - result.deleted, - result.transformed, - }); + return try ant_json.valueAlloc(alloc, result, .{ .emit_null_optional_fields = false }); } pub fn encodeBatchRequest(alloc: std.mem.Allocator, req: db_mod.types.BatchRequest) ![]u8 { diff --git a/zig/pkg/antfly/src/api/distributed_graph.zig b/zig/pkg/antfly/src/api/distributed_graph.zig index 5e3887c0f9..418474789d 100644 --- a/zig/pkg/antfly/src/api/distributed_graph.zig +++ b/zig/pkg/antfly/src/api/distributed_graph.zig @@ -1449,6 +1449,9 @@ pub const GraphExpandRequest = struct { exclude_edges: [][]u8, target_constraint_keys: [][]u8 = &.{}, params: graph_query_mod.QueryParams, + metrics: []graph_query_mod.GraphMetricRead = &.{}, + include_metric_status: bool = false, + defer_result_limit: bool = false, tensor_access_path: ?OwnedGraphTensorAccessPath = null, tensor_program: ?query_contract.OwnedAlgebraicTensorProgramEnvelope = null, topology_epoch: u64 = 0, @@ -1474,6 +1477,7 @@ pub const GraphExpandRequest = struct { for (self.target_constraint_keys) |key| alloc.free(key); if (self.target_constraint_keys.len > 0) alloc.free(self.target_constraint_keys); freeConstStrings(alloc, self.params.edge_types); + freeGraphMetricReads(alloc, self.metrics); if (self.tensor_access_path) |*path| path.deinit(alloc); if (self.tensor_program) |*program| program.deinit(alloc); if (self.resolved_doc_filter_owned) { @@ -1749,6 +1753,9 @@ const GraphExpandRequestJson = struct { exclude_nodes: []const GraphNodeIdentityJson = &.{}, exclude_edges: []const []const u8 = &.{}, target_constraint_keys: []const []const u8 = &.{}, + metrics: []const GraphMetricReadJson = &.{}, + include_metric_status: bool = false, + defer_result_limit: bool = false, topology_epoch: u64 = 0, identity_read_generation: ?u64 = null, _resolved_doc_filter: ?std.json.Value = null, @@ -1783,6 +1790,11 @@ const GraphExpandParamsJson = struct { algebraic_semiring: bool = false, }; +const GraphMetricReadJson = struct { + name: []const u8, + freshness: []const u8 = "published", +}; + const GraphTensorAccessPathJson = struct { owner: []const u8, layout: []const u8, @@ -1802,6 +1814,7 @@ const GraphExpansionJson = struct { total: u32, nodes: []const graph_query_mod.GraphResultNode, hits: []const db_mod.types.SearchHit = &.{}, + metric_status: []const db_mod.types.GraphMetricStatus = &.{}, }; const GraphHydrateRequestJson = struct { @@ -1940,7 +1953,7 @@ fn resultHasIdentitySnapshot( base_result.shard_identity_read_generations.len > 0; } -fn rejectUnstampedResultRefs( +pub fn rejectUnstampedResultRefs( req: db_mod.types.SearchRequest, base_result: db_mod.types.SearchResult, ) !void { @@ -2236,6 +2249,7 @@ const QueryState = struct { name: []u8, nodes: std.ArrayListUnmanaged(graph_query_mod.GraphResultNode) = .empty, hits: std.ArrayListUnmanaged(db_mod.types.SearchHit) = .empty, + metric_status: std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus) = .empty, path_states: std.ArrayListUnmanaged(PathState) = .empty, seen: graph_node_identity.Map(void) = .{}, work_budget: ?*graph_pattern_mod.WorkBudget = null, @@ -2285,6 +2299,8 @@ const QueryState = struct { self.nodes.deinit(alloc); for (self.hits.items) |*hit| hit.deinit(alloc); self.hits.deinit(alloc); + for (self.metric_status.items) |*status| status.deinit(alloc); + self.metric_status.deinit(alloc); for (self.path_states.items) |*path_state| path_state.deinit(alloc); self.path_states.deinit(alloc); self.seen.deinit(alloc); @@ -4141,7 +4157,11 @@ fn executeDistributedTraverse( ) !db_mod.types.GraphSearchResult { const include_paths = graph_query.query.params.include_paths; const result_limit = graph_query.query.params.max_results; - const collection_limit = graph_query_mod.resultCollectionLimit(result_limit); + const defer_result_limit = graphMetricPostProcessingNeedsFullCandidateSet(graph_query.query); + const collection_limit = if (defer_result_limit) + graph_query_mod.graph_metric_candidate_limit + 1 + else + graph_query_mod.resultCollectionLimit(result_limit); const max_depth: u32 = switch (graph_query.query.query_type) { .neighbors => 1, .traverse => graph_query.query.params.max_depth, @@ -4242,6 +4262,7 @@ fn executeDistributedTraverse( for (step_result.expansions) |expansion| { const item = frontier[expansion.frontier_id]; const step_graph = expansion.graph_result; + try mergeGraphMetricStatuses(alloc, &state.metric_status, step_graph.metric_status); for (step_graph.nodes) |node| { const allowed = admitted[admitted_index]; @@ -4337,6 +4358,7 @@ fn executeDistributedTraverse( for (step_result.expansions) |expansion| { const item = frontier[expansion.frontier_id]; const step_graph = expansion.graph_result; + try mergeGraphMetricStatuses(alloc, &state.metric_status, step_graph.metric_status); for (step_graph.nodes) |node| { const allowed = admitted[admitted_index]; @@ -4425,6 +4447,7 @@ fn executeDistributedTraverse( for (step_result.expansions) |expansion| { const item = frontier[expansion.frontier_id]; const step_graph = expansion.graph_result; + try mergeGraphMetricStatuses(alloc, &state.metric_status, step_graph.metric_status); for (step_graph.nodes) |node| { const allowed = admitted[admitted_index]; @@ -4501,12 +4524,25 @@ fn executeDistributedTraverse( frontier = try next_frontier.toOwnedSlice(alloc); } + try validateDistributedGraphMetricExecutionStatus( + state.metric_status.items, + graph_query.query.order_by, + graph_query.query.where_metric, + ); + try filterDistributedGraphNodesByMetric(alloc, &state.nodes, graph_query.query.where_metric); + try orderDistributedGraphNodesByMetric(alloc, state.nodes.items, graph_query.query.order_by); const truncated = graph_query_mod.resultCountIsTruncated(state.nodes.items.len, result_limit); if (truncated) { const public_len: usize = @intCast(result_limit); for (state.nodes.items[public_len..]) |*node| node.deinit(alloc); state.nodes.items.len = public_len; } + try retainDistributedGraphProjectedMetrics(alloc, &state.nodes, graph_query.query.metrics); + if (!graph_query.query.include_metric_status) { + for (state.metric_status.items) |*status| status.deinit(alloc); + state.metric_status.deinit(alloc); + state.metric_status = .empty; + } const hydration_requested = graphResultHydrationRequested(req, graph_query.query); if (hydration_requested) { @@ -4537,9 +4573,11 @@ fn executeDistributedTraverse( const name = state.name; const nodes = try state.nodes.toOwnedSlice(alloc); const hits = try state.hits.toOwnedSlice(alloc); + const metric_status = try state.metric_status.toOwnedSlice(alloc); state.name = try alloc.alloc(u8, 0); state.nodes = .empty; state.hits = .empty; + state.metric_status = .empty; defer { alloc.free(state.name); state.deinitTransient(alloc); @@ -4552,6 +4590,7 @@ fn executeDistributedTraverse( .hits = hits, .total_hits = total_hits, .truncated = truncated, + .metric_status = metric_status, }; } @@ -9960,6 +9999,8 @@ fn makeGraphExpandRequestWithAlgebraicModeAndTargetConstraints( tensor_program = try graphTraversalTensorProgramEnvelopeAlloc(alloc, named_query.query.index_name, target_constraint_keys.len > 0); } + const execution_metrics = try graphMetricExecutionReadsAlloc(alloc, named_query.query); + errdefer freeGraphMetricReads(alloc, execution_metrics); const name = try alloc.dupe(u8, named_query.name); errdefer alloc.free(name); const index_name = try alloc.dupe(u8, named_query.query.index_name); @@ -9982,6 +10023,9 @@ fn makeGraphExpandRequestWithAlgebraicModeAndTargetConstraints( .exclude_edges = owned_exclude_edges, .target_constraint_keys = owned_target_constraint_keys, .params = params, + .metrics = execution_metrics, + .include_metric_status = execution_metrics.len > 0, + .defer_result_limit = graphMetricPostProcessingNeedsFullCandidateSet(named_query.query), .tensor_access_path = tensor_access_path, .tensor_program = tensor_program, }; @@ -10109,11 +10153,14 @@ pub fn frontierItemToSearchRequest( var params = req.params; params.edge_types = try dupConstStrings(alloc, req.params.edge_types); errdefer freeConstStrings(alloc, params.edge_types); + if (req.defer_result_limit) params.max_results = graph_query_mod.graph_metric_candidate_limit + 1; const name = try alloc.dupe(u8, req.name); errdefer alloc.free(name); const index_name = try alloc.dupe(u8, req.index_name); errdefer alloc.free(index_name); + const metrics = try dupGraphMetricReads(alloc, req.metrics); + errdefer freeGraphMetricReads(alloc, metrics); const graph_queries = try alloc.alloc(db_mod.types.NamedGraphQuery, 1); errdefer alloc.free(graph_queries); @@ -10124,6 +10171,8 @@ pub fn frontierItemToSearchRequest( .index_name = index_name, .start_nodes = .{ .keys = frontier_keys }, .params = params, + .metrics = metrics, + .include_metric_status = req.include_metric_status, }, }; @@ -10231,10 +10280,123 @@ pub fn freeExpandSearchRequest(alloc: std.mem.Allocator, req: db_mod.types.Searc } } freeConstStrings(alloc, graph_query.query.params.edge_types); + freeGraphMetricReads(alloc, graph_query.query.metrics); } if (req.graph_queries.len > 0) alloc.free(req.graph_queries); } +fn graphMetricReadJsonAlloc( + alloc: std.mem.Allocator, + metrics: []const graph_query_mod.GraphMetricRead, +) ![]GraphMetricReadJson { + if (metrics.len == 0) return @constCast((&[_]GraphMetricReadJson{})[0..]); + const out = try alloc.alloc(GraphMetricReadJson, metrics.len); + for (metrics, 0..) |metric, i| { + out[i] = .{ + .name = metric.name, + .freshness = switch (metric.freshness) { + .published => "published", + .fresh => "fresh", + }, + }; + } + return out; +} + +fn parseGraphMetricReads( + alloc: std.mem.Allocator, + metrics: []const GraphMetricReadJson, +) ![]graph_query_mod.GraphMetricRead { + if (metrics.len == 0) return @constCast((&[_]graph_query_mod.GraphMetricRead{})[0..]); + const out = try alloc.alloc(graph_query_mod.GraphMetricRead, metrics.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |metric| alloc.free(@constCast(metric.name)); + alloc.free(out); + } + for (metrics, 0..) |metric, i| { + out[i] = .{ + .name = try alloc.dupe(u8, metric.name), + .freshness = if (std.mem.eql(u8, metric.freshness, "fresh")) + .fresh + else if (std.mem.eql(u8, metric.freshness, "published")) + .published + else + return error.InvalidQueryRequest, + }; + initialized += 1; + } + return out; +} + +fn dupGraphMetricReads( + alloc: std.mem.Allocator, + metrics: []const graph_query_mod.GraphMetricRead, +) ![]graph_query_mod.GraphMetricRead { + if (metrics.len == 0) return @constCast((&[_]graph_query_mod.GraphMetricRead{})[0..]); + const out = try alloc.alloc(graph_query_mod.GraphMetricRead, metrics.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |metric| alloc.free(@constCast(metric.name)); + alloc.free(out); + } + for (metrics, 0..) |metric, i| { + out[i] = .{ + .name = try alloc.dupe(u8, metric.name), + .freshness = metric.freshness, + }; + initialized += 1; + } + return out; +} + +fn graphMetricExecutionReadsAlloc( + alloc: std.mem.Allocator, + query: graph_query_mod.GraphQuery, +) ![]graph_query_mod.GraphMetricRead { + var out = std.ArrayListUnmanaged(graph_query_mod.GraphMetricRead).empty; + errdefer { + for (out.items) |metric| alloc.free(@constCast(metric.name)); + out.deinit(alloc); + } + for (query.metrics) |metric| try appendGraphMetricExecutionRead(alloc, &out, metric.name, metric.freshness); + for (query.order_by) |metric| try appendGraphMetricExecutionRead(alloc, &out, metric.name, metric.freshness); + for (query.where_metric) |metric| try appendGraphMetricExecutionRead(alloc, &out, metric.name, metric.freshness); + return try out.toOwnedSlice(alloc); +} + +fn appendGraphMetricExecutionRead( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged(graph_query_mod.GraphMetricRead), + name: []const u8, + freshness: graph_query_mod.GraphMetricFreshness, +) !void { + for (out.items) |*existing| { + if (!std.mem.eql(u8, existing.name, name)) continue; + existing.freshness = stricterGraphMetricFreshness(existing.freshness, freshness); + return; + } + try out.append(alloc, .{ + .name = try alloc.dupe(u8, name), + .freshness = freshness, + }); +} + +fn stricterGraphMetricFreshness( + left: graph_query_mod.GraphMetricFreshness, + right: graph_query_mod.GraphMetricFreshness, +) graph_query_mod.GraphMetricFreshness { + return if (left == .fresh or right == .fresh) .fresh else .published; +} + +fn freeGraphMetricReads( + alloc: std.mem.Allocator, + metrics: []const graph_query_mod.GraphMetricRead, +) void { + for (metrics) |metric| alloc.free(@constCast(metric.name)); + if (metrics.len > 0) alloc.free(@constCast(metrics)); +} + pub fn encodeGraphExpandRequest(alloc: std.mem.Allocator, req: GraphExpandRequest) ![]u8 { try validateGraphExpandTensorAccessPath(alloc, req); var fragment_names: ?[][]const u8 = null; @@ -10264,6 +10426,8 @@ pub fn encodeGraphExpandRequest(alloc: std.mem.Allocator, req: GraphExpandReques .distance = item.distance, }; } + const metrics = try graphMetricReadJsonAlloc(alloc, req.metrics); + defer if (metrics.len > 0) alloc.free(metrics); const exclude_nodes = try alloc.alloc(GraphNodeIdentityJson, req.exclude_nodes.len); defer alloc.free(exclude_nodes); for (req.exclude_nodes, 0..) |identity, i| { @@ -10276,6 +10440,9 @@ pub fn encodeGraphExpandRequest(alloc: std.mem.Allocator, req: GraphExpandReques .exclude_nodes = exclude_nodes, .exclude_edges = req.exclude_edges, .target_constraint_keys = req.target_constraint_keys, + .metrics = metrics, + .include_metric_status = req.include_metric_status, + .defer_result_limit = req.defer_result_limit, .topology_epoch = req.topology_epoch, .identity_read_generation = req.identity_read_generation, .params = .{ @@ -10408,6 +10575,9 @@ pub fn parseGraphExpandRequest(alloc: std.mem.Allocator, body: []const u8) !Grap .exclude_nodes = exclude_nodes, .exclude_edges = exclude_edges, .target_constraint_keys = target_constraint_keys, + .metrics = try parseGraphMetricReads(alloc, parsed.value.metrics), + .include_metric_status = parsed.value.include_metric_status, + .defer_result_limit = parsed.value.defer_result_limit, .topology_epoch = parsed.value.topology_epoch, .identity_read_generation = identity_read_generation, .resolved_doc_filter = if (parsed_filter) |filter| filter.resolved_doc_filter else null, @@ -10453,6 +10623,7 @@ pub fn encodeGraphExpandResponse(alloc: std.mem.Allocator, res: GraphExpandRespo .total = @intCast(expansion.graph_result.total_hits), .nodes = expansion.graph_result.nodes, .hits = expansion.graph_result.hits, + .metric_status = expansion.graph_result.metric_status, }; } return try jsonStringifyAlloc(alloc, GraphExpandResponseJson{ .expansions = expansions }); @@ -10500,6 +10671,7 @@ pub fn parseGraphExpandResponse(alloc: std.mem.Allocator, body: []const u8) !Gra .paths = @constCast((&[_]db_mod.types.GraphPath{})[0..]), .hits = hits, .total_hits = expansion.total, + .metric_status = try cloneGraphMetricStatuses(alloc, expansion.metric_status), }, }; initialized += 1; @@ -10689,6 +10861,12 @@ pub fn filterGraphSearchResult( if (owned_hits.len > 0) alloc.free(owned_hits); } + const metric_status = try cloneGraphMetricStatuses(alloc, src.metric_status); + errdefer { + for (metric_status) |*status| status.deinit(alloc); + if (metric_status.len > 0) alloc.free(metric_status); + } + return .{ .name = name, .nodes = owned_nodes, @@ -10696,6 +10874,7 @@ pub fn filterGraphSearchResult( .matches = owned_matches, .hits = owned_hits, .total_hits = total_hits, + .metric_status = metric_status, }; } @@ -10832,7 +11011,7 @@ fn materializeResultNode( path_state_id: ?u32, ) !graph_query_mod.GraphResultNode { if (!include_paths) { - return try initGraphResultNode( + var result = try initGraphResultNode( alloc, node.key, node_table, @@ -10842,6 +11021,9 @@ fn materializeResultNode( null, null, ); + errdefer result.deinit(alloc); + result.metrics = try cloneGraphMetricValues(alloc, node.metrics); + return result; } const id = path_state_id orelse return error.InvalidQueryRequest; @@ -12120,6 +12302,375 @@ fn cloneSearchHits( return out; } +fn cloneGraphMetricValues( + alloc: std.mem.Allocator, + values: []const graph_query_mod.GraphMetricValue, +) ![]graph_query_mod.GraphMetricValue { + if (values.len == 0) return @constCast((&[_]graph_query_mod.GraphMetricValue{})[0..]); + const out = try alloc.alloc(graph_query_mod.GraphMetricValue, values.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*value| value.deinit(alloc); + alloc.free(out); + } + for (values, 0..) |value, i| { + out[i] = .{ + .name = try alloc.dupe(u8, value.name), + .score = value.score, + }; + initialized += 1; + } + return out; +} + +fn cloneGraphMetricStatuses( + alloc: std.mem.Allocator, + statuses: []const db_mod.types.GraphMetricStatus, +) ![]db_mod.types.GraphMetricStatus { + return db_mod.types.cloneGraphMetricStatuses(alloc, statuses); +} + +fn cloneGraphMetricStatus( + alloc: std.mem.Allocator, + source: db_mod.types.GraphMetricStatus, +) !db_mod.types.GraphMetricStatus { + return source.cloneAlloc(alloc); +} + +fn cloneGraphMetricBuildPageStatuses( + alloc: std.mem.Allocator, + source: []const db_mod.types.GraphMetricBuildPageStatus, +) ![]db_mod.types.GraphMetricBuildPageStatus { + return db_mod.types.cloneGraphMetricBuildPageStatuses(alloc, source); +} + +fn mergeGraphMetricStatuses( + alloc: std.mem.Allocator, + target: *std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus), + statuses: []const db_mod.types.GraphMetricStatus, +) !void { + for (statuses) |status| try mergeGraphMetricStatus(alloc, target, status); +} + +fn mergeGraphMetricStatus( + alloc: std.mem.Allocator, + statuses: *std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus), + source: db_mod.types.GraphMetricStatus, +) !void { + for (statuses.items) |*status| { + if (!std.mem.eql(u8, status.name, source.name)) continue; + try validateDistributedGraphMetricStatusCompatible(status.*, source); + if (status.published_generation != source.published_generation) return error.UnsupportedQueryRequest; + try mergeGraphMetricStatusInto(alloc, status, source); + return; + } + try statuses.append(alloc, try cloneGraphMetricStatus(alloc, source)); +} + +fn validateDistributedGraphMetricStatusCompatible( + existing: db_mod.types.GraphMetricStatus, + incoming: db_mod.types.GraphMetricStatus, +) !void { + if (existing.metadata_version != 0 and + incoming.metadata_version != 0 and + existing.metadata_version != incoming.metadata_version) + { + return error.UnsupportedQueryRequest; + } + if (existing.config_fingerprint != 0 and + incoming.config_fingerprint != 0 and + existing.config_fingerprint != incoming.config_fingerprint) + { + return error.UnsupportedQueryRequest; + } + if (!existing.edge_filter.equivalent(incoming.edge_filter)) return error.UnsupportedQueryRequest; +} + +fn mergeGraphMetricStatusInto( + alloc: std.mem.Allocator, + target: *db_mod.types.GraphMetricStatus, + source: db_mod.types.GraphMetricStatus, +) !void { + target.state = mergeGraphMetricState(target.state, source.state); + target.phase = mergeGraphMetricPhase(target.phase, source.phase); + target.metadata_version = mergeGraphMetricMetadataVersion(target.metadata_version, source.metadata_version); + target.config_fingerprint = mergeComparableGeneration(target.config_fingerprint, source.config_fingerprint); + target.maintenance_paused = target.maintenance_paused or source.maintenance_paused; + target.build_queued = target.build_queued or source.build_queued; + target.published_generation = mergeComparableGeneration(target.published_generation, source.published_generation); + target.edge_generation = @max(target.edge_generation, source.edge_generation); + target.target_edge_generation = @max(target.target_edge_generation, source.target_edge_generation); + target.queued_generation = @max(target.queued_generation, source.queued_generation); + target.building_generation = @max(target.building_generation, source.building_generation); + target.build_job_id = if (target.build_job_id == 0) source.build_job_id else if (source.build_job_id == 0 or source.build_job_id == target.build_job_id) target.build_job_id else 0; + target.build_started_at_ms = if (target.build_started_at_ms == 0) source.build_started_at_ms else if (source.build_started_at_ms == 0 or source.build_started_at_ms == target.build_started_at_ms) target.build_started_at_ms else @min(target.build_started_at_ms, source.build_started_at_ms); + target.build_iteration = @max(target.build_iteration, source.build_iteration); + target.build_lease_expires_at_ms = @max(target.build_lease_expires_at_ms, source.build_lease_expires_at_ms); + target.build_completed_units = @max(target.build_completed_units, source.build_completed_units); + target.build_total_units = @max(target.build_total_units, source.build_total_units); + if (target.build_worker_id.len == 0) { + target.build_worker_id = if (source.build_worker_id.len > 0) try alloc.dupe(u8, source.build_worker_id) else ""; + } else if (source.build_worker_id.len > 0 and !std.mem.eql(u8, target.build_worker_id, source.build_worker_id)) { + alloc.free(@constCast(target.build_worker_id)); + target.build_worker_id = try alloc.dupe(u8, "multiple"); + } + if (target.build_cursor.len == 0 and source.build_cursor.len > 0) { + target.build_cursor = try alloc.dupe(u8, source.build_cursor); + } + if (target.build_pages.len == 0 and source.build_pages.len > 0) { + target.build_pages = try cloneGraphMetricBuildPageStatuses(alloc, source.build_pages); + } else if (source.build_pages.len > 0) { + target.build_pages_truncated = true; + } + target.build_pages_truncated = target.build_pages_truncated or source.build_pages_truncated; + target.retry_count = @max(target.retry_count, source.retry_count); + if (target.last_error.len == 0 and source.last_error.len > 0) { + target.last_error = try alloc.dupe(u8, source.last_error); + } + target.progress = @min(target.progress, source.progress); + target.converged = target.converged and source.converged; + target.iterations_completed = @max(target.iterations_completed, source.iterations_completed); + target.delta = @max(target.delta, source.delta); + target.computed_at_ms = @max(target.computed_at_ms, source.computed_at_ms); +} + +fn mergeGraphMetricMetadataVersion(left: u32, right: u32) u32 { + if (left == 0) return right; + if (right == 0) return left; + if (left == right) return left; + return 0; +} + +fn mergeComparableGeneration(left: u64, right: u64) u64 { + if (left == 0) return right; + if (right == 0) return left; + if (left == right) return left; + return @min(left, right); +} + +fn mergeGraphMetricState( + left: graph_mod.GraphIndex.GraphMetricState, + right: graph_mod.GraphIndex.GraphMetricState, +) graph_mod.GraphIndex.GraphMetricState { + return if (graphMetricStateSeverity(right) > graphMetricStateSeverity(left)) right else left; +} + +fn graphMetricStateSeverity(state: graph_mod.GraphIndex.GraphMetricState) u8 { + return switch (state) { + .disabled => 5, + .failed => 4, + .building => 3, + .not_ready => 2, + .stale => 1, + .fresh => 0, + }; +} + +fn mergeGraphMetricPhase( + left: graph_mod.GraphIndex.GraphMetricBuildPhase, + right: graph_mod.GraphIndex.GraphMetricBuildPhase, +) graph_mod.GraphIndex.GraphMetricBuildPhase { + return if (graphMetricPhaseSeverity(right) > graphMetricPhaseSeverity(left)) right else left; +} + +fn graphMetricPhaseSeverity(phase: graph_mod.GraphIndex.GraphMetricBuildPhase) u8 { + return switch (phase) { + .cleanup_old_generations => 10, + .publish_generation, .publishing => 9, + .check_convergence => 8, + .hits_hub_reduce_ranks => 8, + .hits_hub_contributions => 8, + .reduce_ranks => 7, + .iterate_contributions, .computing => 6, + .initialize_ranks => 5, + .scan_edges_and_out_degree => 4, + .prepare_generation => 3, + .idle => 1, + .complete => 0, + }; +} + +fn validateDistributedGraphMetricExecutionStatus( + statuses: []const db_mod.types.GraphMetricStatus, + orders: []const graph_query_mod.GraphMetricOrder, + filters: []const graph_query_mod.GraphMetricFilter, +) !void { + for (orders) |order| try validateDistributedGraphMetricStatus(statuses, order.name, order.freshness); + for (filters) |filter| try validateDistributedGraphMetricStatus(statuses, filter.name, filter.freshness); +} + +fn validateDistributedGraphMetricStatus( + statuses: []const db_mod.types.GraphMetricStatus, + name: []const u8, + freshness: graph_query_mod.GraphMetricFreshness, +) !void { + for (statuses) |status| { + if (!std.mem.eql(u8, status.name, name)) continue; + if (status.published_generation == 0) return error.MetricNotReady; + if (freshness == .fresh and status.state != .fresh) return error.MetricStale; + return; + } + return error.UnsupportedQueryRequest; +} + +fn filterDistributedGraphNodesByMetric( + alloc: std.mem.Allocator, + nodes: *std.ArrayListUnmanaged(graph_query_mod.GraphResultNode), + filters: []const graph_query_mod.GraphMetricFilter, +) !void { + if (filters.len == 0 or nodes.items.len == 0) return; + var write_index: usize = 0; + for (nodes.items, 0..) |*node, i| { + if (graphNodePassesMetricFilters(node.*, filters)) { + if (write_index != i) nodes.items[write_index] = node.*; + write_index += 1; + } else { + node.deinit(alloc); + } + } + nodes.items.len = write_index; +} + +fn graphNodePassesMetricFilters( + node: graph_query_mod.GraphResultNode, + filters: []const graph_query_mod.GraphMetricFilter, +) bool { + for (filters) |filter| { + const score = graphNodeMetricScore(node, filter.name) orelse return false; + if (!graphMetricFilterMatches(score, filter)) return false; + } + return true; +} + +fn graphMetricFilterMatches(score: f64, filter: graph_query_mod.GraphMetricFilter) bool { + return switch (filter.op) { + .gt => score > filter.value, + .gte => score >= filter.value, + .lt => score < filter.value, + .lte => score <= filter.value, + .eq => score == filter.value, + .neq => score != filter.value, + }; +} + +const DistributedGraphMetricSortNode = struct { + node: graph_query_mod.GraphResultNode, + original_index: usize, +}; + +const DistributedGraphMetricSortContext = struct { + orders: []const graph_query_mod.GraphMetricOrder, + scores: []const ?f64, +}; + +fn orderDistributedGraphNodesByMetric( + alloc: std.mem.Allocator, + nodes: []graph_query_mod.GraphResultNode, + orders: []const graph_query_mod.GraphMetricOrder, +) !void { + if (orders.len == 0 or nodes.len == 0) return; + const score_count = std.math.mul(usize, nodes.len, orders.len) catch return error.QueryCandidateBudgetExceeded; + const scores = try alloc.alloc(?f64, score_count); + defer alloc.free(scores); + const sortable = try alloc.alloc(DistributedGraphMetricSortNode, nodes.len); + defer alloc.free(sortable); + for (nodes, 0..) |node, i| { + const node_scores = scores[i * orders.len ..][0..orders.len]; + for (orders, 0..) |order, order_index| { + node_scores[order_index] = graphNodeMetricScore(node, order.name); + } + sortable[i] = .{ .node = node, .original_index = i }; + } + std.mem.sort(DistributedGraphMetricSortNode, sortable, DistributedGraphMetricSortContext{ + .orders = orders, + .scores = scores, + }, distributedGraphMetricSortLessThan); + for (sortable, 0..) |item, i| nodes[i] = item.node; +} + +fn distributedGraphMetricSortLessThan( + context: DistributedGraphMetricSortContext, + left: DistributedGraphMetricSortNode, + right: DistributedGraphMetricSortNode, +) bool { + for (context.orders, 0..) |order, order_index| { + const cmp = compareOptionalGraphMetricScore( + context.scores[left.original_index * context.orders.len + order_index], + context.scores[right.original_index * context.orders.len + order_index], + order, + ); + if (cmp) |less| return less; + } + return left.original_index < right.original_index; +} + +fn compareOptionalGraphMetricScore( + left: ?f64, + right: ?f64, + order: graph_query_mod.GraphMetricOrder, +) ?bool { + if (left == null and right == null) return null; + if (left == null) return order.nulls == .first; + if (right == null) return order.nulls != .first; + if (left.? == right.?) return null; + return if (order.direction == .desc) left.? > right.? else left.? < right.?; +} + +fn graphNodeMetricScore(node: graph_query_mod.GraphResultNode, name: []const u8) ?f64 { + for (node.metrics) |metric| { + if (std.mem.eql(u8, metric.name, name)) return metric.score; + } + return null; +} + +fn graphMetricPostProcessingNeedsFullCandidateSet(query: graph_query_mod.GraphQuery) bool { + return query.where_metric.len > 0 or query.order_by.len > 0; +} + +fn limitDistributedGraphMetricPostProcessedNodes( + alloc: std.mem.Allocator, + nodes: *std.ArrayListUnmanaged(graph_query_mod.GraphResultNode), + max_results: u32, +) void { + const keep_count: usize = @intCast(max_results); + if (max_results == 0 or nodes.items.len <= keep_count) return; + for (nodes.items[keep_count..]) |*node| node.deinit(alloc); + nodes.items.len = keep_count; +} + +fn retainDistributedGraphProjectedMetrics( + alloc: std.mem.Allocator, + nodes: *std.ArrayListUnmanaged(graph_query_mod.GraphResultNode), + metrics: []const graph_query_mod.GraphMetricRead, +) !void { + for (nodes.items) |*node| { + if (metrics.len == 0) { + for (node.metrics) |*value| value.deinit(alloc); + if (node.metrics.len > 0) alloc.free(node.metrics); + node.metrics = @constCast((&[_]graph_query_mod.GraphMetricValue{})[0..]); + continue; + } + + const retained = try alloc.alloc(graph_query_mod.GraphMetricValue, metrics.len); + var initialized: usize = 0; + errdefer { + for (retained[0..initialized]) |*value| value.deinit(alloc); + alloc.free(retained); + } + for (metrics, 0..) |metric, i| { + retained[i] = .{ + .name = try alloc.dupe(u8, metric.name), + .score = graphNodeMetricScore(node.*, metric.name), + }; + initialized += 1; + } + + for (node.metrics) |*value| value.deinit(alloc); + if (node.metrics.len > 0) alloc.free(node.metrics); + node.metrics = retained; + } +} + fn cloneGraphNode( alloc: std.mem.Allocator, node: graph_query_mod.GraphResultNode, @@ -12146,6 +12697,7 @@ fn cloneGraphNode( .path_edges = path_edges, .provenance = provenance, .table = table, + .metrics = try cloneGraphMetricValues(alloc, node.metrics), }; } @@ -12531,6 +13083,52 @@ test "distributed graph expand request carries constrained semiring target progr try std.testing.expectError(error.InvalidQueryRequest, validateGraphExpandTensorAccessPath(alloc, parsed)); } +test "distributed graph expand request bounds deferred worker metric candidates" { + const alloc = std.testing.allocator; + var frontier = [_]FrontierState{.{ + .key = try alloc.dupe(u8, "doc:a"), + }}; + defer frontier[0].deinit(alloc); + const frontier_ids = [_]u32{0}; + + var req = try makeGraphExpandRequest(alloc, .{ + .name = "walk", + .query = .{ + .query_type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ + .edge_types = &.{"links"}, + .max_depth = 3, + .max_results = 1, + }, + .order_by = &.{.{ + .name = "pagerank", + .direction = .desc, + .freshness = .published, + }}, + }, + }, frontier[0..], frontier_ids[0..], &.{}, &.{}, false); + defer req.deinit(alloc); + try std.testing.expect(req.defer_result_limit); + try std.testing.expect(req.include_metric_status); + try std.testing.expectEqual(@as(u32, 1), req.params.max_results); + + const encoded = try encodeGraphExpandRequest(alloc, req); + defer alloc.free(encoded); + var parsed = try parseGraphExpandRequest(alloc, encoded); + defer parsed.deinit(alloc); + try std.testing.expect(parsed.defer_result_limit); + try std.testing.expectEqual(@as(u32, 1), parsed.params.max_results); + + const search_req = try frontierItemToSearchRequest(alloc, parsed, parsed.frontier[0]); + defer freeExpandSearchRequest(alloc, search_req); + try std.testing.expectEqual(graph_query_mod.graph_metric_candidate_limit + 1, search_req.graph_queries[0].query.params.max_results); + try std.testing.expect(search_req.graph_queries[0].query.include_metric_status); + try std.testing.expectEqual(@as(usize, 1), search_req.graph_queries[0].query.metrics.len); + try std.testing.expectEqualStrings("pagerank", search_req.graph_queries[0].query.metrics[0].name); +} + test "distributed graph detects semiring-enabled graph index config" { const alloc = std.testing.allocator; const FakeCatalog = struct { @@ -14885,3 +15483,146 @@ test "distributed graph fans out per-group expand and hydrate with worker io" { )); try std.testing.expectEqual(@as(u32, 2), state.expand_calls.load(.monotonic)); } + +test "distributed graph metric status merge validates metadata compatibility" { + const alloc = std.testing.allocator; + const left_types = [_][]const u8{ "cites", "mentions" }; + const reordered_types = [_][]const u8{ "mentions", "cites" }; + const different_types = [_][]const u8{"related"}; + + var statuses = std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus).empty; + defer { + for (statuses.items) |*status| status.deinit(alloc); + statuses.deinit(alloc); + } + + try mergeGraphMetricStatuses(alloc, &statuses, &.{ + .{ + .name = @constCast("pagerank"), + .state = .fresh, + .edge_filter = .{ .mode = .types, .types = &left_types }, + .metadata_version = 3, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }); + + try mergeGraphMetricStatuses(alloc, &statuses, &.{ + .{ + .name = @constCast("pagerank"), + .state = .fresh, + .edge_filter = .{ .mode = .types, .types = &reordered_types }, + .metadata_version = 3, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeGraphMetricStatuses(alloc, &statuses, &.{ + .{ + .name = @constCast("pagerank"), + .state = .fresh, + .edge_filter = .{ .mode = .types, .types = &reordered_types }, + .metadata_version = 4, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + })); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeGraphMetricStatuses(alloc, &statuses, &.{ + .{ + .name = @constCast("pagerank"), + .state = .fresh, + .edge_filter = .{ .mode = .types, .types = &different_types }, + .metadata_version = 3, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + })); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeGraphMetricStatuses(alloc, &statuses, &.{ + .{ + .name = @constCast("pagerank"), + .state = .not_ready, + .edge_filter = .{ .mode = .types, .types = &reordered_types }, + .metadata_version = 3, + .published_generation = 0, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 0.0, + .converged = false, + }, + })); +} + +test "distributed graph metric post processing applies max results after filter and order" { + const alloc = std.testing.allocator; + + var nodes = std.ArrayListUnmanaged(graph_query_mod.GraphResultNode).empty; + defer { + for (nodes.items) |*node| node.deinit(alloc); + nodes.deinit(alloc); + } + + try nodes.append(alloc, .{ + .key = try alloc.dupe(u8, "B"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "degree"), .score = 1.0 }, + }), + }); + try nodes.append(alloc, .{ + .key = try alloc.dupe(u8, "C"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "degree"), .score = 3.0 }, + }), + }); + try nodes.append(alloc, .{ + .key = try alloc.dupe(u8, "D"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "degree"), .score = 2.0 }, + }), + }); + + const filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "degree", + .op = .gte, + .value = 2.0, + .freshness = .published, + }}; + const orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "degree", + .direction = .desc, + .freshness = .published, + }}; + + try filterDistributedGraphNodesByMetric(alloc, &nodes, &filters); + try orderDistributedGraphNodesByMetric(alloc, nodes.items, &orders); + limitDistributedGraphMetricPostProcessedNodes(alloc, &nodes, 1); + + try std.testing.expectEqual(@as(usize, 1), nodes.items.len); + try std.testing.expectEqualStrings("C", nodes.items[0].key); +} diff --git a/zig/pkg/antfly/src/api/e2e.zig b/zig/pkg/antfly/src/api/e2e.zig index 5527132748..c904a93576 100644 --- a/zig/pkg/antfly/src/api/e2e.zig +++ b/zig/pkg/antfly/src/api/e2e.zig @@ -343,9 +343,20 @@ const IndexStatusSummary = struct { doc_count: ?u64 = null, node_count: ?u64 = null, edge_count: ?u64 = null, + metric_status: ?std.json.ArrayHashMap(GraphMetricStatusSummary) = null, }, }; +const GraphMetricStatusSummary = struct { + state: []const u8, + published_generation: u64 = 0, + edge_generation: u64 = 0, + converged: bool = false, + iterations_completed: u64 = 0, + delta: f64 = 0.0, + computed_at_ms: u64 = 0, +}; + fn startMetadataAdminListener( alloc: std.mem.Allocator, svc: *metadata_service.MetadataService, @@ -6521,7 +6532,9 @@ test "public api e2e supports graph queries" { var created = try client.createTable(base_uri, "docs", create_body); defer created.deinit(std.testing.allocator); - var graph_index_resp = try client.createTableIndex(base_uri, "docs", "graph_idx", "{\"name\":\"graph_idx\",\"type\":\"graph\"}"); + var graph_index_resp = try client.createTableIndex(base_uri, "docs", "graph_idx", + \\{"name":"graph_idx","type":"graph","metrics":{"pagerank":{"enabled":true,"refresh":"background","max_iterations":40,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}} + ); defer graph_index_resp.deinit(std.testing.allocator); var rounds: usize = 0; @@ -6538,6 +6551,31 @@ test "public api e2e supports graph queries" { var batch = try client.fetchBatch(base_uri, "docs", batch_body); defer batch.deinit(std.testing.allocator); + var graph_index_status = try client.fetchTableIndex(base_uri, "docs", "graph_idx"); + defer graph_index_status.deinit(std.testing.allocator); + var parsed_graph_index_status = try parseJsonBody(IndexStatusSummary, std.testing.allocator, graph_index_status.body); + defer parsed_graph_index_status.deinit(); + const metric_status = parsed_graph_index_status.value.status.metric_status orelse return error.TestUnexpectedResult; + const pagerank_status = metric_status.map.get("pagerank") orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("fresh", pagerank_status.state); + try std.testing.expect(pagerank_status.published_generation > 0); + try std.testing.expect(pagerank_status.edge_generation >= pagerank_status.published_generation); + try std.testing.expect(pagerank_status.iterations_completed > 0); + + var metric_query = try client.fetchQuery(base_uri, "docs", + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank","top_k":2,"metric_freshness":"fresh"}} + ); + defer metric_query.deinit(std.testing.allocator); + var parsed_metric = try std.json.parseFromSlice(metadata_openapi.QueryResponses, std.testing.allocator, metric_query.body, .{}); + defer parsed_metric.deinit(); + const metric_responses = parsed_metric.value.responses orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(@as(usize, 1), metric_responses.len); + const graph_metric_results = metric_responses[0].graph_metric_results orelse return error.TestUnexpectedResult; + const pagerank_result = graph_metric_results.map.get("pagerank") orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("fresh", pagerank_result.status.state); + try std.testing.expectEqual(@as(usize, 2), pagerank_result.scores.len); + try std.testing.expect(pagerank_result.scores[0].score >= pagerank_result.scores[1].score); + const graph_query_body = try test_contract_helpers.encodeGraphNeighborsQueryRequest( std.testing.allocator, "neighbors", diff --git a/zig/pkg/antfly/src/api/generated/mcp_query_input_schema.json b/zig/pkg/antfly/src/api/generated/mcp_query_input_schema.json index 7167977530..9d4fbc48ad 100644 --- a/zig/pkg/antfly/src/api/generated/mcp_query_input_schema.json +++ b/zig/pkg/antfly/src/api/generated/mcp_query_input_schema.json @@ -1 +1 @@ -{"type":"object","additionalProperties":false,"required":["tableName"],"properties":{"tableName":{"type":"string"},"queryRequest":{"anyOf":[{"type":"object","additionalProperties":false,"description":"Raw canonical Antfly query body for POST /tables/{tableName}/query. Use this to access the full OpenAPI query contract. Mutually exclusive with query shorthand arguments.","properties":{"table":{"type":"null"},"query":{"type":"object","additionalProperties":true},"full_text_search":{"type":"object","additionalProperties":true},"full_text_index":{"type":"string","minLength":1,"maxLength":256},"filter_query":{"type":"object","additionalProperties":true},"exclusion_query":{"type":"object","additionalProperties":true},"semantic_search":{"type":"string"},"embedding_template":{"type":"string"},"indexes":{"type":"array","items":{"type":"string"}},"embeddings":{"type":"object","additionalProperties":true},"fields":{"type":"array","items":{"type":"string"}},"hierarchy":{"type":"object","additionalProperties":false,"description":"Returns direct index matches with optional projected ancestor context, or groups\nthose matches at a hierarchy level through `group_by`. A group's nested `matches`\nprojection is independently bounded and defaults to three hits while the top-level\n`limit` continues to control the number of groups.\n\n`children` is a separate sequential-browsing operation. It enumerates every unit\nin the selected source revision, including units with no searchable chunk, and uses\nthe top-level `_sort`/`search_after` cursor contract.\n\nAncestor and nested-match field projections are always explicit to keep response\nsize predictable. The presence of this object selects the canonical contract:\nwithout `group_by` or `children`, including when the object is empty, direct index\nmatches are returned. `ancestors` only controls projected context and never changes result\ncardinality. Omit `hierarchy` entirely to retain the v0.2-compatible implicit\nsource-grouped result shape.\n","allOf":[{"not":{"description":"Relevance grouping and sequential child traversal are separate operations.","required":["group_by","children"]}},{"not":{"description":"Sequential child traversal cannot request per-hit ancestor hydration.","required":["children","ancestors"]}},{"not":{"description":"A source group already carries its projected source document.","required":["group_by","ancestors"],"properties":{"group_by":{"required":["level"],"properties":{"level":{"type":"string","enum":["source"]}}},"ancestors":{"required":["source"]}}}}],"properties":{"group_by":{"type":"object","additionalProperties":false,"required":["level"],"properties":{"level":{"type":"string","enum":["source","unit"],"description":"Hierarchy level used to group the records matched by the targeted index.\nUnit groups are relevance-ranked and do not accept `order_by`, `search_after`,\nor `search_before`; use `hierarchy.children` for sequential, cursor-paginated\nunit traversal.\n"},"matches":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"limit":{"type":"integer","format":"uint32","minimum":1,"maximum":100,"default":3,"description":"Maximum matching descendant hits attached to each group, independent of\nthe top-level query limit. Matches follow the effective query order, and\nthe group score is the score of its best matching descendant. The maximum\nbounds nested response growth. Group selection uses an adaptive candidate\nwindow, then each returned group is expanded with a separately bounded query,\nso a group with fewer matches never forces a global exhaustive scan. To bound\nexecution as well as response growth, grouped queries accept at most 100\ntop-level groups and 1,000 requested matches across the complete result page.\n"},"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include in each nested match. This projection is required because\ngrouped and matching records commonly have different schemas. Use an empty\narray to return match identity and hierarchy metadata without stored fields.\n"}}}}},"ancestors":{"type":"object","additionalProperties":false,"minProperties":1,"properties":{"source":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include from the hydrated hierarchy document. This projection is\nrequired whenever the ancestor is requested so hierarchy hydration cannot\naccidentally return an unbounded document. Use an empty array to return\nhierarchy identity without stored document fields.\n"}}},"unit":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include from the hydrated hierarchy document. This projection is\nrequired whenever the ancestor is requested so hierarchy hydration cannot\naccidentally return an unbounded document. Use an empty array to return\nhierarchy identity without stored document fields.\n"}}}}},"children":{"type":"object","additionalProperties":false,"required":["parent","level"],"properties":{"parent":{"type":"object","additionalProperties":false,"required":["level","id"],"properties":{"level":{"type":"string","enum":["source"]},"id":{"type":"string","minLength":1}}},"level":{"type":"string","enum":["unit"],"description":"Child level to enumerate. Unit traversal reads the versioned extraction\nhierarchy rather than a relevance index, so empty and failed units are included.\n"}}}}},"limit":{"type":"integer"},"offset":{"type":"integer"},"timeout_ms":{"type":"integer","minimum":0},"order_by":{"type":"array"},"search_after":{"type":"array","items":{}},"search_before":{"type":"array","items":{}},"filter_prefix":{"type":"string","format":"byte"},"distance_under":{"type":"number","format":"float"},"distance_over":{"type":"number","format":"float"},"search_effort":{"type":"number","format":"float","minimum":0,"maximum":1,"default":0.5},"merge_config":{"type":"object","additionalProperties":true},"count":{"type":"boolean"},"profile":{"type":"boolean"},"reranker":{"type":"object","additionalProperties":true},"aggregations":{"type":"object","additionalProperties":true},"graph_queries":{"type":"object","additionalProperties":true},"document_renderer":{"type":"string"},"pruner":{"type":"object","additionalProperties":true},"join":{"type":"object","additionalProperties":true},"foreign_sources":{"type":"object","additionalProperties":true}},"not":{"anyOf":[{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["group_by"]}}},{"not":{"required":["fields"]}}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"not":{"required":["fields"]}}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"not":{"required":["order_by"]}}]},{"required":["hierarchy","order_by"],"properties":{"hierarchy":{"required":["children"]},"order_by":{"not":{"type":"array","minItems":1,"maxItems":1,"items":{"type":"object","additionalProperties":false,"required":["field"],"properties":{"field":{"type":"string","enum":["_hierarchy.position"]},"desc":{"type":"boolean","enum":[false],"default":false}}}}}}},{"required":["hierarchy","limit"],"properties":{"hierarchy":{"required":["children"]},"limit":{"type":"integer","anyOf":[{"maximum":0},{"minimum":101}]}}},{"required":["hierarchy","search_after"],"properties":{"hierarchy":{"required":["children"]},"search_after":{"not":{"type":"array","minItems":2,"maxItems":2,"items":{"type":"string"}}}}},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["group_by"],"properties":{"group_by":{"required":["level"],"properties":{"level":{"type":"string","enum":["unit"]}}}}}}},{"anyOf":[{"required":["order_by"]},{"required":["search_after"]},{"required":["search_before"]}]}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"anyOf":[{"required":["query"]},{"required":["full_text_search"]},{"required":["full_text_index"]},{"required":["filter_query"]},{"required":["exclusion_query"]},{"required":["semantic_search"]},{"required":["embedding_template"]},{"required":["indexes"]},{"required":["embeddings"]},{"required":["offset"]},{"required":["search_before"]},{"required":["filter_prefix"]},{"required":["distance_under"]},{"required":["distance_over"]},{"required":["search_effort"]},{"required":["merge_config"]},{"required":["count"]},{"required":["profile"]},{"required":["reranker"]},{"required":["analyses"]},{"required":["aggregations"]},{"required":["graph_queries"]},{"required":["document_renderer"]},{"required":["pruner"]},{"required":["join"]},{"required":["foreign_sources"]}]}]}]},"allOf":[{"not":{"required":["table"],"properties":{"table":{"not":{"type":"null"}}}}}]},{"type":"null"}],"description":"Raw canonical Antfly query body for POST /tables/{tableName}/query. Use this to access the full OpenAPI query contract. Mutually exclusive with query shorthand arguments."},"fullTextSearch":{"anyOf":[{"oneOf":[{"type":"string"},{"type":"object","additionalProperties":true}],"description":"Full-text query string shorthand, or a generic full_text_search object."},{"type":"null"}]},"full_text_search":{"type":["object","null"],"additionalProperties":true},"fullTextSearchField":{"type":["string","null"],"description":"Field to search when fullTextSearch is a string shorthand."},"fullTextIndex":{"type":["string","null"],"minLength":1,"maxLength":256,"description":"Named full-text index used by fullTextSearch; omission uses the active schema index."},"semanticSearch":{"type":["string","null"]},"fields":{"type":["array","null"],"items":{"type":"string"}},"limit":{"type":["integer","null"],"description":"Maximum results; defaults to 10 in shorthand mode."},"orderBy":{"type":["array","null"]},"indexes":{"type":["array","null"],"items":{"type":"string"}},"filterPrefix":{"type":["string","null"]}},"not":{"anyOf":[{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextSearch"],"properties":{"fullTextSearch":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["full_text_search"],"properties":{"full_text_search":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextSearchField"],"properties":{"fullTextSearchField":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextIndex"],"properties":{"fullTextIndex":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["semanticSearch"],"properties":{"semanticSearch":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fields"],"properties":{"fields":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["limit"],"properties":{"limit":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["orderBy"],"properties":{"orderBy":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["indexes"],"properties":{"indexes":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["filterPrefix"],"properties":{"filterPrefix":{"not":{"type":"null"}}}}]}]},"allOf":[{"if":{"required":["fullTextIndex"],"properties":{"fullTextIndex":{"not":{"type":"null"}}}},"then":{"anyOf":[{"required":["fullTextSearch"],"properties":{"fullTextSearch":{"not":{"type":"null"}}}},{"required":["full_text_search"],"properties":{"full_text_search":{"not":{"type":"null"}}}}]}}]} +{"type":"object","additionalProperties":false,"required":["tableName"],"properties":{"tableName":{"type":"string"},"queryRequest":{"anyOf":[{"type":"object","additionalProperties":false,"description":"Raw canonical Antfly query body for POST /tables/{tableName}/query. Use this to access the full OpenAPI query contract. Mutually exclusive with query shorthand arguments.","properties":{"table":{"type":"null"},"query":{"type":"object","additionalProperties":true},"full_text_search":{"type":"object","additionalProperties":true},"full_text_index":{"type":"string","minLength":1,"maxLength":256},"filter_query":{"type":"object","additionalProperties":true},"exclusion_query":{"type":"object","additionalProperties":true},"semantic_search":{"type":"string"},"embedding_template":{"type":"string"},"indexes":{"type":"array","items":{"type":"string"}},"embeddings":{"type":"object","additionalProperties":true},"fields":{"type":"array","items":{"type":"string"}},"hierarchy":{"type":"object","additionalProperties":false,"description":"Returns direct index matches with optional projected ancestor context, or groups\nthose matches at a hierarchy level through `group_by`. A group's nested `matches`\nprojection is independently bounded and defaults to three hits while the top-level\n`limit` continues to control the number of groups.\n\n`children` is a separate sequential-browsing operation. It enumerates every unit\nin the selected source revision, including units with no searchable chunk, and uses\nthe top-level `_sort`/`search_after` cursor contract.\n\nAncestor and nested-match field projections are always explicit to keep response\nsize predictable. The presence of this object selects the canonical contract:\nwithout `group_by` or `children`, including when the object is empty, direct index\nmatches are returned. `ancestors` only controls projected context and never changes result\ncardinality. Omit `hierarchy` entirely to retain the v0.2-compatible implicit\nsource-grouped result shape.\n","allOf":[{"not":{"description":"Relevance grouping and sequential child traversal are separate operations.","required":["group_by","children"]}},{"not":{"description":"Sequential child traversal cannot request per-hit ancestor hydration.","required":["children","ancestors"]}},{"not":{"description":"A source group already carries its projected source document.","required":["group_by","ancestors"],"properties":{"group_by":{"required":["level"],"properties":{"level":{"type":"string","enum":["source"]}}},"ancestors":{"required":["source"]}}}}],"properties":{"group_by":{"type":"object","additionalProperties":false,"required":["level"],"properties":{"level":{"type":"string","enum":["source","unit"],"description":"Hierarchy level used to group the records matched by the targeted index.\nUnit groups are relevance-ranked and do not accept `order_by`, `search_after`,\nor `search_before`; use `hierarchy.children` for sequential, cursor-paginated\nunit traversal.\n"},"matches":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"limit":{"type":"integer","format":"uint32","minimum":1,"maximum":100,"default":3,"description":"Maximum matching descendant hits attached to each group, independent of\nthe top-level query limit. Matches follow the effective query order, and\nthe group score is the score of its best matching descendant. The maximum\nbounds nested response growth. Group selection uses an adaptive candidate\nwindow, then each returned group is expanded with a separately bounded query,\nso a group with fewer matches never forces a global exhaustive scan. To bound\nexecution as well as response growth, grouped queries accept at most 100\ntop-level groups and 1,000 requested matches across the complete result page.\n"},"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include in each nested match. This projection is required because\ngrouped and matching records commonly have different schemas. Use an empty\narray to return match identity and hierarchy metadata without stored fields.\n"}}}}},"ancestors":{"type":"object","additionalProperties":false,"minProperties":1,"properties":{"source":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include from the hydrated hierarchy document. This projection is\nrequired whenever the ancestor is requested so hierarchy hydration cannot\naccidentally return an unbounded document. Use an empty array to return\nhierarchy identity without stored document fields.\n"}}},"unit":{"type":"object","additionalProperties":false,"required":["fields"],"properties":{"fields":{"type":"array","items":{"type":"string"},"description":"Fields to include from the hydrated hierarchy document. This projection is\nrequired whenever the ancestor is requested so hierarchy hydration cannot\naccidentally return an unbounded document. Use an empty array to return\nhierarchy identity without stored document fields.\n"}}}}},"children":{"type":"object","additionalProperties":false,"required":["parent","level"],"properties":{"parent":{"type":"object","additionalProperties":false,"required":["level","id"],"properties":{"level":{"type":"string","enum":["source"]},"id":{"type":"string","minLength":1}}},"level":{"type":"string","enum":["unit"],"description":"Child level to enumerate. Unit traversal reads the versioned extraction\nhierarchy rather than a relevance index, so empty and failed units are included.\n"}}}}},"limit":{"type":"integer"},"offset":{"type":"integer"},"timeout_ms":{"type":"integer","minimum":0},"order_by":{"type":"array"},"search_after":{"type":"array","items":{}},"search_before":{"type":"array","items":{}},"filter_prefix":{"type":"string","format":"byte"},"distance_under":{"type":"number","format":"float"},"distance_over":{"type":"number","format":"float"},"search_effort":{"type":"number","format":"float","minimum":0,"maximum":1,"default":0.5},"merge_config":{"type":"object","additionalProperties":true},"count":{"type":"boolean"},"profile":{"type":"boolean"},"reranker":{"type":"object","additionalProperties":true},"aggregations":{"type":"object","additionalProperties":true},"graph_queries":{"type":"object","additionalProperties":true},"document_renderer":{"type":"string"},"pruner":{"type":"object","additionalProperties":true},"join":{"type":"object","additionalProperties":true},"foreign_sources":{"type":"object","additionalProperties":true}},"not":{"anyOf":[{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["group_by"]}}},{"not":{"required":["fields"]}}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"not":{"required":["fields"]}}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"not":{"required":["order_by"]}}]},{"required":["hierarchy","order_by"],"properties":{"hierarchy":{"required":["children"]},"order_by":{"not":{"type":"array","minItems":1,"maxItems":1,"items":{"type":"object","additionalProperties":false,"required":["field"],"properties":{"field":{"type":"string","enum":["_hierarchy.position"]},"desc":{"type":"boolean","enum":[false],"default":false}}}}}}},{"required":["hierarchy","limit"],"properties":{"hierarchy":{"required":["children"]},"limit":{"type":"integer","anyOf":[{"maximum":0},{"minimum":101}]}}},{"required":["hierarchy","search_after"],"properties":{"hierarchy":{"required":["children"]},"search_after":{"not":{"type":"array","minItems":2,"maxItems":2,"items":{"type":"string"}}}}},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["group_by"],"properties":{"group_by":{"required":["level"],"properties":{"level":{"type":"string","enum":["unit"]}}}}}}},{"anyOf":[{"required":["order_by"]},{"required":["search_after"]},{"required":["search_before"]}]}]},{"allOf":[{"required":["hierarchy"],"properties":{"hierarchy":{"required":["children"]}}},{"anyOf":[{"required":["query"]},{"required":["full_text_search"]},{"required":["full_text_index"]},{"required":["filter_query"]},{"required":["exclusion_query"]},{"required":["semantic_search"]},{"required":["embedding_template"]},{"required":["indexes"]},{"required":["embeddings"]},{"required":["offset"]},{"required":["search_before"]},{"required":["filter_prefix"]},{"required":["distance_under"]},{"required":["distance_over"]},{"required":["search_effort"]},{"required":["merge_config"]},{"required":["count"]},{"required":["profile"]},{"required":["reranker"]},{"required":["graph_metric"]},{"required":["graph_metric_rerank"]},{"required":["analyses"]},{"required":["aggregations"]},{"required":["graph_queries"]},{"required":["document_renderer"]},{"required":["pruner"]},{"required":["join"]},{"required":["foreign_sources"]}]}]}]},"allOf":[{"not":{"required":["table"],"properties":{"table":{"not":{"type":"null"}}}}}]},{"type":"null"}],"description":"Raw canonical Antfly query body for POST /tables/{tableName}/query. Use this to access the full OpenAPI query contract. Mutually exclusive with query shorthand arguments."},"fullTextSearch":{"anyOf":[{"oneOf":[{"type":"string"},{"type":"object","additionalProperties":true}],"description":"Full-text query string shorthand, or a generic full_text_search object."},{"type":"null"}]},"full_text_search":{"type":["object","null"],"additionalProperties":true},"fullTextSearchField":{"type":["string","null"],"description":"Field to search when fullTextSearch is a string shorthand."},"fullTextIndex":{"type":["string","null"],"minLength":1,"maxLength":256,"description":"Named full-text index used by fullTextSearch; omission uses the active schema index."},"semanticSearch":{"type":["string","null"]},"fields":{"type":["array","null"],"items":{"type":"string"}},"limit":{"type":["integer","null"],"description":"Maximum results; defaults to 10 in shorthand mode."},"orderBy":{"type":["array","null"]},"indexes":{"type":["array","null"],"items":{"type":"string"}},"filterPrefix":{"type":["string","null"]}},"not":{"anyOf":[{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextSearch"],"properties":{"fullTextSearch":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["full_text_search"],"properties":{"full_text_search":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextSearchField"],"properties":{"fullTextSearchField":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fullTextIndex"],"properties":{"fullTextIndex":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["semanticSearch"],"properties":{"semanticSearch":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["fields"],"properties":{"fields":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["limit"],"properties":{"limit":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["orderBy"],"properties":{"orderBy":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["indexes"],"properties":{"indexes":{"not":{"type":"null"}}}}]},{"allOf":[{"required":["queryRequest"],"properties":{"queryRequest":{"not":{"type":"null"}}}},{"required":["filterPrefix"],"properties":{"filterPrefix":{"not":{"type":"null"}}}}]}]},"allOf":[{"if":{"required":["fullTextIndex"],"properties":{"fullTextIndex":{"not":{"type":"null"}}}},"then":{"anyOf":[{"required":["fullTextSearch"],"properties":{"fullTextSearch":{"not":{"type":"null"}}}},{"required":["full_text_search"],"properties":{"full_text_search":{"not":{"type":"null"}}}}]}}]} diff --git a/zig/pkg/antfly/src/api/http_client.zig b/zig/pkg/antfly/src/api/http_client.zig index 914e06fded..0951e66f95 100644 --- a/zig/pkg/antfly/src/api/http_client.zig +++ b/zig/pkg/antfly/src/api/http_client.zig @@ -1661,6 +1661,58 @@ pub const ApiHttpClient = struct { return .{ .body = try self.alloc.dupe(u8, resp.body) }; } + pub fn fetchGroupGraphMetricMaintenance( + self: *ApiHttpClient, + base_uri: []const u8, + group_id: u64, + table_name: []const u8, + body: []const u8, + ) !QueryResponse { + return self.fetchGroupGraphMetricMaintenanceWithCancellation(base_uri, group_id, table_name, body, null); + } + + pub fn fetchGroupGraphMetricMaintenanceWithCancellation( + self: *ApiHttpClient, + base_uri: []const u8, + group_id: u64, + table_name: []const u8, + body: []const u8, + cancellation: ?*const http_common.RequestCancellation, + ) !QueryResponse { + const suffix = try std.fmt.allocPrint(self.alloc, "{s}{s}{s}", .{ + routes.Routes.tables_prefix, + table_name, + routes.Routes.graph_metric_maintenance_suffix, + }); + defer self.alloc.free(suffix); + const path = try std.fmt.allocPrint(self.alloc, "{s}{d}{s}", .{ routes.Routes.internal_groups_prefix, group_id, suffix }); + defer self.alloc.free(path); + const uri = try self.joinRoute(base_uri, path); + defer self.alloc.free(uri); + + var resp = try self.executeRequest(.{ + .method = .POST, + .uri = uri, + .content_type = "application/json", + .body = body, + .cancellation = cancellation, + }); + defer resp.deinit(self.alloc); + switch (resp.status) { + 200 => {}, + 400 => if (std.mem.eql(u8, resp.body, @errorName(error.InvalidGraphMetricAction))) + return error.InvalidGraphMetricAction + else + return error.InvalidGraphMetricRuntimeConfig, + 404 => return error.UnknownGroup, + 405 => return error.UnsupportedOperation, + 409 => return remoteGroupConflictError(resp.body), + 503 => return error.LeaderUnavailable, + else => return error.UnexpectedHttpStatus, + } + return .{ .body = try self.alloc.dupe(u8, resp.body) }; + } + pub fn fetchGroupVectorWorker( self: *ApiHttpClient, base_uri: []const u8, @@ -3658,6 +3710,7 @@ fn isDocIdentityNamespaceMismatchConflictMessage(body: []const u8) bool { } fn remoteGroupConflictError(body: []const u8) anyerror { + if (std.mem.eql(u8, body, "IndexGenerationMismatch")) return error.IndexGenerationMismatch; if (std.mem.eql(u8, body, "DecisionConflict") or std.mem.eql(u8, body, "decision conflict")) return error.DecisionConflict; if (transactions_api.isTopologyChangedConflictMessage(body)) return error.TopologyChanged; if (std.mem.eql(u8, body, "TopologyChanged") or std.mem.eql(u8, body, "topology changed")) return error.TopologyChanged; @@ -3722,6 +3775,7 @@ test "api http client preserves public batch retry safety classifications" { } fn remoteStorageReadUnavailableError(body: []const u8) anyerror { + if (std.mem.eql(u8, body, "GenerationTransitionActive")) return error.GenerationTransitionActive; if (std.mem.eql(u8, body, "storage read temporarily unavailable")) { return error.StorageReadTemporarilyUnavailable; } @@ -4301,7 +4355,14 @@ test "api http client preserves group doc identity conflicts" { try std.testing.expectError(error.IdentityReadGenerationChanged, client.fetchGroupQuery(base_uri, 7, "docs", "{}")); try std.testing.expectError(error.IdentityReadGenerationChanged, client.fetchGroupGraphExpand(base_uri, 7, "docs", "{}")); + conflict_executor.body = "IndexGenerationMismatch"; + try std.testing.expectError(error.IndexGenerationMismatch, client.fetchGroupGraphHydrate(base_uri, 7, "docs", "{}")); + try std.testing.expectError(error.IndexGenerationMismatch, client.fetchGroupQuery(base_uri, 7, "docs", "{}")); + conflict_executor.status = 503; + conflict_executor.body = "GenerationTransitionActive"; + try std.testing.expectError(error.GenerationTransitionActive, client.fetchGroupQuery(base_uri, 7, "docs", "{}")); + try std.testing.expectError(error.GenerationTransitionActive, client.fetchGroupGraphHydrate(base_uri, 7, "docs", "{}")); conflict_executor.body = "write unavailable"; try std.testing.expectError(error.LeaderUnavailable, client.fetchGroupBatch(base_uri, 7, "docs", "{}")); @@ -4465,6 +4526,9 @@ test "api http client authenticates only the internal API namespace" { }); nested.deinit(std.testing.allocator); + var maintenance = try client.fetchGroupGraphMetricMaintenance("http://node:8080", 7, "docs", "{}"); + maintenance.deinit(std.testing.allocator); + capture.expected_internal = false; var public = try client.executeRequest(.{ .method = .GET, .uri = "http://node:8080/status" }); public.deinit(std.testing.allocator); diff --git a/zig/pkg/antfly/src/api/http_routes.zig b/zig/pkg/antfly/src/api/http_routes.zig index 3005ccd6cb..901473b501 100644 --- a/zig/pkg/antfly/src/api/http_routes.zig +++ b/zig/pkg/antfly/src/api/http_routes.zig @@ -102,6 +102,7 @@ pub const Routes = struct { pub const graph_expand_suffix = "/graph-expand"; pub const graph_hydrate_suffix = "/graph-hydrate"; pub const graph_edges_suffix = "/graph-edges"; + pub const graph_metric_maintenance_suffix = "/graph-metric-maintenance"; pub const vector_worker_suffix = "/vector-worker"; pub const txn_begin_suffix = "/txn-begin"; pub const txn_prepare_suffix = "/txn-prepare"; @@ -117,6 +118,7 @@ pub const Routes = struct { pub const schema_suffix = "/schema"; pub const indexes_suffix = "/indexes"; pub const indexes_marker = "/indexes/"; + pub const graph_metrics_marker = "/graph-metrics/"; pub const documents_suffix = "/documents"; pub const artifacts_suffix = "/artifacts"; pub const artifact_repair_suffix = "/repair/issues"; @@ -195,6 +197,13 @@ pub const Routes = struct { index_name: []const u8, }; + pub const TableGraphMetricAction = struct { + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + }; + pub const TableArtifact = struct { table_name: []const u8, artifact_name: []const u8, @@ -626,6 +635,32 @@ pub const Routes = struct { }; } + pub fn matchTableGraphMetricAction(path: []const u8) ?TableGraphMetricAction { + if (!std.mem.startsWith(u8, path, tables_prefix)) return null; + const rest = path[tables_prefix.len..]; + const indexes_index = std.mem.indexOf(u8, rest, indexes_marker) orelse return null; + if (indexes_index == 0) return null; + const table_name = rest[0..indexes_index]; + if (std.mem.indexOfScalar(u8, table_name, '/') != null) return null; + + const index_and_action = rest[indexes_index + indexes_marker.len ..]; + const graph_metrics_index = std.mem.indexOf(u8, index_and_action, graph_metrics_marker) orelse return null; + if (graph_metrics_index == 0) return null; + const index_name = index_and_action[0..graph_metrics_index]; + if (std.mem.indexOfScalar(u8, index_name, '/') != null) return null; + + const metric_and_action = index_and_action[graph_metrics_index + graph_metrics_marker.len ..]; + if (std.mem.indexOfScalar(u8, metric_and_action, '/') != null) return null; + const action_separator = std.mem.lastIndexOfScalar(u8, metric_and_action, ':') orelse return null; + if (action_separator == 0 or action_separator + 1 == metric_and_action.len) return null; + return .{ + .table_name = table_name, + .index_name = index_name, + .metric_name = metric_and_action[0..action_separator], + .action = metric_and_action[action_separator + 1 ..], + }; + } + pub fn matchTableDocumentArtifact(path: []const u8) ?TableDocumentArtifact { return matchTableDocumentArtifactWithReprocess(path, false); } @@ -1453,6 +1488,15 @@ test "public api routes compile" { try std.testing.expectEqualStrings("docs", index.table_name); try std.testing.expectEqualStrings("search_idx", index.index_name); try std.testing.expect(Routes.matchTableIndex("/tables/docs/indexes/search_idx/algebraic") == null); + const graph_metric_action = Routes.matchTableGraphMetricAction("/tables/docs/indexes/graph_idx/graph-metrics/pagerank:rebuild").?; + try std.testing.expectEqualStrings("docs", graph_metric_action.table_name); + try std.testing.expectEqualStrings("graph_idx", graph_metric_action.index_name); + try std.testing.expectEqualStrings("pagerank", graph_metric_action.metric_name); + try std.testing.expectEqualStrings("rebuild", graph_metric_action.action); + try std.testing.expect(Routes.matchTableGraphMetricAction("/tables/docs/indexes/graph_idx/graph-metrics/pagerank") == null); + try std.testing.expect(Routes.matchTableGraphMetricAction("/tables/docs/indexes/graph_idx/graph-metrics/:rebuild") == null); + try std.testing.expect(Routes.matchTableGraphMetricAction("/tables/docs/indexes/graph_idx/graph-metrics/pagerank:") == null); + try std.testing.expect(Routes.matchTableGraphMetricAction("/tables/docs/indexes/graph_idx/graph-metrics/pagerank:rebuild/extra") == null); const artifact = Routes.matchTableDocumentArtifact("/tables/docs/documents/doc%2Fa/artifacts/document_units_v1").?; try std.testing.expectEqualStrings("docs", artifact.table_name); try std.testing.expectEqualStrings("doc%2Fa", artifact.key); diff --git a/zig/pkg/antfly/src/api/http_server.zig b/zig/pkg/antfly/src/api/http_server.zig index 65b362eb37..16e9347d12 100644 --- a/zig/pkg/antfly/src/api/http_server.zig +++ b/zig/pkg/antfly/src/api/http_server.zig @@ -10835,6 +10835,7 @@ pub const ApiHttpServer = struct { .execute_table_get_index = executePublicTableGetIndex, .execute_table_create_index = executePublicTableCreateIndex, .execute_table_delete_index = executePublicTableDeleteIndex, + .execute_table_graph_metric_action = executePublicTableGraphMetricAction, .execute_put_artifact_enrichment = executePublicPutArtifactEnrichment, .execute_delete_artifact_enrichment = executePublicDeleteArtifactEnrichment, .execute_list_artifact_enrichments = executePublicListArtifactEnrichments, @@ -11013,6 +11014,9 @@ pub const ApiHttpServer = struct { => return error.StorageReadTemporarilyUnavailable, error.ModelNotFound => return error.ModelNotFound, error.UnsupportedExactSort => return error.UnsupportedExactSort, + error.GraphMetricGlobalMaterializationRequired => return error.GraphMetricGlobalMaterializationRequired, + error.GraphMetricMaterializationRejected => return error.GraphMetricMaterializationRejected, + error.GraphMetricQueryBudgetExceeded => return error.GraphMetricQueryBudgetExceeded, error.QueryCandidateBudgetExceeded => return error.QueryCandidateBudgetExceeded, error.GraphWorkBudgetExceeded => return error.GraphWorkBudgetExceeded, error.GraphMinWeightDomainViolation => return error.GraphMinWeightDomainViolation, @@ -11203,6 +11207,9 @@ pub const ApiHttpServer = struct { error.UnsupportedQueryRequest => return unsupportedPublicTableQueryDispatchError(alloc, body), error.UnsupportedHierarchyGrouping => return error.UnsupportedHierarchyGrouping, error.UnsupportedExactSort => return error.UnsupportedExactSort, + error.GraphMetricGlobalMaterializationRequired => return error.GraphMetricGlobalMaterializationRequired, + error.GraphMetricMaterializationRejected => return error.GraphMetricMaterializationRejected, + error.GraphMetricQueryBudgetExceeded => return error.GraphMetricQueryBudgetExceeded, error.TableNotFound, error.NotFound => return error.NotFound, error.IdentityReadGenerationChanged => return error.IdentityReadGenerationChanged, error.HierarchyCursorStale => return error.HierarchyCursorStale, @@ -11268,6 +11275,9 @@ pub const ApiHttpServer = struct { error.UnsupportedQueryRequest => return unsupportedPublicTableQueryDispatchError(alloc, body), error.UnsupportedHierarchyGrouping => return error.UnsupportedHierarchyGrouping, error.UnsupportedExactSort => return error.UnsupportedExactSort, + error.GraphMetricGlobalMaterializationRequired => return error.GraphMetricGlobalMaterializationRequired, + error.GraphMetricMaterializationRejected => return error.GraphMetricMaterializationRejected, + error.GraphMetricQueryBudgetExceeded => return error.GraphMetricQueryBudgetExceeded, error.ModelNotFound => return error.ModelNotFound, error.QueryCandidateBudgetExceeded => return error.QueryCandidateBudgetExceeded, error.GraphWorkBudgetExceeded, @@ -11362,6 +11372,9 @@ pub const ApiHttpServer = struct { error.UnsupportedQueryRequest => return unsupportedPublicTableQueryDispatchError(alloc, body), error.UnsupportedHierarchyGrouping => return error.UnsupportedHierarchyGrouping, error.UnsupportedExactSort => return error.UnsupportedExactSort, + error.GraphMetricGlobalMaterializationRequired => return error.GraphMetricGlobalMaterializationRequired, + error.GraphMetricMaterializationRejected => return error.GraphMetricMaterializationRejected, + error.GraphMetricQueryBudgetExceeded => return error.GraphMetricQueryBudgetExceeded, error.TableNotFound => return error.NotFound, error.IdentityReadGenerationChanged => return error.IdentityReadGenerationChanged, error.HierarchyCursorStale => return error.HierarchyCursorStale, @@ -12095,10 +12108,28 @@ pub const ApiHttpServer = struct { const start_ns = retryMonotonicNs(retry_io); const retry_deadline_ns = retryDeadlineFromNative(retry_io, req.execution_deadline_ns); var attempts: u32 = 0; + var index_generation_retries: u8 = 0; while (true) : (attempts += 1) { try ensureRequestActive(req.cancellation); if (retryDeadlineExpired(retry_deadline_ns, retryMonotonicNs(retry_io))) return error.Timeout; return source.query(alloc, table_name, req, consistency) catch |err| switch (err) { + error.IndexGenerationMismatch => { + // Release the failed query's entire snapshot before one + // fresh attempt. Never retry just a reverse-edge probe: + // its base scan and cached negative routes share identity. + // Cap expensive graph replay independently of time-based + // storage retries; ongoing reconciliation is a retryable + // readiness response, not an internal failure or a loop. + try ensureRequestActive(req.cancellation); + const now_ns = retryMonotonicNs(retry_io); + if (retryDeadlineExpired(retry_deadline_ns, now_ns)) return error.Timeout; + if (index_generation_retries != 0) return error.IndexRebuilding; + index_generation_retries += 1; + const sleep_ns = boundedRetrySleepNs(retry_deadline_ns, now_ns, start_ns, retry_timeout_ns, retry_poll_ns) orelse return error.IndexRebuilding; + if (sleep_ns == 0) return error.Timeout; + try sleepNsCancellable(retry_io, sleep_ns, req.cancellation); + continue; + }, // FileNotFound surfaces when a read-only replica open races // with the writer reclaiming obsolete LSM runs; reopening // picks up a fresh manifest. TableReadChurn: the read cache @@ -12107,6 +12138,7 @@ pub const ApiHttpServer = struct { error.EndOfStream, error.FileNotFound, error.TableReadChurn, + error.GenerationTransitionActive, error.IdentityReadGenerationChanged, error.TopologyChanged, => { @@ -12114,7 +12146,7 @@ pub const ApiHttpServer = struct { std.log.warn("public table query read failed table={s} err={} attempt={d}", .{ table_name, err, attempts + 1 }); const now_ns = retryMonotonicNs(retry_io); if (retryDeadlineExpired(retry_deadline_ns, now_ns)) return error.Timeout; - const sleep_ns = boundedRetrySleepNs(retry_deadline_ns, now_ns, start_ns, retry_timeout_ns, retry_poll_ns) orelse return err; + const sleep_ns = boundedRetrySleepNs(retry_deadline_ns, now_ns, start_ns, retry_timeout_ns, retry_poll_ns) orelse return if (err == error.GenerationTransitionActive) error.StorageReadTemporarilyUnavailable else err; if (sleep_ns == 0) return error.Timeout; try sleepNsCancellable(retry_io, sleep_ns, req.cancellation); continue; @@ -13517,6 +13549,43 @@ pub const ApiHttpServer = struct { }) orelse error.MethodNotAllowed; } + fn executePublicTableGraphMetricAction( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + request: api_operation.RequestContext, + ) public_table_http.TableApi.ExecuteGraphMetricActionError![]u8 { + const self: *ApiHttpServer = @ptrCast(@alignCast(ptr)); + try ensureTableOperationActive(request); + const source = self.table_writes orelse return error.MethodNotAllowed; + var status = (source.graphMetricActionWithCancellation(alloc, table_name, index_name, metric_name, action, request.cancellation) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.HAReadOnlyStandby, + error.HAPromotedStandbyRequiresPrimaryOpen, + error.HAFencedPrimary, + => return error.NotLeader, + error.TableTransitionActive, + error.TableGenerationChanged, + error.GraphMetricDisabled, + error.GraphMetricStatusConflict, + error.GraphMetricActionPartialOutcome, + => return error.Conflict, + error.LeaderUnavailable, error.UnknownGroup => return error.NotLeader, + error.PersistentDescriptorAdmissionExhausted, error.ResourceBudgetExceeded, error.BackendRuntimeShuttingDown => return error.Backpressured, + error.InvalidGraphMetricAction => return error.InvalidGraphMetricAction, + error.TableNotFound, error.IndexNotFound, error.MetricNotReady => return error.NotFound, + else => { + std.log.err("public graph metric action failed table={s} index={s} metric={s} action={s} err={}", .{ table_name, index_name, metric_name, action, err }); + return error.InternalFailure; + }, + }) orelse return error.NotFound; + defer status.deinit(alloc); + return indexes_api.encodeGraphMetricStatusResponse(alloc, status) catch return error.InternalFailure; + } + fn executePublicClusterBackupList( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -13838,6 +13907,7 @@ pub const ApiHttpServer = struct { operation_control.ensureActive() catch |err| switch (err) { error.Canceled => return error.Canceled, error.Timeout => return error.DeadlineExceeded, + else => return trace.internal(err), }; if (lease_heartbeat.lost.load(.acquire)) return trace.internal(error.BackupAttemptLeaseLost); @@ -13937,6 +14007,7 @@ pub const ApiHttpServer = struct { operation_control.ensureActive() catch |err| switch (err) { error.Canceled => return error.Canceled, error.Timeout => return error.DeadlineExceeded, + else => return trace.internal(err), }; // Close the last renewal/publication race. If another process took // an expired lease, this conditional renewal fences publication. @@ -13957,6 +14028,7 @@ pub const ApiHttpServer = struct { operation_control.ensureActive() catch |err| switch (err) { error.Canceled => return error.Canceled, error.Timeout => return error.DeadlineExceeded, + else => return trace.internal(err), }; cluster_cleanup_safe = false; backups_api.writeClusterManifestToLocationWithIoAndCancellation( @@ -15199,6 +15271,21 @@ pub const ApiHttpServer = struct { error.UnsupportedFilterQueryRequest => try contextualPublicFilterQueryErrorResponseForBody(self.alloc, body, "filter_query", .unsupported), error.UnsupportedExclusionQueryRequest => try contextualPublicFilterQueryErrorResponseForBody(self.alloc, body, "exclusion_query", .unsupported), error.UnsupportedExactSort => try contextualUnsupportedExactSortResponse(self.alloc), + error.GraphMetricGlobalMaterializationRequired => contextual_operations.jsonWithStatus( + 422, + try public_table_http.graphMetricGlobalMaterializationRequiredBody(self.alloc), + false, + ), + error.GraphMetricMaterializationRejected => contextual_operations.jsonWithStatus( + 422, + try public_table_http.graphMetricMaterializationRejectedBody(self.alloc), + false, + ), + error.GraphMetricQueryBudgetExceeded => contextual_operations.jsonWithStatus( + 422, + try public_table_http.graphMetricQueryBudgetExceededBody(self.alloc), + false, + ), error.UnsupportedQueryRequest => if (queryBodyHasSortPageControls(self.alloc, body)) try contextualUnsupportedExactSortResponse(self.alloc) else @@ -19746,6 +19833,10 @@ pub fn requiredPermissionForRequest(alloc: std.mem.Allocator, method: http_commo if (routes.Routes.matchTableBatch(path)) |batch| return try tablePermission(alloc, batch.table_name, .write); if (routes.Routes.matchTableMerge(path)) |merge| return try tablePermission(alloc, merge.table_name, .write); if (routes.Routes.matchTableSchema(path)) |schema| return try tablePermission(alloc, schema.table_name, .admin); + if (routes.Routes.matchTableGraphMetricAction(path)) |metric_action| return switch (method) { + .POST => try tablePermission(alloc, metric_action.table_name, .admin), + .GET, .PUT, .DELETE => null, + }; if (routes.Routes.matchTableIndexes(path)) |indexes| return try tablePermission(alloc, indexes.table_name, switch (method) { .GET => .read, .POST => .admin, @@ -19910,6 +20001,38 @@ test "inference connection invocation requires inference write permission" { )) == null); } +test "graph metric operational actions require table admin permission" { + const alloc = std.testing.allocator; + const required = (try requiredPermissionForRequest( + alloc, + .POST, + "/tables/docs%20archive/indexes/graph_idx/graph-metrics/pagerank:delete", + )).?; + defer required.deinit(alloc); + try std.testing.expectEqual(usermgr.ResourceType.table, required.resource_type); + try std.testing.expectEqualStrings("docs archive", required.resource); + try std.testing.expectEqual(usermgr.PermissionType.admin, required.permission_type); + + const reader_permissions = [_]usermgr.Permission{.{ + .resource_type = .table, + .resource = @constCast("docs archive"), + .type = .read, + }}; + try std.testing.expect(!permissionsAllow(&reader_permissions, required.resource_type, required.resource, required.permission_type)); + + const admin_permissions = [_]usermgr.Permission{.{ + .resource_type = .table, + .resource = @constCast("docs archive"), + .type = .admin, + }}; + try std.testing.expect(permissionsAllow(&admin_permissions, required.resource_type, required.resource, required.permission_type)); + try std.testing.expect((try requiredPermissionForRequest( + alloc, + .GET, + "/tables/docs%20archive/indexes/graph_idx/graph-metrics/pagerank:delete", + )) == null); +} + test "internal service credentials cannot authorize public inference routes" { const FakeSource = struct { fn iface() StatusSource { @@ -23472,6 +23595,57 @@ test "api http retries identity generation and topology churn from a fresh query defer topology_response.deinit(std.testing.allocator); try std.testing.expectEqual(@as(u32, 2), reads.attempts); try std.testing.expectEqualStrings("{\"responses\":[]}", topology_response.json); + reads.attempts = 0; + reads.transient = error.GenerationTransitionActive; + var transition_response = (try ApiHttpServer.queryWithTransientReadRetry(std.testing.allocator, null, reads.source(), "docs", .{}, .read_index, .none)).?; + defer transition_response.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, 2), reads.attempts); +} + +test "api http index generation retry refreshes once and preserves readiness cancellation and deadlines" { + const FakeReads = struct { + attempts: usize = 0, + fail_count: usize = 1, + cancel: ?*std.atomic.Value(bool) = null, + cancel_at: usize = 1, + + fn source(self: *@This()) table_reads.TableReadSource { + return .{ .ptr = self, .vtable = &.{ .lookup = lookup, .scan = scan, .query = query } }; + } + fn lookup(_: *anyopaque, _: std.mem.Allocator, _: []const u8, _: []const u8, _: db_mod.types.LookupOptions, _: raft_mod.ReadConsistency) anyerror!?table_reads.LookupResponse { + return error.UnsupportedOperation; + } + fn scan(_: *anyopaque, _: std.mem.Allocator, _: []const u8, _: []const u8, _: []const u8, _: db_mod.types.ScanOptions, _: raft_mod.ReadConsistency) anyerror!?table_reads.ScanResponse { + return error.UnsupportedOperation; + } + fn query(ptr: *anyopaque, alloc: std.mem.Allocator, _: []const u8, _: db_mod.types.SearchRequest, _: raft_mod.ReadConsistency) anyerror!?query_api.QueryResponse { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.attempts += 1; + if (self.cancel) |cancel| if (self.attempts == self.cancel_at) cancel.store(true, .release); + if (self.attempts <= self.fail_count) return error.IndexGenerationMismatch; + return .{ .json = try alloc.dupe(u8, "{\"responses\":[]}") }; + } + }; + const alloc = std.testing.allocator; + var reads = FakeReads{}; + var response = (try ApiHttpServer.queryWithTransientReadRetry(alloc, null, reads.source(), "docs", .{}, .read_index, .none)).?; + defer response.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), reads.attempts); + + reads = .{ .fail_count = 100 }; + try std.testing.expectError(error.IndexRebuilding, ApiHttpServer.queryWithTransientReadRetry(alloc, null, reads.source(), "docs", .{}, .read_index, .none)); + try std.testing.expectEqual(@as(usize, 2), reads.attempts); + + for ([_]usize{ 1, 2 }) |cancel_at| { + var canceled = std.atomic.Value(bool).init(false); + reads = .{ .fail_count = 100, .cancel = &canceled, .cancel_at = cancel_at }; + try std.testing.expectError(error.Cancelled, ApiHttpServer.queryWithTransientReadRetry(alloc, null, reads.source(), "docs", .{ .cancellation = CancellationToken.fromAtomic(&canceled) }, .read_index, .none)); + try std.testing.expectEqual(cancel_at, reads.attempts); + } + + reads = .{}; + try std.testing.expectError(error.Timeout, ApiHttpServer.queryWithTransientReadRetry(alloc, null, reads.source(), "docs", .{ .execution_deadline_ns = 0 }, .read_index, .none)); + try std.testing.expectEqual(@as(usize, 0), reads.attempts); } test "api http maps missing physical index only for rebuilding lifecycle" { diff --git a/zig/pkg/antfly/src/api/httpx_handler.zig b/zig/pkg/antfly/src/api/httpx_handler.zig index 2156a31d89..6d48b6d1ab 100644 --- a/zig/pkg/antfly/src/api/httpx_handler.zig +++ b/zig/pkg/antfly/src/api/httpx_handler.zig @@ -1049,6 +1049,7 @@ pub const AntflyApiHandler = struct { try server.post(table_prefix ++ routes.graph_expand_suffix, httpx.Handler.bind(self, internalGraphExpand)); try server.post(table_prefix ++ routes.graph_hydrate_suffix, httpx.Handler.bind(self, internalGraphHydrate)); try server.post(table_prefix ++ routes.graph_edges_suffix, httpx.Handler.bind(self, internalGraphEdges)); + try server.post(table_prefix ++ routes.graph_metric_maintenance_suffix, httpx.Handler.bind(self, internalGraphMetricMaintenance)); try server.post(table_prefix ++ routes.text_stats_suffix, httpx.Handler.bind(self, internalTextStats)); try server.post(table_prefix ++ routes.algebraic_partials_suffix, httpx.Handler.bind(self, internalAlgebraicPartials)); try server.post(table_prefix ++ routes.routed_batch_suffix, httpx.Handler.bind(self, internalGroupRoutedBatch)); @@ -1865,6 +1866,14 @@ pub const AntflyApiHandler = struct { /// this is only the shared wire projection at the `httpx` boundary. fn sharedInternalHttpErrorSpec(err: anyerror) ?InternalHttpErrorSpec { return switch (err) { + error.IndexGenerationMismatch => .{ + .status = 409, + .message = "IndexGenerationMismatch", + }, + error.GenerationTransitionActive => .{ + .status = 503, + .message = "GenerationTransitionActive", + }, error.DocIdentityNamespaceMismatch => .{ .status = 409, .message = "doc identity namespace mismatch", @@ -1877,7 +1886,7 @@ pub const AntflyApiHandler = struct { }; } - fn internalGroupErrorResponse(ctx: *httpx.Context, err: internal_group_operations.Error) !httpx.Response { + fn internalGroupErrorResponse(ctx: *httpx.Context, err: anyerror) !httpx.Response { if (sharedInternalHttpErrorSpec(err)) |spec| return textResponse(ctx, spec.status, spec.message); return switch (err) { @@ -2780,6 +2789,22 @@ pub const AntflyApiHandler = struct { return jsonResponse(ctx, 200, encoded); } + fn internalGraphMetricMaintenance(self: *AntflyApiHandler, ctx: *httpx.Context) !httpx.Response { + var params = (try internalGroupTableParams(ctx)) orelse return textResponse(ctx, 400, "invalid path parameter"); + defer params.deinit(ctx.allocator); + try operationContext(ctx, null).ensureActive(); + const writes = self.api_server.table_writes orelse return textResponse(ctx, 404, "not found"); + const body = (try ctx.body()) orelse ""; + const json = (writes.graphMetricMaintenanceGroupLocal(ctx.allocator, params.group_id, params.table_name, body) catch |err| switch (err) { + error.InvalidGraphMetricRuntimeConfig, error.InvalidGraphMetricBuildWorker, error.InvalidGraphMetricAction => return textResponse(ctx, 400, @errorName(err)), + error.UnknownGroup, error.NotFound => return textResponse(ctx, 404, "not found"), + error.ReadOnly, error.StorageUnavailable => return textResponse(ctx, 503, @errorName(err)), + else => return internalGroupErrorResponse(ctx, err), + }) orelse return textResponse(ctx, 404, "not found"); + defer ctx.allocator.free(json); + return jsonResponse(ctx, 200, json); + } + fn internalTextStats(self: *AntflyApiHandler, ctx: *httpx.Context) !httpx.Response { var params = (try internalGroupTableParams(ctx)) orelse return textResponse(ctx, 400, "invalid path parameter"); defer params.deinit(ctx.allocator); @@ -6071,6 +6096,27 @@ pub const AntflyApiHandler = struct { return respondOwnedApiResponse(ctx, &resp); } + pub fn executeGraphMetricAction( + self: *AntflyApiHandler, + ctx: *httpx.Context, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) !httpx.Response { + var authenticated_identity: ?AuthenticatedIdentity = null; + defer if (authenticated_identity) |*identity| identity.deinit(self.api_server.alloc); + if (try self.authorizeRequest(ctx, &authenticated_identity)) |resp| return resp; + const decoded_table_name = (try decodePathParamOrBadRequest(ctx, table_name)) orelse return ctx.text("invalid path parameter"); + defer ctx.allocator.free(decoded_table_name); + const decoded_index_name = (try decodePathParamOrBadRequest(ctx, index_name)) orelse return ctx.text("invalid path parameter"); + defer ctx.allocator.free(decoded_index_name); + const decoded_metric_name = (try decodePathParamOrBadRequest(ctx, metric_name)) orelse return ctx.text("invalid path parameter"); + defer ctx.allocator.free(decoded_metric_name); + var resp = try public_table_http.handleTableGraphMetricAction(ctx.allocator, decoded_table_name, decoded_index_name, decoded_metric_name, action, self.api_server.tableApi(operationContext(ctx, authenticated_identity))); + return respondOwnedApiResponse(ctx, &resp); + } + pub fn putArtifactEnrichment(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8, artifact_name: []const u8) !httpx.Response { var authenticated_identity: ?AuthenticatedIdentity = null; defer if (authenticated_identity) |*identity| identity.deinit(self.api_server.alloc); @@ -7162,6 +7208,12 @@ const SchemaReconcileWriteSource = struct { }; test "typed internal HTTP errors preserve conflict semantics" { + const transition = AntflyApiHandler.sharedInternalHttpErrorSpec(error.GenerationTransitionActive).?; + try std.testing.expectEqual(@as(u16, 503), transition.status); + try std.testing.expectEqualStrings("GenerationTransitionActive", transition.message); + const stale_index = AntflyApiHandler.sharedInternalHttpErrorSpec(error.IndexGenerationMismatch).?; + try std.testing.expectEqual(@as(u16, 409), stale_index.status); + try std.testing.expectEqualStrings("IndexGenerationMismatch", stale_index.message); const spec = AntflyApiHandler.sharedInternalHttpErrorSpec(error.DocIdentityNamespaceMismatch).?; try std.testing.expectEqual(@as(u16, 409), spec.status); try std.testing.expectEqualStrings("doc identity namespace mismatch", spec.message); diff --git a/zig/pkg/antfly/src/api/indexes.zig b/zig/pkg/antfly/src/api/indexes.zig index 245d259c55..db540a069a 100644 --- a/zig/pkg/antfly/src/api/indexes.zig +++ b/zig/pkg/antfly/src/api/indexes.zig @@ -29,11 +29,22 @@ const indexes_openapi = @import("antfly_indexes_openapi"); const chunking_openapi = @import("antfly_chunking_openapi"); const chunking_api_openapi = @import("antfly_chunking_api_openapi"); const enrichment_config_validation = @import("../storage/db/enrichment/config_validation.zig"); +const query_contract = @import("query_contract.zig"); const public_index_contract = @import("public_index_contract.zig"); const index_repair_status = @import("../common/index_repair_status.zig"); const credential_safety = @import("../common/credential_safety.zig"); const table_index_config = @import("table_index_config.zig"); +pub fn encodeGraphMetricStatusResponse( + alloc: std.mem.Allocator, + status: db_mod.types.GraphMetricStatus, +) ![]u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const public_status = try query_contract.toOpenApiGraphMetricStatus(arena.allocator(), status); + return try std.json.Stringify.valueAlloc(alloc, .{ .status = public_status }, .{ .emit_null_optional_fields = false }); +} + pub fn parseCreateIndexRequest(alloc: std.mem.Allocator, body: []const u8) ![]u8 { if (body.len == 0) return error.InvalidCreateIndexRequest; var parsed = try std.json.parseFromSlice(std.json.Value, alloc, body, .{}); @@ -1401,9 +1412,13 @@ fn appendPublicConfigValue( // deny-list cannot safely project them into a public response, // so preserve the table-status invariant and omit the entire // write-only document. - if (public_index_contract.isWriteOnlyConfigField(entry.key_ptr.*)) continue; - if (isSensitivePublicConfigField(entry.key_ptr.*)) continue; - if (isSensitivePublicConfigValue(entry.key_ptr.*, entry.value_ptr.*)) continue; + // Metric map keys are user-owned names, not credential fields; + // their values are still projected through a closed schema. + if (object_shape != .graph_metrics) { + if (public_index_contract.isWriteOnlyConfigField(entry.key_ptr.*)) continue; + if (isSensitivePublicConfigField(entry.key_ptr.*)) continue; + if (isSensitivePublicConfigValue(entry.key_ptr.*, entry.value_ptr.*)) continue; + } if (!public_index_contract.createdFieldValueMatches(object_shape, entry.key_ptr.*, entry.value_ptr.*)) continue; const child_shape = public_index_contract.createdObjectShapeForChild(object_shape, entry.key_ptr.*); if (!public_index_contract.createdValueMatchesShape(child_shape, entry.value_ptr.*)) continue; @@ -1684,7 +1699,7 @@ fn appendIndexRuntimeStatus( defer alloc.free(key); try appendJsonString(alloc, out, key); try out.append(alloc, ':'); - try appendSingleIndexRuntimeStatus(alloc, out, index_type, item, item_runtime.stats.source_doc_count, embeddings_coverage_policy, embeddings_sparse, coverage_generation, coverage_config_hash, item_runtime.stats.async_indexing, if (index_type == .embeddings) item_runtime.stats.enrichment else null, item_runtime.stats.resolution, item_runtime.stats.promotion, item_runtime.stats.resolver_replay, item_runtime.metadata, runtime_status.statusHasRuntimeFacts(item_runtime)); + try appendSingleIndexRuntimeStatusWithGraphMetricRuntime(alloc, out, index_type, item, item_runtime.stats.source_doc_count, embeddings_coverage_policy, embeddings_sparse, coverage_generation, coverage_config_hash, item_runtime.stats.async_indexing, if (index_type == .embeddings) item_runtime.stats.enrichment else null, item_runtime.stats.graph_metric_runtime, item_runtime.stats.resolution, item_runtime.stats.promotion, item_runtime.stats.resolver_replay, item_runtime.metadata, runtime_status.statusHasRuntimeFacts(item_runtime)); } } if (expected_group_ids.len > 0) { @@ -1720,7 +1735,7 @@ fn appendIndexRuntimeStatus( return; }; canonicalizeConfiguredSourceReplay(&item, configured_sources); - try appendSingleIndexRuntimeStatus(alloc, out, index_type, item, item.table_doc_count, embeddings_coverage_policy, embeddings_sparse, coverage_generation, coverage_config_hash, item.async_indexing, if (index_type == .embeddings) item.enrichment else null, item.resolution, item.promotion, item.resolver_replay, null, item.runtime_present); + try appendSingleIndexRuntimeStatusWithGraphMetricRuntime(alloc, out, index_type, item, item.table_doc_count, embeddings_coverage_policy, embeddings_sparse, coverage_generation, coverage_config_hash, item.async_indexing, if (index_type == .embeddings) item.enrichment else null, item.graph_metric_runtime, item.resolution, item.promotion, item.resolver_replay, null, item.runtime_present); } fn appendMinimalIndexRuntimeStatus( @@ -2079,6 +2094,7 @@ const AggregatedIndexStatus = struct { async_indexing: db_mod.types.AsyncIndexingStats = .{}, enrichment: db_mod.types.EnrichmentStats = .{}, enrichment_observation_count: u64 = 0, + graph_metric_runtime: db_mod.types.GraphMetricRuntimeStats = .{}, resolution: db_mod.types.ReplayStageStats = .{}, promotion: db_mod.types.ReplayStageStats = .{}, resolver_replay: db_mod.types.ResolverReplayDiagnostics = .{}, @@ -2748,6 +2764,13 @@ fn aggregateIndexStatusIndexed( } } } + if (index_observation_fresh) { + // Graph-metric maintenance is a runtime-wide observation, not an + // index-incarnation artifact. Preserve fresh ownership and worker + // progress even while this index's replacement materialization is + // still converging; shard status exposes the same live fact. + aggregateGraphMetricRuntimeStats(&aggregate.graph_metric_runtime, runtime.stats.graph_metric_runtime); + } // Preserve immutable counters from an exact-incarnation cached // observation even when its owner heartbeat is stale. Convergence is // fenced independently below, so these remain progress facts rather @@ -3119,6 +3142,48 @@ fn projectionCheckpointStatusRank(status: []const u8) u8 { return 10; } +fn aggregateGraphMetricRuntimeStats(dst: *db_mod.types.GraphMetricRuntimeStats, src: db_mod.types.GraphMetricRuntimeStats) void { + const had_facts = dst.hasRuntimeFacts(); + dst.enabled = dst.enabled or src.enabled; + if (src.role) |role| { + if (!had_facts) { + dst.role = role; + } else if (dst.role) |current| { + if (current != role) dst.role = null; + } + } + dst.runtime_id_hash ^= src.runtime_id_hash; + dst.owner_id_hash ^= src.owner_id_hash; + dst.lease_key_hash ^= src.lease_key_hash; + dst.worker_id_hash ^= src.worker_id_hash; + dst.worker_count +|= src.worker_count; + dst.lease_owned = dst.lease_owned or src.lease_owned; + dst.has_lease = dst.has_lease or src.has_lease; + dst.acquisition_count +|= src.acquisition_count; + dst.takeover_count +|= src.takeover_count; + dst.lease_acquire_failures +|= src.lease_acquire_failures; + dst.lost_leases +|= src.lost_leases; + dst.last_acquired_ms = @max(dst.last_acquired_ms, src.last_acquired_ms); + dst.lease_expires_at_ms = @max(dst.lease_expires_at_ms, src.lease_expires_at_ms); + dst.lease_renew_after_ms = @max(dst.lease_renew_after_ms, src.lease_renew_after_ms); + dst.renewal_count +|= src.renewal_count; + dst.started = dst.started or src.started; + dst.shutdown = dst.shutdown or src.shutdown; + dst.notified = dst.notified or src.notified; + inline for (.{ + "ticks_started", "ticks_completed", "durable_progress_ticks", "idle_ticks", "error_ticks", + "total_retired_input_records", "last_retired_input_records", "total_metrics_scanned", "total_active_builds", "total_builds_started", + "total_worker_steps", "total_coordinator_steps", "total_pages_claimed", "total_pages_completed", "total_phases_advanced", + "total_published", "total_failed_builds", "last_metrics_scanned", "last_active_builds", "last_builds_started", + "last_worker_steps", "last_coordinator_steps", "last_pages_claimed", "last_pages_completed", "last_phases_advanced", + "last_published", "last_failed_builds", + }) |field_name| { + @field(dst, field_name) +|= @field(src, field_name); + } + if (dst.last_error_name == null and src.last_error_name != null) dst.last_error_name = src.last_error_name; + dst.last_budget_exhausted = dst.last_budget_exhausted or src.last_budget_exhausted; +} + fn aggregateTextMergeStats(dst: *db_mod.types.TextMergeStats, src: db_mod.types.TextMergeStats) void { dst.active_indexes += src.active_indexes; dst.active_segments += src.active_segments; @@ -5832,6 +5897,46 @@ fn appendSingleIndexRuntimeStatus( resolver_replay: db_mod.types.ResolverReplayDiagnostics, metadata: ?runtime_status.RuntimeStatusMetadata, runtime_present: bool, +) !void { + return appendSingleIndexRuntimeStatusWithGraphMetricRuntime( + alloc, + out, + index_type, + item, + table_doc_count, + embeddings_coverage_policy, + embeddings_sparse, + coverage_generation, + coverage_config_hash, + async_indexing, + enrichment, + .{}, + resolution, + promotion, + resolver_replay, + metadata, + runtime_present, + ); +} + +fn appendSingleIndexRuntimeStatusWithGraphMetricRuntime( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged(u8), + index_type: ApiIndexType, + item: anytype, + table_doc_count: u64, + embeddings_coverage_policy: EmbeddingsCoveragePolicy, + embeddings_sparse: bool, + coverage_generation: u64, + coverage_config_hash: u64, + async_indexing: db_mod.types.AsyncIndexingStats, + enrichment: ?db_mod.types.EnrichmentStats, + graph_metric_runtime: db_mod.types.GraphMetricRuntimeStats, + resolution: ?db_mod.types.ReplayStageStats, + promotion: ?db_mod.types.ReplayStageStats, + resolver_replay: db_mod.types.ResolverReplayDiagnostics, + metadata: ?runtime_status.RuntimeStatusMetadata, + runtime_present: bool, ) !void { const authority = classifyIndexObservation( item, @@ -6321,6 +6426,12 @@ fn appendSingleIndexRuntimeStatus( try out.appendSlice(alloc, ",\"result_nodes\":"); try appendIntValue(alloc, out, item.algebraic_graph_traversal_result_node_count); try out.appendSlice(alloc, "}}"); + if (graph_metric_runtime.hasRuntimeFacts()) { + const encoded_runtime = try std.json.Stringify.valueAlloc(alloc, graph_metric_runtime, .{ .emit_null_optional_fields = false }); + defer alloc.free(encoded_runtime); + try out.appendSlice(alloc, ",\"graph_metric_runtime\":"); + try out.appendSlice(alloc, encoded_runtime); + } } if (index_type == .algebraic) try appendAlgebraicIndexStatsFields(alloc, out, item); try out.appendSlice(alloc, ",\"replay_applied_sequence\":"); @@ -7545,6 +7656,22 @@ test "public index config encoders omit root write-only producer documents" { ); } +test "created graph metric configuration projects closed nested schemas" { + const alloc = std.testing.allocator; + const config = + \\{"type":"graph","metrics":{"api_key":{"kind":"pagerank","max_iterations":20,"edge_filter":{"types":["selected"],"secret":"private"},"credentials":"private"}}} + ; + const expected = + \\{"name":"graph_idx","type":"graph","metrics":{"api_key":{"kind":"pagerank","max_iterations":20,"edge_filter":{"types":["selected"]}}}} + ; + const created = try encodeCreatedIndexConfig(alloc, "graph_idx", config); + defer alloc.free(created); + try ant_json.testing.expectEqualJsonText(alloc, expected, created); + const stored = (try encodeSingleIndexConfig(alloc, "{\"graph_idx\":" ++ config ++ "}", "graph_idx")).?; + defer alloc.free(stored); + try ant_json.testing.expectEqualJsonText(alloc, expected, stored); +} + test "created graph index response projects closed nested schemas" { const config = \\{"type":"graph","template":{"client_value":"private-root-value"},"source":{"artifact":"relations_v1","path":"$.relations[*]","format":{"client_value":"private-format-value"},"client_value":"private-source-value","settings":{"opaque":"private-source-settings"},"nodes":{"model":"document","target":"{{ _item.target.text }}","client_value":"private-node-value"},"edge":{"type":"{{ _item.predicate }}","weight":0.75,"metadata":{"source":"extractor","api_key":"private-metadata-key","nested":{"label":"public","authorization":"private-authorization"}},"client_value":"private-edge-mapping-value"},"context":{"doc_fields":["title","body"],"client_value":"private-context-value"}},"artifact":{"name":"relations_v1","kind":"asset","source":{"type":"template","value":"{{ body }}","client_value":"private-source-value"},"content_type":{"client_value":"private-content-type"},"producer_json":{"provider":"private","api_key":"private-key"},"execution":{"batch_items":8,"settings":{"opaque":"private-execution-settings"}},"client_value":"private-artifact-value","settings":{"opaque":"private-artifact-settings"}},"algebraic_planning":{"bounded_traversal":{"law":"provenance_semiring","enabled":true,"client_value":"private-bounded-value"},"client_value":"private-planning-value"},"edge_types":[{"name":"mentions","field":"relations","topology":"graph","max_weight":0.9,"min_weight":0,"allow_self_loops":false,"required_metadata":["source","confidence"],"client_value":"private-edge-value"},{"name":"malformed","required_metadata":{"client_value":"private-metadata-value"}},{"name":{"client_value":"private-required-value"}}],"resolvers":["private-malformed-resolver",{"name":"kg","table":"entities","source_artifact":"relations_v1","resolution_artifact":"resolution_v1","key_template":"{{label}}","candidate_search":"prefix","candidate_limit":{"client_value":"private-limit-value"},"client_value":"private-resolver-value","settings":{"opaque":"private-resolver-settings"}}]} @@ -10201,3 +10328,268 @@ test "single embeddings index encoder keeps partial backfill active while indexe try std.testing.expect(std.mem.indexOf(u8, encoded, "\"replay_target_sequence\":1") != null); try std.testing.expect(std.mem.indexOf(u8, encoded, "\"replay_catch_up_required\":false") != null); } +test "index encoders expose graph metric runtime ownership summary" { + const alloc = std.testing.allocator; + const shard_a_indexes = try alloc.alloc(db_mod.types.DBIndexStats, 1); + defer alloc.free(shard_a_indexes); + shard_a_indexes[0] = .{ + .name = try alloc.dupe(u8, "graph_idx"), + .kind = .graph, + .edge_count = 12, + }; + defer alloc.free(shard_a_indexes[0].name); + + const shard_b_indexes = try alloc.alloc(db_mod.types.DBIndexStats, 1); + defer alloc.free(shard_b_indexes); + shard_b_indexes[0] = .{ + .name = try alloc.dupe(u8, "graph_idx"), + .kind = .graph, + .edge_count = 8, + }; + defer alloc.free(shard_b_indexes[0].name); + + const local_items = try alloc.alloc(runtime_status.LocalTableRuntimeStatus, 2); + defer alloc.free(local_items); + local_items[0] = .{ + .group_id = 7, + .metadata = .{ .source = .cached_snapshot, .freshness = .fresh }, + .stats = .{ + .doc_count = 4, + .index_count = 1, + .indexes = shard_a_indexes, + .graph_metric_runtime = .{ + .enabled = true, + .role = .worker_pool, + .owner_id_hash = 0x11, + .worker_id_hash = 0x21, + .worker_count = 2, + .lease_owned = true, + .has_lease = true, + .takeover_count = 1, + .ticks_started = 4, + .ticks_completed = 3, + .durable_progress_ticks = 2, + .total_pages_claimed = 5, + .total_pages_completed = 4, + .last_pages_claimed = 2, + .last_pages_completed = 1, + }, + }, + }; + local_items[1] = .{ + .group_id = 8, + .metadata = .{ .source = .cached_snapshot, .freshness = .fresh }, + .stats = .{ + .doc_count = 3, + .index_count = 1, + .indexes = shard_b_indexes, + .graph_metric_runtime = .{ + .enabled = true, + .role = .worker_pool, + .owner_id_hash = 0x22, + .worker_id_hash = 0x42, + .worker_count = 1, + .lost_leases = 3, + .ticks_started = 6, + .ticks_completed = 5, + .durable_progress_ticks = 4, + .total_pages_claimed = 7, + .total_pages_completed = 6, + .last_pages_claimed = 4, + .last_pages_completed = 3, + }, + }, + }; + var local_status = runtime_status.LocalTableRuntimeStatuses{ .items = local_items }; + + const snapshot: metadata_api.AdminSnapshot = .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .indexes_json = "{\"graph_idx\":{\"type\":\"graph\"}}", + .placement_role = "data", + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{})[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + }; + + const encoded = (try encodeSingleIndex(alloc, &snapshot, "docs", "graph_idx", &local_status)).?; + defer alloc.free(encoded); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"status\":{\"index_type\":\"graph\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"graph_metric_runtime\":{\"enabled\":true,\"role\":\"worker_pool\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"owner_id_hash\":51") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"worker_count\":3") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"takeover_count\":1") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"lost_leases\":3") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"ticks_started\":10") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"total_pages_claimed\":12") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"last_pages_completed\":4") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"shard_status\":{\"7\":{\"index_type\":\"graph\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"owner_id_hash\":17") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"8\":{\"index_type\":\"graph\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"owner_id_hash\":34") != null); +} + +test "index encoders expose mixed graph metric runtime roles without aggregate role" { + const alloc = std.testing.allocator; + const shard_a_indexes = try alloc.alloc(db_mod.types.DBIndexStats, 1); + defer alloc.free(shard_a_indexes); + shard_a_indexes[0] = .{ + .name = try alloc.dupe(u8, "graph_idx"), + .kind = .graph, + .edge_count = 12, + }; + defer alloc.free(shard_a_indexes[0].name); + + const shard_b_indexes = try alloc.alloc(db_mod.types.DBIndexStats, 1); + defer alloc.free(shard_b_indexes); + shard_b_indexes[0] = .{ + .name = try alloc.dupe(u8, "graph_idx"), + .kind = .graph, + .edge_count = 8, + }; + defer alloc.free(shard_b_indexes[0].name); + + const local_items = try alloc.alloc(runtime_status.LocalTableRuntimeStatus, 2); + defer alloc.free(local_items); + local_items[0] = .{ + .group_id = 7, + .metadata = .{ .source = .cached_snapshot, .freshness = .fresh }, + .stats = .{ + .doc_count = 4, + .index_count = 1, + .indexes = shard_a_indexes, + .graph_metric_runtime = .{ + .enabled = true, + .role = .coordinator, + .owner_id_hash = 0x11, + .worker_count = 0, + .has_lease = true, + .total_coordinator_steps = 3, + .total_retired_input_records = 512, + .last_retired_input_records = 256, + .total_published = 1, + }, + }, + }; + local_items[1] = .{ + .group_id = 8, + .metadata = .{ .source = .cached_snapshot, .freshness = .fresh }, + .stats = .{ + .doc_count = 3, + .index_count = 1, + .indexes = shard_b_indexes, + .graph_metric_runtime = .{ + .enabled = true, + .role = .worker_pool, + .owner_id_hash = 0x22, + .worker_id_hash = 0x42, + .worker_count = 2, + .has_lease = true, + .total_worker_steps = 5, + .total_pages_completed = 4, + }, + }, + }; + var local_status = runtime_status.LocalTableRuntimeStatuses{ .items = local_items }; + + const snapshot: metadata_api.AdminSnapshot = .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .indexes_json = "{\"graph_idx\":{\"type\":\"graph\"}}", + .placement_role = "data", + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{})[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + }; + + const encoded = (try encodeSingleIndex(alloc, &snapshot, "docs", "graph_idx", &local_status)).?; + defer alloc.free(encoded); + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, encoded, .{}); + defer parsed.deinit(); + + const aggregate_runtime = parsed.value.object.get("status").?.object.get("graph_metric_runtime").?.object; + try std.testing.expect(aggregate_runtime.get("role") == null); + try std.testing.expectEqual(@as(i64, 0x33), aggregate_runtime.get("owner_id_hash").?.integer); + try std.testing.expectEqual(@as(i64, 2), aggregate_runtime.get("worker_count").?.integer); + try std.testing.expectEqual(@as(i64, 3), aggregate_runtime.get("total_coordinator_steps").?.integer); + try std.testing.expectEqual(@as(i64, 512), aggregate_runtime.get("total_retired_input_records").?.integer); + try std.testing.expectEqual(@as(i64, 256), aggregate_runtime.get("last_retired_input_records").?.integer); + try std.testing.expectEqual(@as(i64, 5), aggregate_runtime.get("total_worker_steps").?.integer); + try std.testing.expectEqual(@as(i64, 4), aggregate_runtime.get("total_pages_completed").?.integer); + + const shard_status = parsed.value.object.get("shard_status").?.object; + const shard_a_runtime = shard_status.get("7").?.object.get("graph_metric_runtime").?.object; + const shard_b_runtime = shard_status.get("8").?.object.get("graph_metric_runtime").?.object; + try std.testing.expectEqualStrings("coordinator", shard_a_runtime.get("role").?.string); + try std.testing.expectEqualStrings("worker_pool", shard_b_runtime.get("role").?.string); +} + +test "graph metric status encoder exposes active build pages" { + var pages = [_]db_mod.types.GraphMetricBuildPageStatus{ + .{ + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 4, + .state = .leased, + .range_kind = .reverse_edges, + .worker_id = "worker-a", + .lease_expires_at_ms = 12345, + .attempt = 2, + .cursor = "edge:42", + .completed_units = 7, + .total_units = 11, + }, + .{ + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 5, + .state = .failed, + .range_kind = .reverse_edges, + .worker_id = "worker-b", + .attempt = 3, + .last_error = "boom", + }, + }; + + const encoded = try encodeGraphMetricStatusResponse(std.testing.allocator, .{ + .name = @constCast("pagerank"), + .state = .building, + .phase = .scan_edges_and_out_degree, + .config_fingerprint = std.math.maxInt(u64), + .build_job_id = 9, + .build_worker_id = "coordinator", + .build_cursor = "phase:scan", + .build_completed_units = 17, + .build_total_units = 100, + .build_pages = pages[0..], + .build_pages_truncated = true, + }); + defer std.testing.allocator.free(encoded); + + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"build_pages\":[") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"phase\":\"scan_edges_and_out_degree\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"state\":\"leased\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"state\":\"failed\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"range_kind\":\"reverse_edges\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"worker_id\":\"worker-a\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"lease_expires_at_ms\":12345") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"cursor\":\"edge:42\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"completed_units\":7") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"total_units\":11") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"last_error\":\"boom\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded, "\"build_pages_truncated\":true") != null); + var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + const status_object = parsed.value.object.get("status").?.object; + try std.testing.expectEqualStrings("ffffffffffffffff", status_object.get("config_fingerprint").?.string); +} diff --git a/zig/pkg/antfly/src/api/internal_group_operations.zig b/zig/pkg/antfly/src/api/internal_group_operations.zig index 557e6b80d0..eea873fa8d 100644 --- a/zig/pkg/antfly/src/api/internal_group_operations.zig +++ b/zig/pkg/antfly/src/api/internal_group_operations.zig @@ -26,6 +26,8 @@ const platform_time = @import("antfly_platform").time; pub const Error = operation.ApiError || error{ TopologyChanged, IdentityReadGenerationChanged, + IndexGenerationMismatch, + GenerationTransitionActive, HierarchyCursorStale, DocIdentityNamespaceMismatch, StorageReadTemporarilyUnavailable, @@ -144,6 +146,8 @@ pub const Operations = struct { error.Cancelled, error.Canceled => error.Canceled, error.TopologyChanged => error.TopologyChanged, error.IdentityReadGenerationChanged => error.IdentityReadGenerationChanged, + error.IndexGenerationMismatch => error.IndexGenerationMismatch, + error.GenerationTransitionActive => error.GenerationTransitionActive, error.DocIdentityNamespaceMismatch => error.DocIdentityNamespaceMismatch, error.StorageReadTemporarilyUnavailable => error.StorageReadTemporarilyUnavailable, error.CatalogRoutingUnavailable, @@ -898,6 +902,7 @@ pub const Operations = struct { return (reads.graphHydrateGroupLocal(alloc, group_id, table_name, input, .read_index) catch |err| { if (mapCommonReadError(err)) |mapped| return mapped; return switch (err) { + error.InvalidArgument => error.InvalidArgument, error.UnknownGroup, error.TableNotFound => error.NotFound, else => error.Internal, }; @@ -1670,6 +1675,8 @@ test "typed internal query workers preserve identity generation validation" { test "typed internal group reads preserve retryable resident storage failures" { const alloc = std.testing.allocator; + try std.testing.expectEqual(error.GenerationTransitionActive, Operations.mapCommonReadError(error.GenerationTransitionActive).?); + try std.testing.expectEqual(error.IndexGenerationMismatch, Operations.mapCommonReadError(error.IndexGenerationMismatch).?); try std.testing.expectEqual( error.DeadlineExceeded, Operations.mapCommonReadError(error.CatalogRoutingSnapshotTimeout).?, diff --git a/zig/pkg/antfly/src/api/openapi_contract.zig b/zig/pkg/antfly/src/api/openapi_contract.zig index 26be108ac6..7056ebba7e 100644 --- a/zig/pkg/antfly/src/api/openapi_contract.zig +++ b/zig/pkg/antfly/src/api/openapi_contract.zig @@ -462,6 +462,26 @@ pub fn expectPublicIndexRuntimeStatusMetadata() !void { try std.testing.expect(@hasField(indexes_generated.GraphIndexStats, "projection_checkpoint_config_fingerprint")); try std.testing.expect(@hasField(indexes_generated.GraphIndexStats, "checkpoint_replay_tail_sequence_count")); try std.testing.expect(@hasField(indexes_generated.GraphIndexStats, "repair_scan_issue_count")); + try std.testing.expect(@hasDecl(indexes_generated, "GraphMetricRuntimeStats")); + try std.testing.expect(@hasDecl(indexes_generated, "GraphMetricQuery")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricQuery, "name")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricQuery, "top_k")); + try std.testing.expect(@hasField(generated.QueryRequest, "graph_metric")); + try std.testing.expect(@hasField(metadata_generated.QueryRequest, "graph_metric")); + try std.testing.expect(@hasField(client_generated.QueryRequest, "graph_metric")); + try std.testing.expect(@hasField(indexes_generated.GraphIndexStats, "graph_metric_runtime")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "owner_id_hash")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "worker_count")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "takeover_count")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "lost_leases")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "lease_expires_at_ms")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "lease_renew_after_ms")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "renewal_count")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "total_pages_claimed")); + try std.testing.expect(@hasField(indexes_generated.GraphMetricRuntimeStats, "last_pages_completed")); + try std.testing.expect(@hasDecl(client_generated, "GraphMetricRuntimeStats")); + try std.testing.expect(@hasField(client_generated.GraphIndexStats, "graph_metric_runtime")); + try std.testing.expect(@hasField(client_generated.GraphMetricRuntimeStats, "owner_id_hash")); try std.testing.expect(@hasDecl(indexes_generated, "AlgebraicIndexStats")); try std.testing.expect(@hasField(indexes_generated.AlgebraicIndexStats, "index_type")); try std.testing.expect(@hasField(indexes_generated.AlgebraicIndexStats, "projection_checkpoint_status")); @@ -504,6 +524,52 @@ test "public index contract exposes runtime status metadata" { try expectPublicIndexRuntimeStatusMetadata(); } +test "indexes openapi parses graph metric runtime summary" { + const alloc = std.testing.allocator; + var parsed = try std.json.parseFromSlice(indexes_generated.IndexStats, alloc, + \\{"index_type":"graph","total_edges":4,"graph_metric_runtime":{"enabled":true,"role":"worker_pool","owner_id_hash":17,"worker_id_hash":23,"worker_count":3,"lease_owned":true,"has_lease":true,"takeover_count":2,"lost_leases":1,"ticks_started":9,"ticks_completed":8,"durable_progress_ticks":7,"total_pages_claimed":6,"total_pages_completed":5,"last_pages_claimed":4,"last_pages_completed":3}} + , .{ .allocate = .alloc_always, .ignore_unknown_fields = true }); + defer parsed.deinit(); + + switch (parsed.value) { + .graph_index_stats => |stats| { + const runtime = stats.graph_metric_runtime orelse return error.UnexpectedOpenApiVariant; + try std.testing.expect(runtime.enabled.?); + try std.testing.expectEqualStrings("worker_pool", runtime.role.?); + try std.testing.expectEqual(@as(i64, 17), runtime.owner_id_hash.?); + try std.testing.expectEqual(@as(i64, 3), runtime.worker_count.?); + try std.testing.expectEqual(@as(i64, 2), runtime.takeover_count.?); + try std.testing.expectEqual(@as(i64, 1), runtime.lost_leases.?); + try std.testing.expectEqual(@as(i64, 6), runtime.total_pages_claimed.?); + try std.testing.expectEqual(@as(i64, 3), runtime.last_pages_completed.?); + }, + else => return error.UnexpectedOpenApiVariant, + } +} + +test "client openapi parses graph metric runtime summary" { + const alloc = std.testing.allocator; + var parsed = try std.json.parseFromSlice(client_generated.IndexStats, alloc, + \\{"index_type":"graph","total_edges":4,"graph_metric_runtime":{"enabled":true,"role":"coordinator","owner_id_hash":99,"worker_count":1,"takeover_count":2,"lost_leases":1,"total_pages_claimed":6,"last_pages_completed":3}} + , .{ .allocate = .alloc_always, .ignore_unknown_fields = true }); + defer parsed.deinit(); + + switch (parsed.value) { + .graph_index_stats => |stats| { + const runtime = stats.graph_metric_runtime orelse return error.UnexpectedOpenApiVariant; + try std.testing.expect(runtime.enabled.?); + try std.testing.expectEqualStrings("coordinator", runtime.role.?); + try std.testing.expectEqual(@as(i64, 99), runtime.owner_id_hash.?); + try std.testing.expectEqual(@as(i64, 1), runtime.worker_count.?); + try std.testing.expectEqual(@as(i64, 2), runtime.takeover_count.?); + try std.testing.expectEqual(@as(i64, 1), runtime.lost_leases.?); + try std.testing.expectEqual(@as(i64, 6), runtime.total_pages_claimed.?); + try std.testing.expectEqual(@as(i64, 3), runtime.last_pages_completed.?); + }, + else => return error.UnexpectedOpenApiVariant, + } +} + test "indexes openapi parses algebraic status as algebraic stats" { const alloc = std.testing.allocator; var source = try std.json.parseFromSlice(std.json.Value, alloc, @@ -943,6 +1009,7 @@ test "metadata openapi module generates extractor surface for routed endpoints" var found_create_index = false; var found_drop_index = false; var found_get_index = false; + var found_execute_graph_metric_action = false; var found_put_artifact_enrichment = false; var found_delete_artifact_enrichment = false; for (server.routes) |route| { @@ -973,6 +1040,7 @@ test "metadata openapi module generates extractor surface for routed endpoints" if (std.mem.eql(u8, route.operation_id, "createIndex")) found_create_index = true; if (std.mem.eql(u8, route.operation_id, "dropIndex")) found_drop_index = true; if (std.mem.eql(u8, route.operation_id, "getIndex")) found_get_index = true; + if (std.mem.eql(u8, route.operation_id, "executeGraphMetricAction")) found_execute_graph_metric_action = true; if (std.mem.eql(u8, route.operation_id, "putArtifactEnrichment")) found_put_artifact_enrichment = true; if (std.mem.eql(u8, route.operation_id, "deleteArtifactEnrichment")) found_delete_artifact_enrichment = true; } @@ -1003,6 +1071,7 @@ test "metadata openapi module generates extractor surface for routed endpoints" try std.testing.expect(found_create_index); try std.testing.expect(found_drop_index); try std.testing.expect(found_get_index); + try std.testing.expect(found_execute_graph_metric_action); try std.testing.expect(found_put_artifact_enrichment); try std.testing.expect(found_delete_artifact_enrichment); } diff --git a/zig/pkg/antfly/src/api/operation.zig b/zig/pkg/antfly/src/api/operation.zig index 39902d23df..f19656b641 100644 --- a/zig/pkg/antfly/src/api/operation.zig +++ b/zig/pkg/antfly/src/api/operation.zig @@ -61,6 +61,47 @@ pub const AdmissionReservation = struct { } }; +/// Bounded, request-owned diagnostic context. Operations may populate this +/// while unwinding an error so ingress adapters can return actionable details +/// without thread-local state or heap allocation. +pub const GraphMetricRejectionDiagnostic = struct { + graph_index_name: [256]u8 = undefined, + graph_index_name_len: u16 = 0, + metric_name: [128]u8 = undefined, + metric_name_len: u8 = 0, + materializer_fingerprint: u64 = 0, + + pub fn graphIndexName(self: *const GraphMetricRejectionDiagnostic) []const u8 { + return self.graph_index_name[0..self.graph_index_name_len]; + } + + pub fn metricName(self: *const GraphMetricRejectionDiagnostic) []const u8 { + return self.metric_name[0..self.metric_name_len]; + } +}; + +pub const RequestDiagnostics = struct { + graph_metric_rejection: ?GraphMetricRejectionDiagnostic = null, + + pub fn recordGraphMetricRejection( + self: *RequestDiagnostics, + graph_index_name: []const u8, + metric_name: []const u8, + materializer_fingerprint: u64, + ) void { + var diagnostic = GraphMetricRejectionDiagnostic{ + .materializer_fingerprint = materializer_fingerprint, + }; + const graph_len = @min(graph_index_name.len, diagnostic.graph_index_name.len); + @memcpy(diagnostic.graph_index_name[0..graph_len], graph_index_name[0..graph_len]); + diagnostic.graph_index_name_len = @intCast(graph_len); + const metric_len = @min(metric_name.len, diagnostic.metric_name.len); + @memcpy(diagnostic.metric_name[0..metric_len], metric_name[0..metric_len]); + diagnostic.metric_name_len = @intCast(metric_len); + self.graph_metric_rejection = diagnostic; + } +}; + pub const RequestContext = struct { cancellation: CancellationToken = .none, /// Absolute monotonic deadline. This deliberately does not use a wall @@ -72,6 +113,9 @@ pub const RequestContext = struct { request_id: []const u8 = "", principal: ?Principal = null, admission: ?*AdmissionReservation = null, + /// Borrowed for the duration of the operation. This is deliberately + /// request-scoped: std.Io tasks may resume on a different worker thread. + diagnostics: ?*RequestDiagnostics = null, /// Durable hash of an externally sourced table definition that was /// authorized before asynchronous restore admission. destination_authorization_fingerprint: []const u8 = "", @@ -178,3 +222,20 @@ test "admission reservation releases exactly once" { reservation.release(); try std.testing.expectEqual(@as(usize, 1), counter.count); } + +test "serverless request diagnostics remain isolated across interleaved operations" { + var first = RequestDiagnostics{}; + var second = RequestDiagnostics{}; + + first.recordGraphMetricRejection("graph-a", "pagerank", 11); + second.recordGraphMetricRejection("graph-b", "degree", 22); + + const first_rejection = first.graph_metric_rejection.?; + const second_rejection = second.graph_metric_rejection.?; + try std.testing.expectEqualStrings("graph-a", first_rejection.graphIndexName()); + try std.testing.expectEqualStrings("pagerank", first_rejection.metricName()); + try std.testing.expectEqual(@as(u64, 11), first_rejection.materializer_fingerprint); + try std.testing.expectEqualStrings("graph-b", second_rejection.graphIndexName()); + try std.testing.expectEqualStrings("degree", second_rejection.metricName()); + try std.testing.expectEqual(@as(u64, 22), second_rejection.materializer_fingerprint); +} diff --git a/zig/pkg/antfly/src/api/public_index_contract.zig b/zig/pkg/antfly/src/api/public_index_contract.zig index 850bdf662b..2f368aac70 100644 --- a/zig/pkg/antfly/src/api/public_index_contract.zig +++ b/zig/pkg/antfly/src/api/public_index_contract.zig @@ -48,6 +48,9 @@ pub const CreatedObjectShape = enum { graph_context, graph_algebraic_planning, graph_bounded_traversal, + graph_metrics, + graph_metric, + graph_metric_filter, edge_types, edge_type, graph_resolvers, @@ -94,6 +97,7 @@ pub fn isAllowedConfigField(kind: Kind, field: []const u8) bool { std.mem.eql(u8, field, "execution") or std.mem.eql(u8, field, "sources"), .graph => std.mem.eql(u8, field, "summarizer") or + std.mem.eql(u8, field, "metrics") or std.mem.eql(u8, field, "template") or std.mem.eql(u8, field, "edge_types") or std.mem.eql(u8, field, "max_edges_per_document") or @@ -161,6 +165,8 @@ pub fn createdObjectShapeForRootField(kind: Kind, field: []const u8) CreatedObje .graph_artifact else if (std.mem.eql(u8, field, "algebraic_planning")) .graph_algebraic_planning + else if (std.mem.eql(u8, field, "metrics")) + .graph_metrics else if (std.mem.eql(u8, field, "edge_types")) .edge_types else if (std.mem.eql(u8, field, "resolvers")) @@ -203,6 +209,7 @@ fn createdObjectHasRequiredFields(shape: CreatedObjectShape, object: std.json.Ob .edge_type => &.{"name"}, .graph_resolver => &.{ "name", "table", "source_artifact", "resolution_artifact", "key_template" }, .graph_bounded_traversal => &.{"law"}, + .graph_metrics, .graph_metric, .graph_metric_filter => &.{}, .unrestricted, .enrichments, .artifact_sources, .full_text_sources, .graph_sources, .edge_types, .graph_resolvers, .graph_nodes, .graph_edge, .graph_context, .graph_algebraic_planning, .chunker_text, .chunker_audio, .index_execution, .execution_policy => &.{}, }; for (required_fields) |field| { @@ -214,6 +221,8 @@ fn createdObjectHasRequiredFields(shape: CreatedObjectShape, object: std.json.Ob pub fn createdObjectShapeForChild(parent: CreatedObjectShape, field: []const u8) CreatedObjectShape { return switch (parent) { + .graph_metrics => .graph_metric, + .graph_metric => if (std.mem.eql(u8, field, "edge_filter")) .graph_metric_filter else .unrestricted, .enrichment => if (std.mem.eql(u8, field, "execution")) .execution_policy else .unrestricted, .chunker => if (std.mem.eql(u8, field, "text")) .chunker_text @@ -243,6 +252,12 @@ pub fn createdObjectShapeForChild(parent: CreatedObjectShape, field: []const u8) pub fn isAllowedCreatedObjectField(shape: CreatedObjectShape, field: []const u8) bool { return switch (shape) { + .graph_metrics => field.len > 0 and std.mem.indexOfScalar(u8, field, 0) == null, + .graph_metric => std.mem.eql(u8, field, "enabled") or std.mem.eql(u8, field, "kind") or + std.mem.eql(u8, field, "refresh") or std.mem.eql(u8, field, "damping") or + std.mem.eql(u8, field, "tolerance") or std.mem.eql(u8, field, "max_iterations") or + std.mem.eql(u8, field, "edge_filter"), + .graph_metric_filter => std.mem.eql(u8, field, "mode") or std.mem.eql(u8, field, "types"), .unrestricted => true, .enrichments, .artifact_sources, .full_text_sources, .graph_sources, .edge_types, .graph_resolvers => false, .provider => isAllowedCreatedProviderField(field), @@ -308,6 +323,7 @@ pub fn rootFieldValueMatches(kind: Kind, field: []const u8, value: std.json.Valu value == .array else if (std.mem.eql(u8, field, "summarizer") or std.mem.eql(u8, field, "source") or + std.mem.eql(u8, field, "metrics") or std.mem.eql(u8, field, "artifact") or std.mem.eql(u8, field, "algebraic_planning")) value == .object @@ -321,6 +337,12 @@ pub fn rootFieldValueMatches(kind: Kind, field: []const u8, value: std.json.Valu /// object. `full_text_index` is the sole intentionally dynamic subtree. pub fn createdFieldValueMatches(shape: CreatedObjectShape, field: []const u8, value: std.json.Value) bool { return switch (shape) { + .graph_metrics => value == .object, + .graph_metric => graphMetricFieldValueMatches(field, value), + .graph_metric_filter => if (std.mem.eql(u8, field, "types")) + isNonEmptyStringArray(value) and value.array.items.len > 0 + else + isString(value) and std.mem.eql(u8, value.string, "all"), .unrestricted => true, .enrichments, .artifact_sources, .full_text_sources, .graph_sources, .edge_types, .graph_resolvers => false, .provider => providerFieldValueMatches(field, value), @@ -345,6 +367,28 @@ pub fn createdFieldValueMatches(shape: CreatedObjectShape, field: []const u8, va }; } +fn graphMetricFieldValueMatches(field: []const u8, value: std.json.Value) bool { + if (std.mem.eql(u8, field, "enabled")) return isBool(value); + if (std.mem.eql(u8, field, "edge_filter")) return value == .object; + if (std.mem.eql(u8, field, "max_iterations")) return value == .integer and value.integer > 0 and value.integer <= 1000; + if (std.mem.eql(u8, field, "damping") or std.mem.eql(u8, field, "tolerance")) { + const number: f64 = switch (value) { + .integer => |v| @floatFromInt(v), + .float => |v| v, + else => return false, + }; + return std.math.isFinite(number) and number > 0 and + (!std.mem.eql(u8, field, "damping") or number < 1); + } + if (!isString(value)) return false; + const allowed: []const []const u8 = if (std.mem.eql(u8, field, "refresh")) + &.{ "background", "manual" } + else + &.{ "pagerank", "degree", "eigenvector", "hits_authority", "hits_hub" }; + for (allowed) |name| if (std.mem.eql(u8, value.string, name)) return true; + return false; +} + fn providerFieldValueMatches(field: []const u8, value: std.json.Value) bool { if (std.mem.eql(u8, field, "models")) return isStringArray(value); if (std.mem.eql(u8, field, "dimension") or diff --git a/zig/pkg/antfly/src/api/public_table_http.zig b/zig/pkg/antfly/src/api/public_table_http.zig index ec5dbb4018..f051d9a9e5 100644 --- a/zig/pkg/antfly/src/api/public_table_http.zig +++ b/zig/pkg/antfly/src/api/public_table_http.zig @@ -106,6 +106,8 @@ pub const TableApi = struct { pub const ExecuteBatchError = error{ InvalidBatchRequest, UnsupportedSyncLevel, + GraphMetricFeatureNotEnabled, + GraphMetricMaterializationRejected, NotFound, Conflict, MethodNotAllowed, @@ -117,6 +119,7 @@ pub const TableApi = struct { OutcomeUnknown, CommittedPending, CommittedRepairRequired, + CommittedGraphMetricMaterializationRejected, WriteOutcomeUnknown, DocIdentityUnavailable, HAReadOnlyStandby, @@ -144,6 +147,10 @@ pub const TableApi = struct { IndexRebuilding, ModelNotFound, UnsupportedExactSort, + GraphMetricGlobalMaterializationRequired, + GraphMetricFeatureNotEnabled, + GraphMetricMaterializationRejected, + GraphMetricQueryBudgetExceeded, QueryCandidateBudgetExceeded, RerankerCandidateLimitExceeded, GraphWorkBudgetExceeded, @@ -279,6 +286,7 @@ pub const TableApi = struct { Conflict, MethodNotAllowed, InvalidIndexRequest, + GraphMetricConfigurationLimitExceeded, MissingEmbeddingArtifactEnrichment, MissingEmbeddingArtifactProducer, InvalidEmbeddingArtifactProducer, @@ -304,6 +312,18 @@ pub const TableApi = struct { InternalFailure, }; + pub const ExecuteGraphMetricActionError = error{ + Canceled, + DeadlineExceeded, + NotLeader, + Conflict, + Backpressured, + InvalidGraphMetricAction, + NotFound, + MethodNotAllowed, + InternalFailure, + }; + pub const ExecutePutArtifactEnrichmentError = error{ Canceled, DeadlineExceeded, @@ -462,6 +482,15 @@ pub const TableApi = struct { index_name: []const u8, request: operation.RequestContext, ) ExecuteDeleteIndexError!void, + execute_table_graph_metric_action: ?*const fn ( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + request: operation.RequestContext, + ) ExecuteGraphMetricActionError![]u8 = null, execute_put_artifact_enrichment: ?*const fn ( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -616,6 +645,19 @@ pub const TableApi = struct { return try self.vtable.execute_table_delete_index(self.ptr, alloc, table_name, index_name, self.request); } + pub fn executeTableGraphMetricAction( + self: TableApi, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) ExecuteGraphMetricActionError![]u8 { + try self.ensureActive(); + const fn_ptr = self.vtable.execute_table_graph_metric_action orelse return error.MethodNotAllowed; + return try fn_ptr(self.ptr, alloc, table_name, index_name, metric_name, action, self.request); + } + pub fn executePutArtifactEnrichment( self: TableApi, alloc: std.mem.Allocator, @@ -970,6 +1012,162 @@ fn unsupportedExactSortBody(alloc: std.mem.Allocator) ![]u8 { }, .{}); } +pub fn graphMetricGlobalMaterializationRequiredBody(alloc: std.mem.Allocator) ![]u8 { + return try std.json.Stringify.valueAlloc(alloc, .{ + .code = "graph_metric_global_materialization_required", + .message = "graph metric scoring is unavailable for multi-shard tables until a globally coordinated metric snapshot is published", + .retryable = false, + }, .{}); +} + +pub fn graphMetricMaterializationRejectedBody(alloc: std.mem.Allocator) ![]u8 { + return try std.json.Stringify.valueAlloc(alloc, .{ + .code = "graph_metric_materialization_rejected", + .message = "graph metric materialization exceeded the serverless work budget; narrow the graph or metric edge filter, then update the metric configuration to rebuild", + .reason = "build_budget_exceeded", + .retryable = false, + }, .{}); +} + +pub fn graphMetricFeatureNotEnabledBody(alloc: std.mem.Allocator) ![]u8 { + return try std.json.Stringify.valueAlloc(alloc, .{ + .code = "graph_metric_feature_not_enabled", + .message = "graph metric publication is not enabled for this serverless deployment; an operator must complete the manifest writer rollout before graph metric queries or full_index synchronization can be used", + .retryable = false, + }, .{}); +} + +pub fn graphMetricMaterializationRejectedBodyWithContext( + alloc: std.mem.Allocator, + graph_index_name: []const u8, + metric_name: []const u8, + materializer_fingerprint: u64, +) ![]u8 { + const policy = try std.fmt.allocPrint(alloc, "{x:0>16}", .{materializer_fingerprint}); + defer alloc.free(policy); + const message = try std.fmt.allocPrint( + alloc, + "graph metric '{s}' on index '{s}' exceeded the current serverless materialization policy; narrow the graph or edge filter, or change the metric configuration to rebuild", + .{ metric_name, graph_index_name }, + ); + defer alloc.free(message); + return try std.json.Stringify.valueAlloc(alloc, .{ + .code = "graph_metric_materialization_rejected", + .message = message, + .reason = "build_budget_exceeded", + .retryable = false, + .graph_index_name = graph_index_name, + .metric_name = metric_name, + .materializer_fingerprint = policy, + }, .{}); +} + +test "multi-shard graph metric rejection is actionable and non-retryable" { + const encoded = try graphMetricGlobalMaterializationRequiredBody(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const parsed = try std.json.parseFromSlice(struct { + code: []const u8, + message: []const u8, + retryable: bool, + }, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + try std.testing.expectEqualStrings("graph_metric_global_materialization_required", parsed.value.code); + try std.testing.expect(std.mem.indexOf(u8, parsed.value.message, "globally coordinated metric snapshot") != null); + try std.testing.expect(!parsed.value.retryable); +} + +test "serverless graph metric build-budget rejection is actionable and non-retryable" { + const encoded = try graphMetricMaterializationRejectedBody(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const parsed = try std.json.parseFromSlice(struct { + code: []const u8, + message: []const u8, + reason: []const u8, + retryable: bool, + }, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + try std.testing.expectEqualStrings("graph_metric_materialization_rejected", parsed.value.code); + try std.testing.expectEqualStrings("build_budget_exceeded", parsed.value.reason); + try std.testing.expect(!parsed.value.retryable); + try std.testing.expect(std.mem.indexOf(u8, parsed.value.message, "narrow the graph") != null); + + const contextual = try graphMetricMaterializationRejectedBodyWithContext(std.testing.allocator, "graph_idx", "pagerank", 0x1234); + defer std.testing.allocator.free(contextual); + const contextual_parsed = try std.json.parseFromSlice(struct { + graph_index_name: []const u8, + metric_name: []const u8, + materializer_fingerprint: []const u8, + }, std.testing.allocator, contextual, .{ .ignore_unknown_fields = true }); + defer contextual_parsed.deinit(); + try std.testing.expectEqualStrings("graph_idx", contextual_parsed.value.graph_index_name); + try std.testing.expectEqualStrings("pagerank", contextual_parsed.value.metric_name); + try std.testing.expectEqualStrings("0000000000001234", contextual_parsed.value.materializer_fingerprint); +} + +test "serverless graph metric rollout rejection is explicit and non-retryable" { + const encoded = try graphMetricFeatureNotEnabledBody(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const parsed = try std.json.parseFromSlice(struct { + code: []const u8, + message: []const u8, + retryable: bool, + }, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + try std.testing.expectEqualStrings("graph_metric_feature_not_enabled", parsed.value.code); + try std.testing.expect(std.mem.indexOf(u8, parsed.value.message, "manifest writer rollout") != null); + try std.testing.expect(!parsed.value.retryable); +} + +test "public table query maps multi-shard graph metric rejection" { + const Backend = struct { + fn iface() TableApi { + return .{ + .ptr = undefined, + .request = .{}, + .vtable = &.{ + .execute_table_batch = unsupportedBatch, + .execute_table_query_request = executeTableQueryRequest, + .execute_table_query_view = unsupportedQueryView, + .execute_table_backup = unsupportedBackup, + .execute_table_restore = unsupportedRestore, + .execute_table_list_indexes = unsupportedListIndexes, + .execute_table_get_index = unsupportedGetIndex, + .execute_table_create_index = unsupportedCreateIndex, + .execute_table_delete_index = unsupportedDeleteIndex, + }, + }; + } + + fn executeTableQueryRequest( + _: *anyopaque, + _: std.mem.Allocator, + _: []const u8, + _: []const u8, + _: ?[]const u8, + _: operation.RequestContext, + ) TableApi.ExecuteQueryError![]u8 { + return error.GraphMetricGlobalMaterializationRequired; + } + }; + + var response = try handleTableQueryRequest(std.testing.allocator, "docs", + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank"}} + , null, Backend.iface()); + defer response.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u16, 422), response.status); + try std.testing.expect(response.json); + + const parsed = try std.json.parseFromSlice(struct { + code: []const u8, + message: []const u8, + retryable: bool, + }, std.testing.allocator, response.body, .{}); + defer parsed.deinit(); + try std.testing.expectEqualStrings("graph_metric_global_materialization_required", parsed.value.code); + try std.testing.expect(std.mem.indexOf(u8, parsed.value.message, "globally coordinated metric snapshot") != null); + try std.testing.expect(!parsed.value.retryable); +} + fn queryCandidateBudgetExceededBody(alloc: std.mem.Allocator) ![]u8 { const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse db_mod.SortRejectionDiagnostic{ .reason = "candidate_budget_exceeded", @@ -1015,6 +1213,16 @@ pub fn graphWorkBudgetExceededBody(alloc: std.mem.Allocator) ![]u8 { }, .{}); } +pub fn graphMetricQueryBudgetExceededBody(alloc: std.mem.Allocator) ![]u8 { + return try std.json.Stringify.valueAlloc(alloc, .{ + .status = @as(u16, 422), + .@"error" = "graph_metric_query_budget_exceeded", + .message = "graph metric reads exceeded the request-wide I/O or memory budget", + .retryable = false, + .remediation = "request fewer graph metrics or candidates, reduce graph result breadth, or split the query", + }, .{}); +} + pub fn graphPathWeightDomainErrorBody(alloc: std.mem.Allocator) ![]u8 { const diagnostic = graph_path_weight_diagnostic.take() orelse return error.MissingGraphPathWeightDiagnostic; @@ -1370,6 +1578,16 @@ pub fn handleTableBatch( api.executeTableBatch(alloc, table_name, batch_req.req) catch |err| switch (err) { error.InvalidBatchRequest => return .{ .status = 400, .body = try alloc.dupe(u8, "invalid batch request") }, error.UnsupportedSyncLevel => return .{ .status = 400, .body = try alloc.dupe(u8, "unsupported sync_level") }, + error.GraphMetricFeatureNotEnabled => return .{ + .status = 422, + .body = try graphMetricFeatureNotEnabledBody(alloc), + .json = true, + }, + error.GraphMetricMaterializationRejected => return .{ + .status = 422, + .body = try graphMetricMaterializationRejectedBody(alloc), + .json = true, + }, error.NotFound => return .{ .status = 404, .body = try alloc.dupe(u8, "not found") }, error.Conflict => return .{ .status = 409, .body = try alloc.dupe(u8, "batch transaction conflicted") }, error.MethodNotAllowed => return .{ .status = 405, .body = try alloc.dupe(u8, "method not allowed") }, @@ -1420,6 +1638,16 @@ pub fn handleTableBatch( .body = try batch_api.encodeBatchResponse(alloc, batch_req.resultWithStatus("committed_repair_required")), .json = true, }, + error.CommittedGraphMetricMaterializationRejected => return .{ + .status = 202, + .body = try batch_api.encodeBatchResponse(alloc, batch_req.resultWithFailure(.{ + .code = "graph_metric_materialization_rejected", + .message = "graph metric materialization exceeded the serverless work budget; narrow the graph or metric edge filter, then update the metric configuration to rebuild", + .reason = "build_budget_exceeded", + .retryable = false, + })), + .json = true, + }, // Do not use a retryable 5xx: clients must reconcile an ambiguous // commit result instead of blindly replaying non-idempotent transforms. error.WriteOutcomeUnknown => return .{ .status = 409, .body = try alloc.dupe(u8, "write outcome unknown") }, @@ -1553,6 +1781,10 @@ pub fn handleTableQueryRequest( std.log.warn("public table query candidate budget exceeded table={s} err={}", .{ table_name, err }); return .{ .status = 422, .body = try queryCandidateBudgetExceededBody(alloc), .json = true }; }, + error.GraphMetricQueryBudgetExceeded => { + std.log.warn("public table graph metric request budget exceeded table={s}", .{table_name}); + return .{ .status = 422, .body = try graphMetricQueryBudgetExceededBody(alloc), .json = true }; + }, error.RerankerCandidateLimitExceeded => { std.log.warn("public table query exceeds reranker provider candidate limit table={s}", .{table_name}); return .{ .status = 422, .body = try rerankerCandidateLimitExceededBody(alloc), .json = true }; @@ -1661,6 +1893,18 @@ pub fn handleTableQueryRequest( std.log.warn("public table query unsupported exact sort table={s} err={}", .{ table_name, err }); return .{ .status = 422, .body = try unsupportedExactSortBody(alloc), .json = true }; }, + error.GraphMetricGlobalMaterializationRequired => { + std.log.info("public table query requires global graph metric materialization table={s}", .{table_name}); + return .{ .status = 422, .body = try graphMetricGlobalMaterializationRequiredBody(alloc), .json = true }; + }, + error.GraphMetricFeatureNotEnabled => { + std.log.info("public table query graph metric publication is not enabled table={s}", .{table_name}); + return .{ .status = 422, .body = try graphMetricFeatureNotEnabledBody(alloc), .json = true }; + }, + error.GraphMetricMaterializationRejected => { + std.log.info("public table query graph metric materialization rejected table={s}", .{table_name}); + return .{ .status = 422, .body = try graphMetricMaterializationRejectedBody(alloc), .json = true }; + }, error.Canceled => return error.Canceled, error.DeadlineExceeded => return error.DeadlineExceeded, error.InternalFailure => { @@ -2012,6 +2256,7 @@ pub fn handleTableCreateIndex( error.Conflict => return .{ .status = 409, .body = try alloc.dupe(u8, "{\"error\":\"table_mutation_conflict\",\"message\":\"table mutation conflict; retry request\",\"retryable\":true}"), .json = true }, error.MethodNotAllowed => return .{ .status = 405, .body = try alloc.dupe(u8, "{\"error\":\"method_not_allowed\",\"message\":\"method not allowed\",\"retryable\":false}"), .json = true }, error.InvalidIndexRequest => return .{ .status = 400, .body = try alloc.dupe(u8, "{\"error\":\"invalid_index_request\",\"message\":\"unsupported index configuration\",\"retryable\":false}"), .json = true }, + error.GraphMetricConfigurationLimitExceeded => return .{ .status = 400, .body = try alloc.dupe(u8, "{\"error\":\"graph_metric_configuration_limit_exceeded\",\"message\":\"graph metric configuration exceeds serverless limits (maximum 16 graph metric indexes, 16 metrics per graph, 64 metrics total, 64 edge types per filter, 256-byte index names, 128-byte metric names, and 256-byte edge type names)\",\"retryable\":false}"), .json = true }, error.MissingEmbeddingArtifactEnrichment => return .{ .status = 400, .body = try alloc.dupe(u8, "{\"error\":\"missing_embedding_artifact_enrichment\",\"message\":\"embedding index source has no matching embedding enrichment\",\"retryable\":false}"), .json = true }, error.MissingEmbeddingArtifactProducer => return .{ .status = 400, .body = try alloc.dupe(u8, "{\"error\":\"missing_embedding_artifact_producer\",\"message\":\"embedding enrichment has no producer configuration\",\"retryable\":false}"), .json = true }, error.InvalidEmbeddingArtifactProducer => return .{ .status = 400, .body = try alloc.dupe(u8, "{\"error\":\"invalid_embedding_artifact_producer\",\"message\":\"embedding enrichment producer is not runnable\",\"retryable\":false}"), .json = true }, @@ -2061,6 +2306,32 @@ pub fn handleTableDeleteIndex( return .{ .status = 201, .body = try alloc.dupe(u8, "{}"), .json = true }; } +pub fn handleTableGraphMetricAction( + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + api: TableApi, +) !OwnedResponse { + const response_body = api.executeTableGraphMetricAction(alloc, table_name, index_name, metric_name, action) catch |err| switch (err) { + error.Canceled, error.DeadlineExceeded => return err, + error.NotLeader => return err, + error.Conflict => return .{ .status = 409, .body = try alloc.dupe(u8, "table mutation conflict; retry request") }, + error.Backpressured => return .{ + .status = 429, + .body = try alloc.dupe(u8, "{\"code\":\"storage_resource_exhausted\",\"message\":\"storage descriptors are temporarily exhausted\",\"retryable\":true,\"retry_after_ms\":1000}"), + .json = true, + .retry_after_seconds = 1, + }, + error.InvalidGraphMetricAction => return .{ .status = 400, .body = try alloc.dupe(u8, "invalid graph metric action") }, + error.NotFound => return .{ .status = 404, .body = try alloc.dupe(u8, "not found") }, + error.MethodNotAllowed => return .{ .status = 405, .body = try alloc.dupe(u8, "method not allowed") }, + error.InternalFailure => return .{ .status = 500, .body = try alloc.dupe(u8, "graph metric action failed") }, + }; + return .{ .status = 200, .body = response_body, .json = true }; +} + pub fn handlePutArtifactEnrichment( alloc: std.mem.Allocator, table_name: []const u8, @@ -3288,6 +3559,40 @@ test "public table batch handler identifies committed repair-required writes" { try std.testing.expect(std.mem.indexOf(u8, resp.body, "\"status\":\"committed_repair_required\"") != null); } +test "serverless public table batch handler identifies committed graph metric rejections" { + const Backend = struct { + fn iface() TableApi { + return .{ .ptr = undefined, .request = .{}, .vtable = &.{ + .execute_table_batch = executeTableBatch, + .execute_table_query_request = unsupportedQueryRequest, + .execute_table_query_view = unsupportedQueryView, + .execute_table_backup = unsupportedBackup, + .execute_table_restore = unsupportedRestore, + .execute_table_list_indexes = unsupportedListIndexes, + .execute_table_get_index = unsupportedGetIndex, + .execute_table_create_index = unsupportedCreateIndex, + .execute_table_delete_index = unsupportedDeleteIndex, + } }; + } + + fn executeTableBatch(_: *anyopaque, _: std.mem.Allocator, _: []const u8, _: db_mod.types.BatchRequest, _: operation.RequestContext) TableApi.ExecuteBatchError!void { + return error.CommittedGraphMetricMaterializationRejected; + } + }; + + var resp = try handleTableBatch(std.testing.allocator, "docs", + \\{"inserts":{"doc-a":{"title":"alpha"}}} + , Backend.iface()); + defer resp.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u16, 202), resp.status); + try std.testing.expect(resp.json); + try ant_json.testing.expectEqualJsonText( + std.testing.allocator, + "{\"status\":\"committed_repair_required\",\"inserted\":1,\"deleted\":0,\"transformed\":0,\"failure\":{\"code\":\"graph_metric_materialization_rejected\",\"message\":\"graph metric materialization exceeded the serverless work budget; narrow the graph or metric edge filter, then update the metric configuration to rebuild\",\"reason\":\"build_budget_exceeded\",\"retryable\":false}}", + resp.body, + ); +} + test "public table batch handler preserves ambiguous write outcomes" { const Backend = struct { fn iface() TableApi { @@ -4180,6 +4485,60 @@ test "public table query handler maps candidate budget exhaustion" { try std.testing.expectEqualStrings("full_text_index_v0", parsed.value.sort_rejection_field); } +test "serverless public table query handler maps aggregate graph metric budget exhaustion" { + const Backend = struct { + fn iface() TableApi { + return .{ + .ptr = undefined, + .request = .{}, + .vtable = &.{ + .execute_table_batch = unsupportedBatch, + .execute_table_query_request = executeTableQueryRequest, + .execute_table_query_view = unsupportedQueryView, + .execute_table_backup = unsupportedBackup, + .execute_table_restore = unsupportedRestore, + .execute_table_list_indexes = unsupportedListIndexes, + .execute_table_get_index = unsupportedGetIndex, + .execute_table_create_index = unsupportedCreateIndex, + .execute_table_delete_index = unsupportedDeleteIndex, + }, + }; + } + + fn executeTableQueryRequest( + _: *anyopaque, + _: std.mem.Allocator, + _: []const u8, + _: []const u8, + _: ?[]const u8, + _: operation.RequestContext, + ) TableApi.ExecuteQueryError![]u8 { + return error.GraphMetricQueryBudgetExceeded; + } + }; + + var resp = try handleTableQueryRequest(std.testing.allocator, "docs", + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank"}} + , null, Backend.iface()); + defer resp.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(u16, 422), resp.status); + try std.testing.expect(resp.json); + const parsed = try std.json.parseFromSlice(struct { + status: u16, + @"error": []const u8, + message: []const u8, + retryable: bool, + remediation: []const u8, + }, std.testing.allocator, resp.body, .{}); + defer parsed.deinit(); + try std.testing.expectEqual(@as(u16, 422), parsed.value.status); + try std.testing.expectEqualStrings("graph_metric_query_budget_exceeded", parsed.value.@"error"); + try std.testing.expect(parsed.value.message.len > 0); + try std.testing.expect(!parsed.value.retryable); + try std.testing.expect(parsed.value.remediation.len > 0); +} + test "public table query handler maps exact graph execution failures" { const Kind = enum { work_budget, @@ -5912,3 +6271,53 @@ test "public document artifact range reprocess handler returns bounded summary" try std.testing.expectEqual(@as(usize, 1), parsed.value.shard_cursors[0].failed); try std.testing.expectEqual(@as(u32, 10), parsed.value.shard_cursors[0].limit); } +test "public table graph metric action handler returns status response" { + const Backend = struct { + called: bool = false, + + fn iface(self: *@This()) TableApi { + return .{ + .ptr = self, + .request = .{}, + .vtable = &.{ + .execute_table_batch = unsupportedBatch, + .execute_table_query_request = unsupportedQueryRequest, + .execute_table_query_view = unsupportedQueryView, + .execute_table_backup = unsupportedBackup, + .execute_table_restore = unsupportedRestore, + .execute_table_list_indexes = unsupportedListIndexes, + .execute_table_get_index = unsupportedGetIndex, + .execute_table_create_index = unsupportedCreateIndex, + .execute_table_delete_index = unsupportedDeleteIndex, + .execute_table_graph_metric_action = executeGraphMetricAction, + }, + }; + } + + fn executeGraphMetricAction( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + _: operation.RequestContext, + ) TableApi.ExecuteGraphMetricActionError![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + if (!std.mem.eql(u8, table_name, "docs")) return error.InternalFailure; + if (!std.mem.eql(u8, index_name, "graph_idx")) return error.InternalFailure; + if (!std.mem.eql(u8, metric_name, "degree")) return error.InternalFailure; + if (!std.mem.eql(u8, action, "pause")) return error.InternalFailure; + self.called = true; + return alloc.dupe(u8, "{\"status\":{\"state\":\"fresh\",\"maintenance_paused\":true}}") catch return error.InternalFailure; + } + }; + + var backend = Backend{}; + var resp = try handleTableGraphMetricAction(std.testing.allocator, "docs", "graph_idx", "degree", "pause", backend.iface()); + defer resp.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(u16, 200), resp.status); + try std.testing.expect(backend.called); + try std.testing.expectEqualStrings("{\"status\":{\"state\":\"fresh\",\"maintenance_paused\":true}}", resp.body); +} diff --git a/zig/pkg/antfly/src/api/query.zig b/zig/pkg/antfly/src/api/query.zig index 33d2ae0c9a..267ec0ca61 100644 --- a/zig/pkg/antfly/src/api/query.zig +++ b/zig/pkg/antfly/src/api/query.zig @@ -20,6 +20,7 @@ const db_query_search = @import("../storage/db/query/search_exec.zig"); const hierarchy_navigation = @import("../storage/hierarchy_navigation.zig"); const runtime_schema_mod = @import("../storage/schema.zig"); const graph_paths = @import("../graph/paths.zig"); +const graph_mod = @import("../graph/graph.zig"); const graph_pattern = @import("../graph/pattern.zig"); const graph_query_mod = @import("../graph/query.zig"); const graph_node_identity = @import("../graph/node_identity.zig"); @@ -35,6 +36,7 @@ pub const OwnedQueryRequest = query_contract.OwnedQueryRequest; pub const PublicFilterQueryErrorKind = query_contract.PublicFilterQueryErrorKind; pub const parseQueryRequest = query_contract.parseQueryRequest; +pub const parseGraphMetricRequestsAlloc = query_contract.parseGraphMetricRequestsAlloc; pub const parsePublicQueryRequest = query_contract.parsePublicQueryRequest; pub const parsePublicQueryRequestWithDeadline = query_contract.parsePublicQueryRequestWithDeadline; pub const isPublicQueryValidationError = query_contract.isPublicQueryValidationError; @@ -124,10 +126,24 @@ fn mergeGenericSearchResultsWithRuntimeSchema( ) !db_mod.types.SearchResult { var total_hits: u32 = 0; var total_hits_relation: db_mod.types.TotalHitsRelation = .exact; + var has_graph_results = false; + var has_graph_metric_results = false; + var has_graph_metric_rerank_status = false; for (results) |result| { + if (result.graph_results.len > 0) has_graph_results = true; + if (result.graph_metric_results.len > 0) has_graph_metric_results = true; + if (result.graph_metric_rerank_status != null) has_graph_metric_rerank_status = true; total_hits +|= result.total_hits; if (result.total_hits_relation == .gte) total_hits_relation = .gte; } + if (req.graph_queries.len == 0 and has_graph_results) return error.UnsupportedQueryRequest; + if (req.graph_metric_queries.len == 0 and has_graph_metric_results) return error.UnsupportedQueryRequest; + + var graph_metric_rerank_status = if (req.graph_metric_rerank != null or has_graph_metric_rerank_status) + try mergeGraphMetricRerankStatus(alloc, req, results) + else + null; + errdefer if (graph_metric_rerank_status) |*status| status.deinit(alloc); if (req.order_by.len > 0 or req.search_after.len > 0 or req.search_before.len > 0) { var merge_req = req; @@ -139,8 +155,18 @@ fn mergeGenericSearchResultsWithRuntimeSchema( for (final_hits) |*hit| hit.deinit(alloc); if (final_hits.len > 0) alloc.free(final_hits); } + for (final_hits) |hit| try validateSearchHitGraphMetricRerankPayload(req, graph_metric_rerank_status, hit); + + const graph_metric_results = if (req.graph_metric_queries.len > 0 or has_graph_metric_results) + try mergeGraphMetricResults(alloc, req, results) + else + @constCast((&[_]db_mod.types.GraphMetricResult{})[0..]); + errdefer { + for (graph_metric_results) |*metric_result| metric_result.deinit(alloc); + if (graph_metric_results.len > 0) alloc.free(graph_metric_results); + } - const graph_results = if (req.graph_queries.len > 0) + const graph_results = if (req.graph_queries.len > 0 or has_graph_results) try mergeGraphSearchResultsWithLimits(alloc, req.graph_queries, results, req.graph_execution_limits) else @constCast((&[_]db_mod.types.GraphSearchResult{})[0..]); @@ -148,6 +174,7 @@ fn mergeGenericSearchResultsWithRuntimeSchema( for (graph_results) |*graph_result| graph_result.deinit(alloc); if (graph_results.len > 0) alloc.free(graph_results); } + stripUnrequestedGraphSearchMetricStatuses(alloc, req, graph_results); if (results.len > 1) { clearMergedDocOrdinals(final_hits); @@ -162,6 +189,8 @@ fn mergeGenericSearchResultsWithRuntimeSchema( .identity_read_generation = mergedSearchResultIdentityReadGeneration(req, results), .sort_profile = sorted_merge.sort_profile, .graph_results = graph_results, + .graph_metric_results = graph_metric_results, + .graph_metric_rerank_status = graph_metric_rerank_status, }; } @@ -186,6 +215,7 @@ fn mergeGenericSearchResultsWithRuntimeSchema( if (use_bounded_window) try bounded_hits.ensureTotalCapacity(alloc, retained_window); for (results) |result| { for (result.hits) |*hit| { + try validateSearchHitGraphMetricRerankPayload(req, graph_metric_rerank_status, hit.*); if (score_ordered_merge) try validateScoreOrderedMergeHit(hit.*); if (!use_bounded_window) { try merged_hits.append(alloc, hit); @@ -218,12 +248,21 @@ fn mergeGenericSearchResultsWithRuntimeSchema( alloc.free(final_hits); } + const graph_metric_results = if (req.graph_metric_queries.len > 0 or has_graph_metric_results) + try mergeGraphMetricResults(alloc, req, results) + else + @constCast((&[_]db_mod.types.GraphMetricResult{})[0..]); + errdefer { + for (graph_metric_results) |*metric_result| metric_result.deinit(alloc); + if (graph_metric_results.len > 0) alloc.free(graph_metric_results); + } + for (candidate_refs[start..end], 0..) |hit, i| { final_hits[i] = try hit.clone(alloc); moved += 1; } - const graph_results = if (req.graph_queries.len > 0) + const graph_results = if (req.graph_queries.len > 0 or has_graph_results) try mergeGraphSearchResultsWithLimits(alloc, req.graph_queries, results, req.graph_execution_limits) else @constCast((&[_]db_mod.types.GraphSearchResult{})[0..]); @@ -231,6 +270,7 @@ fn mergeGenericSearchResultsWithRuntimeSchema( for (graph_results) |*graph_result| graph_result.deinit(alloc); if (graph_results.len > 0) alloc.free(graph_results); } + stripUnrequestedGraphSearchMetricStatuses(alloc, req, graph_results); if (results.len > 1) { clearMergedDocOrdinals(final_hits); @@ -244,7 +284,50 @@ fn mergeGenericSearchResultsWithRuntimeSchema( .total_hits_relation = total_hits_relation, .identity_read_generation = mergedSearchResultIdentityReadGeneration(req, results), .graph_results = graph_results, + .graph_metric_results = graph_metric_results, + .graph_metric_rerank_status = graph_metric_rerank_status, + }; +} + +fn validateSearchHitGraphMetricRerankPayload( + req: db_mod.types.SearchRequest, + rerank_status: ?db_mod.types.GraphMetricStatus, + hit: db_mod.types.SearchHit, +) !void { + const rerank = req.graph_metric_rerank orelse { + if (hit.score_details != null) return error.UnsupportedQueryRequest; + return; }; + + const hit_score = hit.score orelse return error.UnsupportedQueryRequest; + if (!std.math.isFinite(hit_score)) return error.UnsupportedQueryRequest; + const details = hit.score_details orelse return; + if (!std.mem.eql(u8, details.index_name, rerank.index_name)) return error.UnsupportedQueryRequest; + if (!std.mem.eql(u8, details.metric_name, rerank.metric_name)) return error.UnsupportedQueryRequest; + const status = rerank_status orelse return error.UnsupportedQueryRequest; + if (details.published_generation == 0 or details.published_generation != status.published_generation) return error.UnsupportedQueryRequest; + if (!std.math.isFinite(details.base_score) or !std.math.isFinite(details.base_weight)) return error.UnsupportedQueryRequest; + if (@abs(details.base_weight - rerank.base_weight) > 0.000001) return error.UnsupportedQueryRequest; + if (details.metric_score) |score| { + if (!std.math.isFinite(score) or details.missing_score_used) return error.UnsupportedQueryRequest; + if (@abs(score - details.metric_score_used) > 0.000001) return error.UnsupportedQueryRequest; + } else { + if (!details.missing_score_used) return error.UnsupportedQueryRequest; + if (@abs(details.metric_score_used - rerank.missing_score) > 0.000001) return error.UnsupportedQueryRequest; + } + if (!std.math.isFinite(details.metric_score_used) or !std.math.isFinite(details.metric_weight)) return error.UnsupportedQueryRequest; + if (@abs(details.metric_weight - rerank.weight) > 0.000001) return error.UnsupportedQueryRequest; + if (!std.math.isFinite(details.final_score)) return error.UnsupportedQueryRequest; + const expected_final_score = clampF64ToF32(details.base_score * details.base_weight + details.metric_score_used * details.metric_weight); + if (@abs(@as(f64, expected_final_score) - details.final_score) > 0.000001) return error.UnsupportedQueryRequest; + if (@abs(@as(f64, hit_score) - details.final_score) > 0.000001) return error.UnsupportedQueryRequest; +} + +fn clampF64ToF32(value: f64) f32 { + const max = std.math.floatMax(f32); + if (value > max) return max; + if (value < -max) return -max; + return @floatCast(value); } fn searchHitRefComesBefore( @@ -792,7 +875,8 @@ fn requestUsesScoreOrderedMerge(req: db_mod.types.SearchRequest) bool { req.dense_queries.len > 0 or req.sparse_queries.len > 0 or req.merge_config != null or - req.reranker != null; + req.reranker != null or + req.graph_metric_rerank != null; } fn searchRequestHasScoreBearingSource(req: db_mod.types.SearchRequest) bool { @@ -968,15 +1052,19 @@ const GraphAggregateResultBuilder = struct { const GraphSearchResultBuilder = struct { name: []u8, nodes: std.ArrayListUnmanaged(graph_query_mod.GraphResultNode) = .empty, + node_identities: graph_node_identity.BorrowedMap(void) = .{}, paths: std.ArrayListUnmanaged(db_mod.types.GraphPath) = .empty, matches: std.ArrayListUnmanaged(db_mod.types.GraphPatternMatch) = .empty, aggregates: std.ArrayListUnmanaged(GraphAggregateResultBuilder) = .empty, hits: std.ArrayListUnmanaged(db_mod.types.SearchHit) = .empty, + hit_identities: graph_node_identity.BorrowedMap(void) = .{}, + metric_status: std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus) = .empty, total_hits: u32 = 0, truncated: bool = false, fn deinit(self: *GraphSearchResultBuilder, alloc: std.mem.Allocator) void { if (self.name.len > 0) alloc.free(self.name); + self.node_identities.deinit(alloc); for (self.nodes.items) |*node| node.deinit(alloc); self.nodes.deinit(alloc); for (self.paths.items) |path| graph_paths.freePath(alloc, path); @@ -987,6 +1075,9 @@ const GraphSearchResultBuilder = struct { self.aggregates.deinit(alloc); for (self.hits.items) |*hit| hit.deinit(alloc); self.hits.deinit(alloc); + self.hit_identities.deinit(alloc); + for (self.metric_status.items) |*status| status.deinit(alloc); + self.metric_status.deinit(alloc); self.* = undefined; } @@ -1062,6 +1153,17 @@ const GraphSearchResultBuilder = struct { for (hits) |*hit| hit.deinit(alloc); if (hits.len > 0) alloc.free(hits); } + try retainGraphMergeBytes( + budget, + self.name, + query, + retainedBytesForSlice(db_mod.types.GraphMetricStatus, self.metric_status.items.len), + ); + const metric_status = try self.metric_status.toOwnedSlice(alloc); + errdefer { + for (metric_status) |*status| status.deinit(alloc); + if (metric_status.len > 0) alloc.free(metric_status); + } const name = self.name; self.name = &.{}; @@ -1073,11 +1175,39 @@ const GraphSearchResultBuilder = struct { .aggregates = aggregates, .hits = hits, .total_hits = self.total_hits, + .metric_status = metric_status, .truncated = self.truncated, }; } }; +fn ensureGraphIdentityIndexCapacity( + alloc: std.mem.Allocator, + budget: *graph_work_budget.WorkBudget, + operation: []const u8, + query: graph_query_mod.GraphQuery, + index: *graph_node_identity.BorrowedMap(void), + next_count: usize, +) !void { + const target_capacity = try graph_work_budget.hashMapCapacityForCount( + next_count, + std.hash_map.default_max_load_percentage, + ); + if (target_capacity <= index.capacity()) return; + const old_bytes = try graph_work_budget.hashMapRetainedBytes( + graph_node_identity.Ref, + void, + index.capacity(), + ); + const new_bytes = try graph_work_budget.hashMapRetainedBytes( + graph_node_identity.Ref, + void, + target_capacity, + ); + try retainGraphMergeBytes(budget, operation, query, new_bytes - old_bytes); + try index.ensureTotalCapacity(alloc, next_count); +} + fn graphQueryByName( queries: []const db_mod.types.NamedGraphQuery, name: []const u8, @@ -1171,6 +1301,33 @@ fn graphResultNodeRetainedBytes(node: graph_query_mod.GraphResultNode) usize { retainedPathEdgeSliceBytes(graph_query_mod.PathEdgeInfo, edges), ); if (node.provenance) |provenance| retainedAdd(&total, retainedStringSliceBytes(provenance)); + if (node.metrics_owned) { + retainedAdd(&total, retainedBytesForSlice(graph_query_mod.GraphMetricValue, node.metrics.len)); + for (node.metrics) |metric| if (metric.name_owned) retainedAdd(&total, metric.name.len); + } + return total; +} + +fn graphMetricStatusRetainedBytes(status: db_mod.types.GraphMetricStatus) usize { + var total: usize = 0; + retainedAdd(&total, status.name.len); + retainedAdd(&total, retainedStringSliceBytes(status.edge_filter.types)); + retainedAdd(&total, status.build_worker_id.len); + retainedAdd(&total, status.build_cursor.len); + retainedAdd( + &total, + retainedBytesForSlice(db_mod.types.GraphMetricBuildPageStatus, status.build_pages.len), + ); + for (status.build_pages) |page| { + retainedAdd(&total, page.worker_id.len); + retainedAdd(&total, page.cursor.len); + retainedAdd(&total, page.last_error.len); + } + retainedAdd(&total, status.last_error.len); + retainedAdd( + &total, + retainedBytesForSlice(graph_mod.GraphIndex.GraphMetricEvent, status.recent_events.len), + ); return total; } @@ -1281,6 +1438,19 @@ pub fn graphResultsRetainedUsage( for (result.nodes) |node| { retainedAdd(&usage.state_bytes, graphResultNodeRetainedBytes(node)); } + retainedAdd( + &usage.state_bytes, + retainedBytesForSlice(graph_query_mod.GraphMetricValue, result.metric_values_slab.len), + ); + retainedAdd(&usage.state_bytes, retainedBytesForSlice([]u8, result.metric_value_names.len)); + for (result.metric_value_names) |name| retainedAdd(&usage.state_bytes, name.len); + retainedAdd( + &usage.state_bytes, + retainedBytesForSlice(db_mod.types.GraphMetricStatus, result.metric_status.len), + ); + for (result.metric_status) |status| { + retainedAdd(&usage.state_bytes, graphMetricStatusRetainedBytes(status)); + } retainedAdd( &usage.state_bytes, @@ -1465,1506 +1635,5874 @@ test "distributed graph result admission is cumulative across shard payloads" { ); } -fn mergeGraphSearchResults( - alloc: std.mem.Allocator, - queries: []const db_mod.types.NamedGraphQuery, - results: []const db_mod.types.SearchResult, -) ![]db_mod.types.GraphSearchResult { - return mergeGraphSearchResultsWithLimits(alloc, queries, results, .{}); +test "distributed graph result accounting includes shared metric storage and status details" { + var metric_values = [_]graph_query_mod.GraphMetricValue{.{ + .name = "pagerank", + .score = 0.75, + .name_owned = false, + }}; + var metric_names = [_][]u8{@constCast("pagerank")}; + const edge_types = [_][]const u8{"links_to"}; + var events = [_]graph_mod.GraphIndex.GraphMetricEvent{.{ .kind = .publish }}; + var pages = [_]db_mod.types.GraphMetricBuildPageStatus{.{ + .worker_id = "worker", + .cursor = "cursor", + .last_error = "page-error", + }}; + var statuses = [_]db_mod.types.GraphMetricStatus{.{ + .name = @constCast("pagerank"), + .edge_filter = .{ .mode = .types, .types = &edge_types }, + .build_worker_id = "worker", + .build_cursor = "cursor", + .build_pages = &pages, + .last_error = "metric-error", + .recent_events = &events, + }}; + var graph_results = [_]db_mod.types.GraphSearchResult{.{ + .name = @constCast("walk"), + .nodes = &.{}, + .hits = &.{}, + .total_hits = 0, + }}; + const baseline = graphResultsRetainedUsage(&graph_results).state_bytes; + + graph_results[0].metric_values_slab = &metric_values; + graph_results[0].metric_value_names = &metric_names; + graph_results[0].metric_status = &statuses; + const usage = graphResultsRetainedUsage(&graph_results).state_bytes; + const expected_extra = @sizeOf(graph_query_mod.GraphMetricValue) + + @sizeOf([]u8) + "pagerank".len + + @sizeOf(db_mod.types.GraphMetricStatus) + "pagerank".len + + @sizeOf([]const u8) + "links_to".len + + "worker".len + "cursor".len + + @sizeOf(db_mod.types.GraphMetricBuildPageStatus) + + "worker".len + "cursor".len + "page-error".len + + "metric-error".len + @sizeOf(graph_mod.GraphIndex.GraphMetricEvent); + try std.testing.expectEqual(baseline + expected_extra, usage); +} + +fn graphMetricScoreComesBefore( + left: db_mod.types.GraphMetricScore, + right: db_mod.types.GraphMetricScore, +) bool { + if (left.score != right.score) return left.score > right.score; + return std.mem.lessThan(u8, left.node, right.node); } -fn mergeGraphSearchResultsWithLimits( - alloc: std.mem.Allocator, - queries: []const db_mod.types.NamedGraphQuery, - results: []const db_mod.types.SearchResult, - limits: graph_work_budget.Limits, -) ![]db_mod.types.GraphSearchResult { - var accumulator = try GraphSearchResultsAccumulator.init(alloc, queries, limits); - defer accumulator.deinit(); - // Batch callers already own every input simultaneously. Preserve the - // exact peak accounting used by the legacy batch merge while sharing the - // same folding implementation as incremental distributed fanout. - for (results) |result| try accumulator.admission.admit(result.graph_results); - for (results) |result| try accumulator.appendAdmitted(result.graph_results); - return accumulator.toOwned(); +fn graphMetricScoreWorstFirst( + _: void, + left: db_mod.types.GraphMetricScore, + right: db_mod.types.GraphMetricScore, +) std.math.Order { + if (graphMetricScoreComesBefore(left, right)) return .gt; + if (graphMetricScoreComesBefore(right, left)) return .lt; + return .eq; } -/// Request-wide graph merge state. Distributed coordinators append one shard -/// at a time, then destroy that shard payload before fetching the next. This -/// keeps both graph memory and count(distinct) identity admission independent -/// of shard count while retaining one-pass O(total input) merge behavior. -pub const GraphSearchResultsAccumulator = struct { - alloc: std.mem.Allocator, - queries: []const db_mod.types.NamedGraphQuery, - admission: GraphPayloadAdmission, - builders: std.ArrayListUnmanaged(GraphSearchResultBuilder) = .empty, - shard_count: usize = 0, - finished: bool = false, +const GraphMetricScoreQueue = std.PriorityQueue( + db_mod.types.GraphMetricScore, + void, + graphMetricScoreWorstFirst, +); + +const GraphMetricResultBuilder = struct { + name: []u8, + index_name: []u8, + metric_name: []u8, + scores: GraphMetricScoreQueue = .empty, + score_nodes: std.StringHashMapUnmanaged(void) = .empty, + score_limit: usize, + status: ?db_mod.types.GraphMetricStatus = null, + + fn deinit(self: *GraphMetricResultBuilder, alloc: std.mem.Allocator) void { + if (self.name.len > 0) alloc.free(self.name); + if (self.index_name.len > 0) alloc.free(self.index_name); + if (self.metric_name.len > 0) alloc.free(self.metric_name); + for (self.scores.items) |*score| score.deinit(alloc); + self.scores.deinit(alloc); + self.score_nodes.deinit(alloc); + if (self.status) |*status| status.deinit(alloc); + self.* = undefined; + } + + fn toOwned(self: *GraphMetricResultBuilder, alloc: std.mem.Allocator, top_k: u32) !db_mod.types.GraphMetricResult { + const status = self.status orelse return error.InvalidQueryResponse; + std.sort.pdq(db_mod.types.GraphMetricScore, self.scores.items, {}, struct { + fn lessThan(_: void, a: db_mod.types.GraphMetricScore, b: db_mod.types.GraphMetricScore) bool { + return graphMetricScoreComesBefore(a, b); + } + }.lessThan); + + const count = if (top_k == 0) + self.scores.items.len + else + @min(self.scores.items.len, @as(usize, @intCast(top_k))); + const scores = try alloc.alloc(db_mod.types.GraphMetricScore, count); + for (self.scores.items[0..count], 0..) |score, i| scores[i] = score; + for (self.scores.items[count..]) |*score| score.deinit(alloc); + self.scores.items.len = 0; - pub fn init( - alloc: std.mem.Allocator, - queries: []const db_mod.types.NamedGraphQuery, - limits: graph_work_budget.Limits, - ) !GraphSearchResultsAccumulator { - try limits.validate(); return .{ - .alloc = alloc, - .queries = queries, - .admission = GraphPayloadAdmission.init(queries, limits), + .name = self.name, + .index_name = self.index_name, + .metric_name = self.metric_name, + .scores = scores, + .status = status, }; } +}; - pub fn deinit(self: *GraphSearchResultsAccumulator) void { - for (self.builders.items) |*builder| builder.deinit(self.alloc); - self.builders.deinit(self.alloc); - self.* = undefined; - } +fn mergeGraphMetricResults( + alloc: std.mem.Allocator, + req: db_mod.types.SearchRequest, + results: []const db_mod.types.SearchResult, +) ![]db_mod.types.GraphMetricResult { + try validateRequestedGraphMetricFanIn(req, results); - pub fn appendOwned( - self: *GraphSearchResultsAccumulator, - source_alloc: std.mem.Allocator, - graph_results: *[]db_mod.types.GraphSearchResult, - ) !void { - if (self.finished) return error.InvalidRemoteResponse; - const lease = try self.admission.reserve(graph_results.*); - defer self.admission.release(lease); - try self.appendAdmitted(graph_results.*); - for (graph_results.*) |*graph_result| graph_result.deinit(source_alloc); - if (graph_results.*.len > 0) source_alloc.free(graph_results.*); - graph_results.* = &.{}; + var builders = std.ArrayListUnmanaged(GraphMetricResultBuilder).empty; + defer { + for (builders.items) |*builder| builder.deinit(alloc); + builders.deinit(alloc); } - fn appendAdmitted( - self: *GraphSearchResultsAccumulator, - graph_results: []const db_mod.types.GraphSearchResult, - ) !void { - if (self.finished) return error.InvalidRemoteResponse; - try validateGraphQueriesForShard(self.queries, graph_results); - self.shard_count = std.math.add(usize, self.shard_count, 1) catch - return error.InvalidRemoteResponse; - const merge_work_budget = &self.admission.work_budget; - const distinct_budget = &self.admission.distinct_budget; - for (graph_results) |graph_result| { - try validateGraphAggregateShard(self.queries, graph_result); - const merge_query = graphQueryByName(self.queries, graph_result.name) orelse - return error.InvalidRemoteResponse; + for (results) |result| { + for (result.graph_metric_results) |metric_result| { const idx = blk: { - for (self.builders.items, 0..) |builder, i| { - if (std.mem.eql(u8, builder.name, graph_result.name)) break :blk i; + for (builders.items, 0..) |builder, i| { + if (std.mem.eql(u8, builder.name, metric_result.name)) break :blk i; } - try ensureGraphMergeListCapacity( - GraphSearchResultBuilder, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &self.builders, - self.builders.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - graph_result.name.len, - ); - const name = try self.alloc.dupe(u8, graph_result.name); - self.builders.appendAssumeCapacity(.{ .name = name }); - break :blk self.builders.items.len - 1; + const name = try alloc.dupe(u8, metric_result.name); + errdefer alloc.free(name); + const index_name = try alloc.dupe(u8, metric_result.index_name); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, metric_result.metric_name); + errdefer alloc.free(metric_name); + try builders.append(alloc, .{ + .name = name, + .index_name = index_name, + .metric_name = metric_name, + .score_limit = graphMetricQueryTopK(req, metric_result.name), + }); + break :blk builders.items.len - 1; }; - var builder = &self.builders.items[idx]; - builder.total_hits +|= graph_result.total_hits; - builder.truncated = builder.truncated or graph_result.truncated; - - for (graph_result.nodes) |node| { - if (builder.nodes.items.len >= public_limits.max_graph_result_items) - return error.QueryCandidateBudgetExceeded; - try ensureGraphMergeListCapacity( - graph_query_mod.GraphResultNode, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &builder.nodes, - builder.nodes.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - graphResultNodeRetainedBytes(node), - ); - const owned = try cloneGraphResultNode(self.alloc, node); - builder.nodes.appendAssumeCapacity(owned); - } - for (graph_result.paths) |path| { - if (builder.paths.items.len >= public_limits.max_graph_result_items) - return error.QueryCandidateBudgetExceeded; - try ensureGraphMergeListCapacity( - db_mod.types.GraphPath, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &builder.paths, - builder.paths.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - graphPathRetainedBytes(path), - ); - const owned = try cloneGraphPath(self.alloc, path); - builder.paths.appendAssumeCapacity(owned); - } - for (graph_result.matches) |match| { - const limit = graphQueryReturnLimit(self.queries, graph_result.name); - const effective_limit = if (limit > 0) - @min(@as(usize, limit), public_limits.max_graph_result_items) - else - public_limits.max_graph_result_items; - if (builder.matches.items.len >= effective_limit) { - builder.truncated = true; - continue; + var builder = &builders.items[idx]; + try ensureGraphMetricResultComparable(builder, metric_result); + for (metric_result.scores) |score| { + if (!std.math.isFinite(score.score)) return error.UnsupportedQueryRequest; + if (builder.score_nodes.contains(score.node)) return error.UnsupportedQueryRequest; + try builder.score_nodes.ensureUnusedCapacity(alloc, 1); + builder.score_nodes.putAssumeCapacityNoClobber(score.node, {}); + const should_retain = builder.score_limit == 0 or + builder.scores.count() < builder.score_limit or + graphMetricScoreComesBefore(score, builder.scores.peek().?); + if (!should_retain) continue; + const owned = db_mod.types.GraphMetricScore{ + .node = try alloc.dupe(u8, score.node), + .score = score.score, + }; + errdefer alloc.free(owned.node); + if (builder.score_limit != 0 and builder.scores.count() == builder.score_limit) { + var removed = builder.scores.pop().?; + removed.deinit(alloc); } - try ensureGraphMergeListCapacity( - db_mod.types.GraphPatternMatch, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &builder.matches, - builder.matches.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - graphPatternMatchRetainedBytes(match), - ); - const owned = try cloneGraphPatternMatch(self.alloc, match); - builder.matches.appendAssumeCapacity(owned); + try builder.scores.push(alloc, owned); } - for (graph_result.aggregates) |aggregate| { - const distinct = graphAggregateIsDistinct(self.queries, graph_result.name, aggregate.name); - var found = false; - for (builder.aggregates.items) |*existing| { - if (!std.mem.eql(u8, existing.name, aggregate.name)) continue; - if (existing.distinct != distinct) return error.InvalidRemoteResponse; - existing.exact = existing.exact and aggregate.exact; - if (distinct) { - existing.appendDistinctValues(self.alloc, aggregate) catch |err| { - if (err == error.GraphDistinctBudgetExceeded) { - graph_distinct_budget_diagnostic.recordBudget(graph_result.name, distinct_budget); - } - return err; - }; - } else { - existing.value = std.math.add(u128, existing.value, aggregate.value) catch return error.Overflow; - } - found = true; - break; - } - if (!found) { - try ensureGraphMergeListCapacity( - GraphAggregateResultBuilder, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &builder.aggregates, - builder.aggregates.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - aggregate.name.len, - ); - const name = try self.alloc.dupe(u8, aggregate.name); - var owned = GraphAggregateResultBuilder{ - .name = name, - .value = if (distinct) 0 else aggregate.value, - .exact = aggregate.exact, - .distinct = distinct, - .distinct_budget = distinct_budget, - }; - var owned_active = true; - errdefer if (owned_active) owned.deinit(self.alloc); - if (distinct) owned.appendDistinctValues(self.alloc, aggregate) catch |err| { - if (err == error.GraphDistinctBudgetExceeded) { - graph_distinct_budget_diagnostic.recordBudget(graph_result.name, distinct_budget); - } - return err; - }; - builder.aggregates.appendAssumeCapacity(owned); - owned_active = false; - } - } - for (graph_result.hits) |hit| { - if (builder.hits.items.len >= public_limits.max_graph_hydrated_bindings) - return error.QueryCandidateBudgetExceeded; - try ensureGraphMergeListCapacity( - db_mod.types.SearchHit, - self.alloc, - merge_work_budget, - graph_result.name, - merge_query, - &builder.hits, - builder.hits.items.len + 1, - ); - try retainGraphMergeBytes( - merge_work_budget, - graph_result.name, - merge_query, - graphHitRetainedBytes(hit), - ); - const owned = try hit.clone(self.alloc); - builder.hits.appendAssumeCapacity(owned); + if (builder.status) |*status| { + try mergeGraphMetricStatusInto(alloc, status, metric_result.status); + } else { + builder.status = try cloneGraphMetricStatus(alloc, metric_result.status); } } } - pub fn toOwned(self: *GraphSearchResultsAccumulator) ![]db_mod.types.GraphSearchResult { - if (self.finished) return error.InvalidRemoteResponse; - self.finished = true; - const merge_work_budget = &self.admission.work_budget; - const distinct_budget = &self.admission.distinct_budget; - for (self.builders.items) |builder| { - const query = graphQueryByName(self.queries, builder.name) orelse - return error.InvalidRemoteResponse; - try retainGraphMergeBytes( - merge_work_budget, - builder.name, - query, - @sizeOf(db_mod.types.GraphSearchResult), - ); - } - const merged = try self.alloc.alloc(db_mod.types.GraphSearchResult, self.builders.items.len); - var initialized: usize = 0; - errdefer { - for (merged[0..initialized]) |*graph_result| graph_result.deinit(self.alloc); - self.alloc.free(merged); - } - for (self.builders.items, 0..) |*builder, i| { - const query = graphQueryByName(self.queries, builder.name) orelse - return error.InvalidRemoteResponse; - merged[i] = builder.toOwned(self.alloc, merge_work_budget, query) catch |err| { - if (err == error.GraphDistinctBudgetExceeded) { - graph_distinct_budget_diagnostic.recordBudget(builder.name, distinct_budget); - } - return err; - }; - if (self.shard_count > 1) clearMergedDocOrdinals(merged[i].hits); - initialized += 1; - } - return merged; - } -}; - -/// A shard graph response is operation-keyed just like the public contract. -/// Empty operations are represented by an empty result, never by omitting the -/// operation, so version skew or a broken worker cannot silently turn an exact -/// graph request into a partial success. -fn validateGraphQueriesForShard( - queries: []const db_mod.types.NamedGraphQuery, - graph_results: []const db_mod.types.GraphSearchResult, -) !void { - for (queries) |named| { - var occurrences: usize = 0; - for (graph_results) |graph_result| { - if (std.mem.eql(u8, named.name, graph_result.name)) occurrences += 1; - } - if (occurrences != 1) return error.InvalidRemoteResponse; + const merged = try alloc.alloc(db_mod.types.GraphMetricResult, builders.items.len); + var initialized: usize = 0; + errdefer { + for (merged[0..initialized]) |*metric_result| metric_result.deinit(alloc); + alloc.free(merged); } - - for (graph_results) |graph_result| { - var requested = false; - for (queries) |named| { - if (!std.mem.eql(u8, named.name, graph_result.name)) continue; - requested = true; - break; - } - if (!requested) return error.InvalidRemoteResponse; + for (builders.items, 0..) |*builder, i| { + merged[i] = try builder.toOwned(alloc, graphMetricQueryTopK(req, builder.name)); + initialized += 1; + builder.name = &.{}; + builder.index_name = &.{}; + builder.metric_name = &.{}; + builder.status = null; } + return merged; } -fn validateGraphAggregateShard( - queries: []const db_mod.types.NamedGraphQuery, - graph_result: db_mod.types.GraphSearchResult, +fn validateRequestedGraphMetricFanIn( + req: db_mod.types.SearchRequest, + results: []const db_mod.types.SearchResult, ) !void { - const query = blk: { - for (queries) |named| { - if (std.mem.eql(u8, named.name, graph_result.name)) break :blk named.query; + if (req.graph_metric_queries.len == 0) return; + try validateGraphMetricQueryNamesUnique(req.graph_metric_queries); + + for (results) |result| { + for (result.graph_metric_results) |metric_result| { + if (!graphMetricQueryRequested(req, metric_result.name)) return error.UnsupportedQueryRequest; } - return error.InvalidRemoteResponse; - }; - if (query.aggregates.len == 0) { - if (graph_result.aggregates.len != 0) return error.InvalidRemoteResponse; - return; - } - if (graph_result.aggregates.len != query.aggregates.len) return error.InvalidRemoteResponse; - for (query.aggregates) |requested| { - var occurrences: usize = 0; - for (graph_result.aggregates) |aggregate| { - if (!std.mem.eql(u8, requested.name, aggregate.name)) continue; - if (!aggregate.exact) return error.QueryCandidateBudgetExceeded; - if (requested.distinct) { - if (aggregate.value != aggregate.distinct_values.len) - return error.InvalidRemoteResponse; - } else if (aggregate.distinct_values.len != 0) { - return error.InvalidRemoteResponse; + for (req.graph_metric_queries) |query| { + var found = false; + for (result.graph_metric_results) |metric_result| { + if (!std.mem.eql(u8, metric_result.name, query.name)) continue; + if (found) return error.UnsupportedQueryRequest; + found = true; + if (!std.mem.eql(u8, metric_result.index_name, query.query.index_name)) return error.UnsupportedQueryRequest; + if (!std.mem.eql(u8, metric_result.metric_name, query.query.metric_name)) return error.UnsupportedQueryRequest; + if (!std.mem.eql(u8, metric_result.status.name, query.query.metric_name)) return error.UnsupportedQueryRequest; + if (metric_result.status.published_generation == 0) return error.UnsupportedQueryRequest; + try validateGraphMetricPublishedStatus(metric_result.status); + try validateGraphMetricFreshness(query.query.freshness, metric_result.status); } - occurrences += 1; + if (!found) return error.UnsupportedQueryRequest; } - if (occurrences != 1) return error.InvalidRemoteResponse; + try validateRequestedHitsPairGraphMetricFanIn(req, result); } } -fn graphAggregateIsDistinct( - queries: []const db_mod.types.NamedGraphQuery, - query_name: []const u8, - aggregate_name: []const u8, -) bool { - for (queries) |named| { - if (!std.mem.eql(u8, named.name, query_name)) continue; - for (named.query.aggregates) |aggregate| { - if (std.mem.eql(u8, aggregate.name, aggregate_name)) return aggregate.distinct; +fn validateGraphMetricQueryNamesUnique(queries: []const db_mod.types.NamedGraphMetricQuery) !void { + for (queries, 0..) |query, i| { + for (queries[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, query.name)) return error.UnsupportedQueryRequest; } - return false; } - return false; } -fn graphQueryReturnLimit( - queries: []const db_mod.types.NamedGraphQuery, - query_name: []const u8, -) u32 { - for (queries) |named| { - if (std.mem.eql(u8, named.name, query_name)) return named.query.return_limit; +fn graphMetricQueryRequested(req: db_mod.types.SearchRequest, query_name: []const u8) bool { + for (req.graph_metric_queries) |query| { + if (std.mem.eql(u8, query.name, query_name)) return true; } - return 0; + return false; } -fn cloneGraphNodeRefs(alloc: std.mem.Allocator, values: []const graph_node_identity.Ref) ![]graph_node_identity.Ref { - var out = std.ArrayListUnmanaged(graph_node_identity.Ref).empty; - errdefer { - for (out.items) |value| { - if (value.table) |table| alloc.free(table); - alloc.free(value.key); +fn validateRequestedHitsPairGraphMetricFanIn( + req: db_mod.types.SearchRequest, + result: db_mod.types.SearchResult, +) !void { + for (req.graph_metric_queries) |authority_query| { + if (!isDefaultHitsAuthorityMetric(authority_query.query.metric_name)) continue; + for (req.graph_metric_queries) |hub_query| { + if (!isDefaultHitsHubMetric(hub_query.query.metric_name)) continue; + if (!std.mem.eql(u8, authority_query.query.index_name, hub_query.query.index_name)) continue; + const authority = graphMetricResultByName(result.graph_metric_results, authority_query.name) orelse return error.UnsupportedQueryRequest; + const hub = graphMetricResultByName(result.graph_metric_results, hub_query.name) orelse return error.UnsupportedQueryRequest; + try validateHitsPairMetricStatusesCompatible(authority.status, hub.status); } - out.deinit(alloc); } - try out.ensureTotalCapacity(alloc, values.len); - for (values) |value| try appendClonedGraphNodeRef(alloc, &out, value); - return try out.toOwnedSlice(alloc); } -fn appendClonedGraphNodeRef( - alloc: std.mem.Allocator, - out: *std.ArrayListUnmanaged(graph_node_identity.Ref), - value: graph_node_identity.Ref, +fn graphMetricResultByName( + results: []const db_mod.types.GraphMetricResult, + name: []const u8, +) ?db_mod.types.GraphMetricResult { + for (results) |result| { + if (std.mem.eql(u8, result.name, name)) return result; + } + return null; +} + +fn isDefaultHitsAuthorityMetric(metric_name: []const u8) bool { + return std.mem.eql(u8, metric_name, "hits_authority"); +} + +fn isDefaultHitsHubMetric(metric_name: []const u8) bool { + return std.mem.eql(u8, metric_name, "hits_hub"); +} + +fn validateHitsPairMetricStatusesCompatible( + authority: db_mod.types.GraphMetricStatus, + hub: db_mod.types.GraphMetricStatus, ) !void { - const table = if (value.table) |table_name| try alloc.dupe(u8, table_name) else null; - errdefer if (table) |table_name| alloc.free(table_name); - const key = try alloc.dupe(u8, value.key); - errdefer alloc.free(key); - try out.append(alloc, .{ .table = table, .key = key }); + if (authority.published_generation == 0 or hub.published_generation == 0) return error.UnsupportedQueryRequest; + if (authority.published_generation != hub.published_generation) return error.UnsupportedQueryRequest; + try validateGraphMetricStatusSchemaAndFilterCompatible(authority, hub); } -fn freeGraphNodeRefs(alloc: std.mem.Allocator, values: []const graph_node_identity.Ref) void { - for (values) |value| { - if (value.table) |table| alloc.free(table); - alloc.free(value.key); +fn validateGraphMetricFreshness( + freshness: db_mod.types.GraphMetricFreshness, + status: db_mod.types.GraphMetricStatus, +) !void { + if (freshness == .fresh and status.state != .fresh) return error.UnsupportedQueryRequest; +} + +fn validateGraphMetricPublishedStatus(status: db_mod.types.GraphMetricStatus) !void { + switch (status.state) { + .fresh, .stale, .building, .failed => {}, + .not_ready, .disabled => return error.UnsupportedQueryRequest, } - if (values.len > 0) alloc.free(values); + try validateGraphMetricStatusGenerationShape(status); + if (!std.math.isFinite(status.progress)) return error.UnsupportedQueryRequest; + if (status.progress < 0.0 or status.progress > 1.0) return error.UnsupportedQueryRequest; + if (!std.math.isFinite(status.delta)) return error.UnsupportedQueryRequest; } -fn cloneGraphSearchResult( - alloc: std.mem.Allocator, - source: db_mod.types.GraphSearchResult, -) !db_mod.types.GraphSearchResult { - const GraphNode = std.meta.Child(@TypeOf(source.nodes)); - const nodes = try alloc.alloc(GraphNode, source.nodes.len); - var initialized_nodes: usize = 0; - errdefer { - for (nodes[0..initialized_nodes]) |*node| node.deinit(alloc); - if (source.nodes.len > 0) alloc.free(nodes); +fn validateGraphMetricUnpublishedStatus(status: db_mod.types.GraphMetricStatus) !void { + if (status.published_generation != 0) return error.UnsupportedQueryRequest; + switch (status.state) { + .not_ready, .building, .failed => {}, + .fresh, .stale, .disabled => return error.UnsupportedQueryRequest, } - for (source.nodes, 0..) |node, i| { - nodes[i] = try cloneGraphResultNode(alloc, node); - initialized_nodes += 1; + if (!std.math.isFinite(status.progress)) return error.UnsupportedQueryRequest; + if (status.progress < 0.0 or status.progress > 1.0) return error.UnsupportedQueryRequest; + if (!std.math.isFinite(status.delta)) return error.UnsupportedQueryRequest; +} + +fn validateGraphMetricStatusGenerationShape(status: db_mod.types.GraphMetricStatus) !void { + const published = status.published_generation; + if (published == 0) return error.UnsupportedQueryRequest; + if (status.edge_generation != 0 and status.edge_generation < published) return error.UnsupportedQueryRequest; + if (status.target_edge_generation != 0 and status.target_edge_generation < published) return error.UnsupportedQueryRequest; + if (status.queued_generation != 0 and status.queued_generation < published) return error.UnsupportedQueryRequest; + if (status.building_generation != 0 and status.building_generation < published) return error.UnsupportedQueryRequest; + if (status.state == .fresh) { + if (status.edge_generation != 0 and status.edge_generation != published) return error.UnsupportedQueryRequest; + if (status.target_edge_generation != 0 and status.target_edge_generation != published) return error.UnsupportedQueryRequest; } +} - const GraphPath = std.meta.Child(@TypeOf(source.paths)); - const paths = try alloc.alloc(GraphPath, source.paths.len); - var initialized_paths: usize = 0; - errdefer { - for (paths[0..initialized_paths]) |path| graph_paths.freePath(alloc, path); - if (source.paths.len > 0) alloc.free(paths); +fn ensureGraphMetricResultComparable( + builder: *const GraphMetricResultBuilder, + metric_result: db_mod.types.GraphMetricResult, +) !void { + if (!std.mem.eql(u8, builder.index_name, metric_result.index_name)) return error.UnsupportedQueryRequest; + if (!std.mem.eql(u8, builder.metric_name, metric_result.metric_name)) return error.UnsupportedQueryRequest; + const existing = builder.status orelse return; + try validateGraphMetricStatusCompatible(existing, metric_result.status); + if (existing.published_generation != 0 and + metric_result.status.published_generation != 0 and + existing.published_generation != metric_result.status.published_generation) + { + return error.UnsupportedQueryRequest; } - for (source.paths, 0..) |path, i| { - paths[i] = try cloneGraphPath(alloc, path); - initialized_paths += 1; + if ((existing.published_generation == 0 and builder.scores.items.len > 0) or + (metric_result.status.published_generation == 0 and metric_result.scores.len > 0)) + { + return error.UnsupportedQueryRequest; } +} - const hits = try alloc.alloc(db_mod.types.SearchHit, source.hits.len); - var initialized_hits: usize = 0; - errdefer { - for (hits[0..initialized_hits]) |*hit| hit.deinit(alloc); - if (source.hits.len > 0) alloc.free(hits); - } - for (source.hits, 0..) |hit, i| { - hits[i] = try hit.clone(alloc); - initialized_hits += 1; +fn validateGraphMetricStatusCompatible( + existing: db_mod.types.GraphMetricStatus, + incoming: db_mod.types.GraphMetricStatus, +) !void { + try validateGraphMetricStatusSchemaAndFilterCompatible(existing, incoming); + if (existing.config_fingerprint != 0 and + incoming.config_fingerprint != 0 and + existing.config_fingerprint != incoming.config_fingerprint) + { + return error.UnsupportedQueryRequest; } +} - const matches = try alloc.alloc(db_mod.types.GraphPatternMatch, source.matches.len); - var initialized_matches: usize = 0; - errdefer { - for (matches[0..initialized_matches]) |*match| match.deinit(alloc); - if (source.matches.len > 0) alloc.free(matches); - } - for (source.matches, 0..) |match, i| { - matches[i] = try cloneGraphPatternMatch(alloc, match); - initialized_matches += 1; +fn validateGraphMetricStatusSchemaAndFilterCompatible( + existing: db_mod.types.GraphMetricStatus, + incoming: db_mod.types.GraphMetricStatus, +) !void { + if (existing.metadata_version != 0 and + incoming.metadata_version != 0 and + existing.metadata_version != incoming.metadata_version) + { + return error.UnsupportedQueryRequest; } + if (!existing.edge_filter.equivalent(incoming.edge_filter)) return error.UnsupportedQueryRequest; +} - const aggregates = try alloc.alloc(db_mod.types.GraphAggregateResult, source.aggregates.len); - var initialized_aggregates: usize = 0; - errdefer { - for (aggregates[0..initialized_aggregates]) |*aggregate| aggregate.deinit(alloc); - if (source.aggregates.len > 0) alloc.free(aggregates); - } - for (source.aggregates, 0..) |aggregate, i| { - const name = try alloc.dupe(u8, aggregate.name); - const distinct_values = cloneGraphNodeRefs(alloc, aggregate.distinct_values) catch |err| { - alloc.free(name); - return err; - }; - aggregates[i] = .{ .name = name, .value = aggregate.value, .exact = aggregate.exact, .distinct_values = distinct_values }; - initialized_aggregates += 1; +fn graphMetricQueryTopK(req: db_mod.types.SearchRequest, query_name: []const u8) u32 { + for (req.graph_metric_queries) |query| { + if (std.mem.eql(u8, query.name, query_name)) return query.query.top_k; } + return 0; +} - return .{ - .name = try alloc.dupe(u8, source.name), - .nodes = nodes, - .paths = paths, - .matches = matches, - .aggregates = aggregates, - .hits = hits, - .total_hits = source.total_hits, - .truncated = source.truncated, +fn mergeGraphMetricRerankStatus( + alloc: std.mem.Allocator, + req: db_mod.types.SearchRequest, + results: []const db_mod.types.SearchResult, +) !?db_mod.types.GraphMetricStatus { + const rerank = req.graph_metric_rerank orelse { + for (results) |result| { + if (result.graph_metric_rerank_status != null) return error.UnsupportedQueryRequest; + } + return null; }; + + var merged: ?db_mod.types.GraphMetricStatus = null; + errdefer if (merged) |*status| status.deinit(alloc); + + for (results) |result| { + const status = result.graph_metric_rerank_status orelse return error.UnsupportedQueryRequest; + if (!std.mem.eql(u8, status.name, rerank.metric_name)) return error.UnsupportedQueryRequest; + if (status.published_generation == 0) return error.UnsupportedQueryRequest; + try validateGraphMetricPublishedStatus(status); + try validateGraphMetricFreshness(rerank.freshness, status); + if (merged) |*existing| { + if (existing.published_generation != status.published_generation) return error.UnsupportedQueryRequest; + try validateGraphMetricStatusCompatible(existing.*, status); + try mergeGraphMetricStatusInto(alloc, existing, status); + } else { + merged = try cloneGraphMetricStatus(alloc, status); + } + } + + return merged; } -fn cloneGraphPatternMatch( - alloc: std.mem.Allocator, - source: db_mod.types.GraphPatternMatch, -) !db_mod.types.GraphPatternMatch { - const bindings = try alloc.alloc(db_mod.types.GraphPatternBinding, source.bindings.len); - var initialized_bindings: usize = 0; - errdefer { - for (bindings[0..initialized_bindings]) |*binding| binding.deinit(alloc); - if (source.bindings.len > 0) alloc.free(bindings); +pub fn mergeGraphMetricStatusInto(alloc: std.mem.Allocator, target: *db_mod.types.GraphMetricStatus, source: db_mod.types.GraphMetricStatus) !void { + target.state = mergeGraphMetricState(target.state, source.state); + target.phase = mergeGraphMetricPhase(target.phase, source.phase); + target.metadata_version = mergeGraphMetricMetadataVersion(target.metadata_version, source.metadata_version); + target.config_fingerprint = mergeComparableGeneration(target.config_fingerprint, source.config_fingerprint); + target.maintenance_paused = target.maintenance_paused or source.maintenance_paused; + target.build_queued = target.build_queued or source.build_queued; + target.published_generation = mergeComparableGeneration(target.published_generation, source.published_generation); + target.edge_generation = @max(target.edge_generation, source.edge_generation); + target.target_edge_generation = @max(target.target_edge_generation, source.target_edge_generation); + target.queued_generation = @max(target.queued_generation, source.queued_generation); + target.building_generation = @max(target.building_generation, source.building_generation); + target.build_job_id = if (target.build_job_id == 0) source.build_job_id else if (source.build_job_id == 0 or source.build_job_id == target.build_job_id) target.build_job_id else 0; + target.build_started_at_ms = if (target.build_started_at_ms == 0) source.build_started_at_ms else if (source.build_started_at_ms == 0 or source.build_started_at_ms == target.build_started_at_ms) target.build_started_at_ms else @min(target.build_started_at_ms, source.build_started_at_ms); + target.build_iteration = @max(target.build_iteration, source.build_iteration); + target.build_lease_expires_at_ms = @max(target.build_lease_expires_at_ms, source.build_lease_expires_at_ms); + target.build_completed_units = @max(target.build_completed_units, source.build_completed_units); + target.build_total_units = @max(target.build_total_units, source.build_total_units); + if (target.build_worker_id.len == 0) { + target.build_worker_id = if (source.build_worker_id.len > 0) try alloc.dupe(u8, source.build_worker_id) else ""; + } else if (source.build_worker_id.len > 0 and !std.mem.eql(u8, target.build_worker_id, source.build_worker_id)) { + alloc.free(target.build_worker_id); + target.build_worker_id = try alloc.dupe(u8, "multiple"); } - for (source.bindings, 0..) |binding, i| { - const alias = try alloc.dupe(u8, binding.alias); - errdefer alloc.free(alias); - const node = try cloneGraphResultNode(alloc, binding.node); - bindings[i] = .{ .alias = alias, .node = node }; - initialized_bindings += 1; + if (target.build_cursor.len == 0 and source.build_cursor.len > 0) { + target.build_cursor = try alloc.dupe(u8, source.build_cursor); } - - const path = try alloc.alloc(graph_query_mod.PathEdgeInfo, source.path.len); - var initialized_path: usize = 0; - errdefer { - for (path[0..initialized_path]) |edge| { - alloc.free(edge.source); - alloc.free(edge.target); - alloc.free(edge.edge_type); - if (edge.metadata.len > 0) alloc.free(edge.metadata); - } - if (source.path.len > 0) alloc.free(path); + if (target.build_pages.len == 0 and source.build_pages.len > 0) { + target.build_pages = try cloneGraphMetricBuildPageStatuses(alloc, source.build_pages); + } else if (source.build_pages.len > 0) { + target.build_pages_truncated = true; } - for (source.path, 0..) |edge, i| { - path[i] = try clonePathEdge(graph_query_mod.PathEdgeInfo, alloc, edge); - initialized_path += 1; + target.build_pages_truncated = target.build_pages_truncated or source.build_pages_truncated; + target.retry_count = @max(target.retry_count, source.retry_count); + if (target.last_error.len == 0 and source.last_error.len > 0) { + target.last_error = try alloc.dupe(u8, source.last_error); } - - const null_aliases = try alloc.alloc([]u8, source.null_aliases.len); - var initialized_null_aliases: usize = 0; - errdefer { - for (null_aliases[0..initialized_null_aliases]) |alias| alloc.free(alias); - if (source.null_aliases.len > 0) alloc.free(null_aliases); + target.progress = @min(target.progress, source.progress); + target.converged = target.converged and source.converged; + target.iterations_completed = @max(target.iterations_completed, source.iterations_completed); + target.delta = @max(target.delta, source.delta); + target.computed_at_ms = @max(target.computed_at_ms, source.computed_at_ms); +} + +/// Operational fan-out may aggregate generations and progress that naturally +/// differ by shard, but it must never conceal divergent metric definitions. +/// Query fan-in performs the same validation before calling the lower-level +/// merge helper; control-plane callers use this checked entry point directly. +pub fn mergeCompatibleGraphMetricStatusInto( + alloc: std.mem.Allocator, + target: *db_mod.types.GraphMetricStatus, + source: db_mod.types.GraphMetricStatus, +) !void { + if (!std.mem.eql(u8, target.name, source.name)) return error.GraphMetricStatusConflict; + if (target.metadata_version != 0 and + source.metadata_version != 0 and + target.metadata_version != source.metadata_version) + { + return error.GraphMetricStatusConflict; } - for (source.null_aliases, 0..) |alias, i| { - null_aliases[i] = try alloc.dupe(u8, alias); - initialized_null_aliases += 1; + if (target.config_fingerprint != 0 and + source.config_fingerprint != 0 and + target.config_fingerprint != source.config_fingerprint) + { + return error.GraphMetricStatusConflict; } + if (!target.edge_filter.equivalent(source.edge_filter)) return error.GraphMetricStatusConflict; + try mergeGraphMetricStatusInto(alloc, target, source); +} - return .{ - .bindings = bindings, - .path = path, - .null_aliases = null_aliases, +test "graph metric operational status aggregation rejects divergent shard definitions" { + const alloc = std.testing.allocator; + const cites_filter = graph_mod.GraphMetricEdgeFilter{ .mode = .types, .types = &.{"cites"} }; + const related_filter = graph_mod.GraphMetricEdgeFilter{ .mode = .types, .types = &.{"related"} }; + + var target = db_mod.types.GraphMetricStatus{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .edge_filter = try cites_filter.cloneAlloc(alloc), + .metadata_version = 2, + .config_fingerprint = 11, + .published_generation = 7, + .progress = 1.0, }; -} + defer target.deinit(alloc); -fn cloneGraphResultNode( - alloc: std.mem.Allocator, - source: graph_query_mod.GraphResultNode, -) !graph_query_mod.GraphResultNode { - const key = try alloc.dupe(u8, source.key); - errdefer alloc.free(key); - const table = if (source.table) |value| try alloc.dupe(u8, value) else null; - errdefer if (table) |value| alloc.free(value); - const path = if (source.path) |items| try cloneStringSlice(alloc, items) else null; - errdefer if (path) |items| freeStringSlice(alloc, items); - const path_tables = if (source.path_tables) |items| try cloneOptionalStringSlice(alloc, items) else null; - errdefer if (path_tables) |items| freeOptionalStringSlice(alloc, items); - const path_edges = if (source.path_edges) |items| - try clonePathEdges(graph_query_mod.PathEdgeInfo, alloc, items) - else - null; - errdefer if (path_edges) |items| freePathEdges(alloc, items); - const provenance = if (source.provenance) |items| - try cloneStringSlice(alloc, items) - else - null; - errdefer if (provenance) |items| freeStringSlice(alloc, items); + var metadata_mismatch = db_mod.types.GraphMetricStatus{ + .name = try alloc.dupe(u8, "pagerank"), + .edge_filter = try cites_filter.cloneAlloc(alloc), + .metadata_version = 3, + }; + defer metadata_mismatch.deinit(alloc); + try std.testing.expectError( + error.GraphMetricStatusConflict, + mergeCompatibleGraphMetricStatusInto(alloc, &target, metadata_mismatch), + ); - return .{ - .key = key, - .depth = source.depth, - .distance = source.distance, - .path = path, - .path_tables = path_tables, - .path_edges = path_edges, - .provenance = provenance, - .table = table, + var filter_mismatch = db_mod.types.GraphMetricStatus{ + .name = try alloc.dupe(u8, "pagerank"), + .edge_filter = try related_filter.cloneAlloc(alloc), + .metadata_version = 2, }; -} + defer filter_mismatch.deinit(alloc); + try std.testing.expectError( + error.GraphMetricStatusConflict, + mergeCompatibleGraphMetricStatusInto(alloc, &target, filter_mismatch), + ); -fn cloneGraphPath( - alloc: std.mem.Allocator, - source: db_mod.types.GraphPath, -) !db_mod.types.GraphPath { - const nodes = try cloneStringSlice(alloc, source.nodes); - errdefer freeStringSlice(alloc, nodes); - const node_tables = try cloneOptionalStringSlice(alloc, source.node_tables); - errdefer freeOptionalStringSlice(alloc, node_tables); - const edges = try clonePathEdges(graph_paths.PathEdge, alloc, source.edges); - errdefer freePathEdges(alloc, edges); + var fingerprint_mismatch = db_mod.types.GraphMetricStatus{ + .name = try alloc.dupe(u8, "pagerank"), + .edge_filter = try cites_filter.cloneAlloc(alloc), + .metadata_version = 2, + .config_fingerprint = 12, + }; + defer fingerprint_mismatch.deinit(alloc); + try std.testing.expectError( + error.GraphMetricStatusConflict, + mergeCompatibleGraphMetricStatusInto(alloc, &target, fingerprint_mismatch), + ); - return .{ - .nodes = nodes, - .node_tables = node_tables, - .edges = edges, - .total_weight = source.total_weight, - .length = source.length, + var compatible = db_mod.types.GraphMetricStatus{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .edge_filter = try cites_filter.cloneAlloc(alloc), + .metadata_version = 2, + .config_fingerprint = 11, + .published_generation = 7, + .progress = 0.5, }; + defer compatible.deinit(alloc); + try mergeCompatibleGraphMetricStatusInto(alloc, &target, compatible); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, target.state); + try std.testing.expectEqual(@as(f64, 0.5), target.progress); } -fn cloneStringSlice( - alloc: std.mem.Allocator, - source: []const []const u8, -) ![][]const u8 { - const out = try alloc.alloc([]const u8, source.len); - var initialized: usize = 0; - errdefer { - for (out[0..initialized]) |item| alloc.free(item); - alloc.free(out); - } - for (source, 0..) |item, i| { - out[i] = try alloc.dupe(u8, item); - initialized += 1; - } - return out; +fn mergeGraphMetricMetadataVersion(left: u32, right: u32) u32 { + if (left == 0) return right; + if (right == 0) return left; + if (left == right) return left; + return 0; } -fn freeStringSlice(alloc: std.mem.Allocator, items: []const []const u8) void { - for (items) |item| alloc.free(item); - alloc.free(items); +fn mergeComparableGeneration(left: u64, right: u64) u64 { + if (left == 0) return right; + if (right == 0) return left; + if (left == right) return left; + return @min(left, right); } -fn cloneOptionalStringSlice( +fn mergeGraphMetricState(left: graph_mod.GraphIndex.GraphMetricState, right: graph_mod.GraphIndex.GraphMetricState) graph_mod.GraphIndex.GraphMetricState { + return if (graphMetricStateSeverity(right) > graphMetricStateSeverity(left)) right else left; +} + +fn graphMetricStateSeverity(state: graph_mod.GraphIndex.GraphMetricState) u8 { + return switch (state) { + .disabled => 5, + .failed => 4, + .building => 3, + .not_ready => 2, + .stale => 1, + .fresh => 0, + }; +} + +fn mergeGraphMetricPhase(left: graph_mod.GraphIndex.GraphMetricBuildPhase, right: graph_mod.GraphIndex.GraphMetricBuildPhase) graph_mod.GraphIndex.GraphMetricBuildPhase { + return if (graphMetricPhaseSeverity(right) > graphMetricPhaseSeverity(left)) right else left; +} + +fn graphMetricPhaseSeverity(phase: graph_mod.GraphIndex.GraphMetricBuildPhase) u8 { + return switch (phase) { + .cleanup_old_generations => 10, + .publish_generation, .publishing => 9, + .check_convergence => 8, + .hits_hub_reduce_ranks => 8, + .hits_hub_contributions => 8, + .reduce_ranks => 7, + .iterate_contributions, .computing => 6, + .initialize_ranks => 5, + .scan_edges_and_out_degree => 4, + .prepare_generation => 3, + .idle => 1, + .complete => 0, + }; +} + +fn stripUnrequestedGraphSearchMetricStatuses( alloc: std.mem.Allocator, - source: []const ?[]const u8, -) ![]?[]const u8 { - if (source.len == 0) return &.{}; - const out = try alloc.alloc(?[]const u8, source.len); - var initialized: usize = 0; - errdefer { - for (out[0..initialized]) |item| if (item) |value| alloc.free(value); - alloc.free(out); + req: db_mod.types.SearchRequest, + graph_results: []db_mod.types.GraphSearchResult, +) void { + for (graph_results) |*graph_result| { + if (graphSearchQueryIncludesMetricStatus(req, graph_result.name)) continue; + freeGraphSearchMetricStatuses(alloc, graph_result); } - for (source, 0..) |item, i| { - out[i] = if (item) |value| try alloc.dupe(u8, value) else null; - initialized += 1; +} + +fn graphSearchQueryIncludesMetricStatus(req: db_mod.types.SearchRequest, name: []const u8) bool { + for (req.graph_queries) |query| { + if (std.mem.eql(u8, query.name, name)) return query.query.include_metric_status; } - return out; + return false; } -fn freeOptionalStringSlice( +fn freeGraphSearchMetricStatuses( alloc: std.mem.Allocator, - items: []const ?[]const u8, + graph_result: *db_mod.types.GraphSearchResult, ) void { - for (items) |item| if (item) |value| alloc.free(value); - if (items.len > 0) alloc.free(items); + for (graph_result.metric_status) |*status| status.deinit(alloc); + if (graph_result.metric_status.len > 0) alloc.free(graph_result.metric_status); + graph_result.metric_status = &.{}; } -fn clonePathEdges( - comptime Edge: type, - alloc: std.mem.Allocator, - source: anytype, -) ![]Edge { - const out = try alloc.alloc(Edge, source.len); - var initialized: usize = 0; - errdefer { - for (out[0..initialized]) |edge| freePathEdge(alloc, edge); - alloc.free(out); +fn validateGraphResultNodePayload(node: graph_query_mod.GraphResultNode) !void { + if (!std.math.isFinite(node.distance)) return error.UnsupportedQueryRequest; + if (node.path) |path_nodes| { + if (node.path_edges) |edges| { + if (path_nodes.len != edges.len + 1) return error.UnsupportedQueryRequest; + if (node.depth != edges.len) return error.UnsupportedQueryRequest; + } + } else if (node.path_edges != null) { + return error.UnsupportedQueryRequest; } - for (source, 0..) |edge, i| { - out[i] = try clonePathEdge(Edge, alloc, edge); - initialized += 1; + if (node.path_edges) |edges| { + for (edges) |edge| { + if (!std.math.isFinite(edge.weight)) return error.UnsupportedQueryRequest; + } } - return out; } -fn clonePathEdge( - comptime Edge: type, - alloc: std.mem.Allocator, - source: anytype, -) !Edge { - const edge_source = try alloc.dupe(u8, source.source); - errdefer alloc.free(edge_source); - const target = try alloc.dupe(u8, source.target); - errdefer alloc.free(target); - const edge_type = try alloc.dupe(u8, source.edge_type); - errdefer alloc.free(edge_type); - const metadata = if (source.metadata.len > 0) - try alloc.dupe(u8, source.metadata) - else - ""; - errdefer if (metadata.len > 0) alloc.free(metadata); - return .{ - .source = edge_source, - .target = target, - .edge_type = edge_type, - .weight = source.weight, - .metadata = metadata, - .traversal_direction = source.traversal_direction, - }; +fn validateGraphPathPayload(path: db_mod.types.GraphPath) !void { + if (path.nodes.len != path.edges.len + 1) return error.UnsupportedQueryRequest; + if (path.length != path.edges.len) return error.UnsupportedQueryRequest; + if (!std.math.isFinite(path.total_weight)) return error.UnsupportedQueryRequest; + for (path.edges) |edge| { + if (!std.math.isFinite(edge.weight)) return error.UnsupportedQueryRequest; + } } -fn freePathEdges(alloc: std.mem.Allocator, edges: anytype) void { - for (edges) |edge| freePathEdge(alloc, edge); - alloc.free(edges); +fn validateGraphPatternMatchPayload(match: db_mod.types.GraphPatternMatch) !void { + for (match.bindings) |binding| try validateGraphResultNodePayload(binding.node); + for (match.path) |edge| { + if (!std.math.isFinite(edge.weight)) return error.UnsupportedQueryRequest; + } } -fn freePathEdge(alloc: std.mem.Allocator, edge: anytype) void { - alloc.free(edge.source); - alloc.free(edge.target); - alloc.free(edge.edge_type); - if (edge.metadata.len > 0) alloc.free(edge.metadata); -} - -test "query parser accepts full text request subset" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"match":{"field":"body","text":"alpha"}},"fields":["title"],"limit":5} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(u32, 5), owned.req.limit); - try std.testing.expectEqual(@as(usize, 1), owned.req.fields.len); - try std.testing.expectEqual(false, owned.req.include_all_fields); +fn validateGraphSearchHitPayload(hit: db_mod.types.SearchHit) !void { + if (hit.score_details != null) return error.UnsupportedQueryRequest; + if (hit.score) |score| { + if (!std.math.isFinite(score)) return error.UnsupportedQueryRequest; + } } -test "query parser accepts generated query request shape" { - const metadata_openapi = @import("antfly_metadata_openapi"); - const full_text = try ant_json.RawValue.init( - \\{"match":{"field":"body","text":"alpha"}} - ); - - const body = try jsonStringifyAlloc(std.testing.allocator, metadata_openapi.QueryRequest{ - .full_text_search = full_text, - .fields = &.{"title"}, - .limit = 5, - .profile = false, - }); - defer std.testing.allocator.free(body); +fn validateRequestedGraphSearchFanIn( + req: db_mod.types.SearchRequest, + results: []const db_mod.types.SearchResult, +) !void { + if (req.graph_queries.len == 0) return; + try validateGraphSearchQueryNamesUnique(req.graph_queries); - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", body); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(u32, 5), owned.req.limit); - try std.testing.expectEqual(@as(usize, 1), owned.req.fields.len); - try std.testing.expectEqualStrings("title", owned.req.fields[0]); + for (results) |result| { + for (result.graph_results) |graph_result| { + if (!graphSearchQueryRequested(req, graph_result.name)) return error.UnsupportedQueryRequest; + } + for (req.graph_queries) |query| { + var found = false; + for (result.graph_results) |graph_result| { + if (!std.mem.eql(u8, graph_result.name, query.name)) continue; + if (found) return error.UnsupportedQueryRequest; + found = true; + try validateGraphSearchMetricStatuses(query, graph_result); + } + if (!found) return error.UnsupportedQueryRequest; + } + } } -test "query parser defers ordinary stored projection to response encoding" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"fields":["id","title"],"limit":5} - ); - defer owned.deinit(std.testing.allocator); - - try std.testing.expectEqual(@as(usize, 2), owned.req.fields.len); - try std.testing.expect(owned.req.defer_stored_projection); +fn validateGraphSearchQueryNamesUnique(queries: []const db_mod.types.NamedGraphQuery) !void { + for (queries, 0..) |query, i| { + for (queries[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, query.name)) return error.UnsupportedQueryRequest; + } + } } -test "query parser defaults to stored documents when fields are omitted" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"limit":5} - ); - defer owned.deinit(std.testing.allocator); - - try std.testing.expectEqual(@as(usize, 0), owned.req.fields.len); - try std.testing.expectEqual(true, owned.req.include_all_fields); - try std.testing.expectEqual(true, owned.req.include_stored); - try std.testing.expectEqual(false, owned.req.defer_stored_projection); +fn graphSearchQueryRequested(req: db_mod.types.SearchRequest, query_name: []const u8) bool { + for (req.graph_queries) |query| { + if (std.mem.eql(u8, query.name, query_name)) return true; + } + return false; } -test "query parser keeps special stored projection in db layer" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"fields":["title","_chunks.*"],"limit":5} - ); - defer owned.deinit(std.testing.allocator); +fn validateGraphSearchMetricStatuses( + query: db_mod.types.NamedGraphQuery, + graph_result: db_mod.types.GraphSearchResult, +) !void { + try validateGraphSearchProjectedMetricNamesUnique(query.query.metrics); + try validateGraphSearchOrderMetricNamesUnique(query.query.order_by); + const expects_metric_status = query.query.include_metric_status or + query.query.metrics.len > 0 or + query.query.order_by.len > 0 or + query.query.where_metric.len > 0; + if (!expects_metric_status and graph_result.metric_status.len > 0) return error.UnsupportedQueryRequest; + try validateGraphSearchMetricStatusList(query, graph_result.metric_status); + if (!query.query.include_metric_status) { + try validateGraphSearchMetricStatusNamesRequested(query, graph_result.metric_status); + } + try validateGraphSearchMetricPayloads(query, graph_result); + for (query.query.metrics) |metric| try validateGraphSearchMetricStatus(graph_result, metric.name, metric.freshness, false); + for (query.query.order_by) |metric| try validateGraphSearchMetricStatus(graph_result, metric.name, metric.freshness, true); + for (query.query.where_metric) |metric| try validateGraphSearchMetricStatus(graph_result, metric.name, metric.freshness, true); +} - try std.testing.expectEqual(@as(usize, 2), owned.req.fields.len); - try std.testing.expect(!owned.req.defer_stored_projection); +fn validateGraphSearchProjectedMetricNamesUnique(metrics: []const graph_query_mod.GraphMetricRead) !void { + for (metrics, 0..) |metric, i| { + for (metrics[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, metric.name)) return error.UnsupportedQueryRequest; + } + } } -test "query parser accepts generated count and profile flags" { - const metadata_openapi = @import("antfly_metadata_openapi"); - const full_text = try ant_json.RawValue.init( - \\{"match":{"field":"body","text":"alpha"}} - ); +fn validateGraphSearchOrderMetricNamesUnique(metrics: []const graph_query_mod.GraphMetricOrder) !void { + for (metrics, 0..) |metric, i| { + for (metrics[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, metric.name)) return error.UnsupportedQueryRequest; + } + } +} - const body = try jsonStringifyAlloc(std.testing.allocator, metadata_openapi.QueryRequest{ - .full_text_search = full_text, - .count = true, - .profile = true, - }); - defer std.testing.allocator.free(body); +fn validateGraphSearchMetricStatusList( + query: db_mod.types.NamedGraphQuery, + statuses: []const db_mod.types.GraphMetricStatus, +) !void { + for (statuses, 0..) |status, i| { + const require_published = graphSearchMetricScoreReadRequested(query, status.name); + const allow_unpublished = !require_published and + (query.query.include_metric_status or graphSearchMetricProjected(query, status.name)); + if (status.published_generation == 0) { + if (!allow_unpublished) return error.UnsupportedQueryRequest; + try validateGraphMetricUnpublishedStatus(status); + } else { + try validateGraphMetricPublishedStatus(status); + } + for (statuses[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, status.name)) return error.UnsupportedQueryRequest; + } + } + try validateHitsPairMetricStatusList(statuses); +} - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", body); - defer owned.deinit(std.testing.allocator); - try std.testing.expect(owned.req.count_only); - try std.testing.expect(owned.req.profile); +fn validateHitsPairMetricStatusList(statuses: []const db_mod.types.GraphMetricStatus) !void { + var authority: ?db_mod.types.GraphMetricStatus = null; + var hub: ?db_mod.types.GraphMetricStatus = null; + for (statuses) |status| { + if (isDefaultHitsAuthorityMetric(status.name)) { + authority = status; + } else if (isDefaultHitsHubMetric(status.name)) { + hub = status; + } + } + if (authority) |authority_status| { + if (hub) |hub_status| try validateGraphSearchHitsPairMetricStatusesCompatible(authority_status, hub_status); + } } -test "query parser accepts aggregations" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"match":{"field":"body","text":"alpha"}},"aggregations":{"price_stats":{"type":"stats","field":"price"},"categories":{"type":"terms","field":"category","size":5}}} - ); - defer owned.deinit(std.testing.allocator); +fn validateGraphSearchHitsPairMetricStatusesCompatible( + authority: db_mod.types.GraphMetricStatus, + hub: db_mod.types.GraphMetricStatus, +) !void { + if (authority.published_generation == 0 or hub.published_generation == 0) { + if (authority.published_generation != 0 or hub.published_generation != 0) return error.UnsupportedQueryRequest; + try validateGraphMetricStatusSchemaAndFilterCompatible(authority, hub); + return; + } + try validateHitsPairMetricStatusesCompatible(authority, hub); +} - try std.testing.expect(owned.req.aggregations_json.len > 0); - try std.testing.expect(std.mem.indexOf(u8, owned.req.aggregations_json, "\"price_stats\"") != null); - try std.testing.expect(std.mem.indexOf(u8, owned.req.aggregations_json, "\"categories\"") != null); +fn validateGraphSearchMetricStatusNamesRequested( + query: db_mod.types.NamedGraphQuery, + statuses: []const db_mod.types.GraphMetricStatus, +) !void { + for (statuses) |status| { + if (!graphSearchMetricNameRequested(query, status.name)) return error.UnsupportedQueryRequest; + } } -test "query parser accepts bleve match query shape" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"match":"alpha","field":"body"},"limit":5} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(u32, 5), owned.req.limit); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .match); - try std.testing.expectEqualStrings("body", owned.req.full_text.?.match.field); - try std.testing.expectEqualStrings("alpha", owned.req.full_text.?.match.text); +fn graphSearchMetricNameRequested( + query: db_mod.types.NamedGraphQuery, + metric_name: []const u8, +) bool { + if (graphSearchMetricProjected(query, metric_name)) return true; + if (graphSearchMetricScoreReadRequested(query, metric_name)) return true; + return false; } -test "query parser accepts bleve match_all query shape" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"match_all":{}},"limit":5} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(u32, 5), owned.req.limit); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .match_all); +fn graphSearchMetricProjected( + query: db_mod.types.NamedGraphQuery, + metric_name: []const u8, +) bool { + for (query.query.metrics) |metric| { + if (std.mem.eql(u8, metric.name, metric_name)) return true; + } + return false; } -test "query parser accepts bleve boolean filter shape" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"filter":{"match_all":{}}}} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .bool_query); +fn graphSearchMetricScoreReadRequested( + query: db_mod.types.NamedGraphQuery, + metric_name: []const u8, +) bool { + for (query.query.order_by) |metric| { + if (std.mem.eql(u8, metric.name, metric_name)) return true; + } + for (query.query.where_metric) |metric| { + if (std.mem.eql(u8, metric.name, metric_name)) return true; + } + return false; } -test "query parser preserves filter and exclusion request JSON" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"match":"alpha","field":"body"},"filter_query":{"term":"published","field":"status"},"exclusion_query":{"term":"draft","field":"status"}} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .match); - try ant_json.testing.expectEqualJsonText(std.testing.allocator, - \\{"term":{"path":"status","term":"published"}} - , owned.req.filter_query_json); - try ant_json.testing.expectEqualJsonText(std.testing.allocator, - \\{"term":{"path":"status","term":"draft"}} - , owned.req.exclusion_query_json); +fn validateGraphSearchMetricPayloads( + query: db_mod.types.NamedGraphQuery, + graph_result: db_mod.types.GraphSearchResult, +) !void { + for (graph_result.nodes) |node| try validateGraphResultNodeMetricPayload(query.query.metrics, node); + for (graph_result.matches) |match| { + for (match.bindings) |binding| try validateGraphResultNodeMetricPayload(query.query.metrics, binding.node); + } } -test "query parser does not use dense fast path when public filters are present" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"dense_idx":[0.1,0.2]},"indexes":["dense_idx"],"filter_query":{"term":{"status":"published"}},"exclusion_query":{"term":{"status":"draft"}},"limit":5} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); - try ant_json.testing.expectEqualJsonText(std.testing.allocator, - \\{"term":{"path":"status","term":"published"}} - , owned.req.filter_query_json); - try ant_json.testing.expectEqualJsonText(std.testing.allocator, - \\{"term":{"path":"status","term":"draft"}} - , owned.req.exclusion_query_json); +fn validateGraphResultNodeMetricPayload( + projected_metrics: []const graph_query_mod.GraphMetricRead, + node: graph_query_mod.GraphResultNode, +) !void { + for (node.metrics, 0..) |metric, i| { + if (!graphQueryMetricProjected(projected_metrics, metric.name)) return error.UnsupportedQueryRequest; + for (node.metrics[0..i]) |previous| { + if (std.mem.eql(u8, previous.name, metric.name)) return error.UnsupportedQueryRequest; + } + if (metric.score) |score| { + if (!std.math.isFinite(score)) return error.UnsupportedQueryRequest; + } + } + for (projected_metrics) |projected| { + var found = false; + for (node.metrics) |metric| { + if (!std.mem.eql(u8, metric.name, projected.name)) continue; + found = true; + break; + } + if (!found) return error.UnsupportedQueryRequest; + } } -test "query parser accepts typed bleve leaf queries through db full_text" { - var fuzzy = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"term":"alph","field":"body","fuzziness":1}} - ); - defer fuzzy.deinit(std.testing.allocator); - try std.testing.expect(fuzzy.req.full_text != null); - try std.testing.expect(fuzzy.req.full_text.? == .fuzzy or fuzzy.req.full_text.? == .term); - switch (fuzzy.req.full_text.?) { - .fuzzy => |q| try std.testing.expectEqualStrings("alph", q.term), - .term => |q| try std.testing.expectEqualStrings("alph", q.term), - else => return error.TestUnexpectedResult, +fn graphQueryMetricProjected( + projected_metrics: []const graph_query_mod.GraphMetricRead, + metric_name: []const u8, +) bool { + for (projected_metrics) |metric| { + if (std.mem.eql(u8, metric.name, metric_name)) return true; } + return false; +} - var numeric = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"field":"score","min":10,"max":20,"inclusive_max":true}} - ); - defer numeric.deinit(std.testing.allocator); - try std.testing.expect(numeric.req.full_text != null); - try std.testing.expect(numeric.req.full_text.? == .numeric_range); - try std.testing.expectEqual(@as(f64, 10), numeric.req.full_text.?.numeric_range.min.?); - try std.testing.expectEqual(@as(f64, 20), numeric.req.full_text.?.numeric_range.max.?); - try std.testing.expectEqual(true, numeric.req.full_text.?.numeric_range.inclusive_max); +fn validateGraphSearchMetricStatus( + graph_result: db_mod.types.GraphSearchResult, + metric_name: []const u8, + freshness: graph_query_mod.GraphMetricFreshness, + require_published: bool, +) !void { + var found = false; + for (graph_result.metric_status) |status| { + if (!std.mem.eql(u8, status.name, metric_name)) continue; + if (found) return error.UnsupportedQueryRequest; + found = true; + try validateGraphQueryMetricFreshness(freshness, status, require_published); + } + if (!found) return error.UnsupportedQueryRequest; +} - var date_range = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"field":"created_at","start":"2026-03-01T00:00:00Z","end":"2026-03-31","inclusive_end":true}} - ); - defer date_range.deinit(std.testing.allocator); - try std.testing.expect(date_range.req.full_text != null); - try std.testing.expect(date_range.req.full_text.? == .date_range); - try std.testing.expect(date_range.req.full_text.?.date_range.start_ns != null); - try std.testing.expect(date_range.req.full_text.?.date_range.end_ns != null); - try std.testing.expectEqual(true, date_range.req.full_text.?.date_range.inclusive_end); +fn validateGraphQueryMetricFreshness( + freshness: graph_query_mod.GraphMetricFreshness, + status: db_mod.types.GraphMetricStatus, + require_published: bool, +) !void { + if (require_published and status.published_generation == 0) return error.UnsupportedQueryRequest; + if (freshness == .fresh and status.published_generation == 0) return error.UnsupportedQueryRequest; + if (freshness == .fresh and status.state != .fresh) return error.UnsupportedQueryRequest; } -test "query parser accepts bleve query string queries" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"body:alpha AND title:\"beta gamma\""},"limit":5} - ); - defer owned.deinit(std.testing.allocator); +fn mergeGraphSearchMetricStatus( + alloc: std.mem.Allocator, + statuses: *std.ArrayListUnmanaged(db_mod.types.GraphMetricStatus), + source: db_mod.types.GraphMetricStatus, +) !void { + for (statuses.items) |*status| { + if (!std.mem.eql(u8, status.name, source.name)) continue; + if (status.published_generation != source.published_generation) return error.UnsupportedQueryRequest; + try validateGraphMetricStatusCompatible(status.*, source); + try mergeGraphMetricStatusInto(alloc, status, source); + return; + } + try statuses.append(alloc, try cloneGraphMetricStatus(alloc, source)); +} - try std.testing.expectEqual(@as(u32, 5), owned.req.limit); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .bool_query); - const root = owned.req.full_text.?.bool_query; - try std.testing.expectEqual(@as(usize, 2), root.must.len); - try std.testing.expect(root.must[0] == .match); - try std.testing.expectEqualStrings("body", root.must[0].match.field); - try std.testing.expectEqualStrings("alpha", root.must[0].match.text); - try std.testing.expect(root.must[1] == .match_phrase); - try std.testing.expectEqualStrings("title", root.must[1].match_phrase.field); - try std.testing.expectEqualStrings("beta gamma", root.must[1].match_phrase.text); +fn mergeGraphSearchResults( + alloc: std.mem.Allocator, + queries: []const db_mod.types.NamedGraphQuery, + results: []const db_mod.types.SearchResult, +) ![]db_mod.types.GraphSearchResult { + return mergeGraphSearchResultsWithLimits(alloc, queries, results, .{}); } -test "query parser accepts bleve query string boosts" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"body:alpha^2 AND title:\"beta gamma\"~3^4"}} - ); - defer owned.deinit(std.testing.allocator); +fn mergeGraphSearchResultsWithLimits( + alloc: std.mem.Allocator, + queries: []const db_mod.types.NamedGraphQuery, + results: []const db_mod.types.SearchResult, + limits: graph_work_budget.Limits, +) ![]db_mod.types.GraphSearchResult { + try validateGraphSearchQueryNamesUnique(queries); + var accumulator = try GraphSearchResultsAccumulator.init(alloc, queries, limits); + defer accumulator.deinit(); + // Batch callers already own every input simultaneously. Preserve the + // exact peak accounting used by the legacy batch merge while sharing the + // same folding implementation as incremental distributed fanout. + for (results) |result| try accumulator.admission.admit(result.graph_results); + for (results) |result| try accumulator.appendAdmitted(result.graph_results); + return accumulator.toOwned(); +} + +/// Request-wide graph merge state. Distributed coordinators append one shard +/// at a time, then destroy that shard payload before fetching the next. This +/// keeps both graph memory and count(distinct) identity admission independent +/// of shard count while retaining one-pass O(total input) merge behavior. +pub const GraphSearchResultsAccumulator = struct { + alloc: std.mem.Allocator, + queries: []const db_mod.types.NamedGraphQuery, + admission: GraphPayloadAdmission, + builders: std.ArrayListUnmanaged(GraphSearchResultBuilder) = .empty, + shard_count: usize = 0, + finished: bool = false, + + pub fn init( + alloc: std.mem.Allocator, + queries: []const db_mod.types.NamedGraphQuery, + limits: graph_work_budget.Limits, + ) !GraphSearchResultsAccumulator { + try limits.validate(); + return .{ + .alloc = alloc, + .queries = queries, + .admission = GraphPayloadAdmission.init(queries, limits), + }; + } + + pub fn deinit(self: *GraphSearchResultsAccumulator) void { + for (self.builders.items) |*builder| builder.deinit(self.alloc); + self.builders.deinit(self.alloc); + self.* = undefined; + } + + pub fn appendOwned( + self: *GraphSearchResultsAccumulator, + source_alloc: std.mem.Allocator, + graph_results: *[]db_mod.types.GraphSearchResult, + ) !void { + if (self.finished) return error.InvalidRemoteResponse; + const lease = try self.admission.reserve(graph_results.*); + defer self.admission.release(lease); + try self.appendAdmitted(graph_results.*); + for (graph_results.*) |*graph_result| graph_result.deinit(source_alloc); + if (graph_results.*.len > 0) source_alloc.free(graph_results.*); + graph_results.* = &.{}; + } + + fn appendAdmitted( + self: *GraphSearchResultsAccumulator, + graph_results: []const db_mod.types.GraphSearchResult, + ) !void { + if (self.finished) return error.InvalidRemoteResponse; + try validateGraphQueriesForShard(self.queries, graph_results); + self.shard_count = std.math.add(usize, self.shard_count, 1) catch + return error.InvalidRemoteResponse; + const merge_work_budget = &self.admission.work_budget; + const distinct_budget = &self.admission.distinct_budget; + for (graph_results) |graph_result| { + try validateGraphAggregateShard(self.queries, graph_result); + const merge_query = graphQueryByName(self.queries, graph_result.name) orelse + return error.InvalidRemoteResponse; + try validateGraphSearchMetricStatuses(.{ + .name = graph_result.name, + .query = merge_query, + }, graph_result); + const idx = blk: { + for (self.builders.items, 0..) |builder, i| { + if (std.mem.eql(u8, builder.name, graph_result.name)) break :blk i; + } + try ensureGraphMergeListCapacity( + GraphSearchResultBuilder, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &self.builders, + self.builders.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + graph_result.name.len, + ); + const name = try self.alloc.dupe(u8, graph_result.name); + self.builders.appendAssumeCapacity(.{ .name = name }); + break :blk self.builders.items.len - 1; + }; + var builder = &self.builders.items[idx]; + builder.total_hits +|= graph_result.total_hits; + builder.truncated = builder.truncated or graph_result.truncated; + + // Reject an oversized shard contribution before walking or cloning it. Besides + // preserving the public candidate-budget diagnostic, this keeps adversarial + // payloads from forcing quadratic duplicate validation once the result is + // already known to be unmergeable. + if (graph_result.nodes.len > + public_limits.max_graph_result_items -| builder.nodes.items.len) + { + return error.QueryCandidateBudgetExceeded; + } + for (graph_result.nodes) |node| { + try validateGraphResultNodePayload(node); + const identity = graph_node_identity.Ref{ .table = node.table, .key = node.key }; + if (builder.node_identities.contains(identity)) return error.UnsupportedQueryRequest; + if (builder.nodes.items.len >= public_limits.max_graph_result_items) + return error.QueryCandidateBudgetExceeded; + try ensureGraphIdentityIndexCapacity( + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.node_identities, + builder.nodes.items.len + 1, + ); + try ensureGraphMergeListCapacity( + graph_query_mod.GraphResultNode, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.nodes, + builder.nodes.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + graphResultNodeRetainedBytes(node), + ); + const owned = try cloneGraphResultNode(self.alloc, node); + builder.nodes.appendAssumeCapacity(owned); + const stored = &builder.nodes.items[builder.nodes.items.len - 1]; + builder.node_identities.putAssumeCapacityNoClobber(.{ + .table = stored.table, + .key = stored.key, + }, {}); + } + for (graph_result.paths) |path| { + try validateGraphPathPayload(path); + if (builder.paths.items.len >= public_limits.max_graph_result_items) + return error.QueryCandidateBudgetExceeded; + try ensureGraphMergeListCapacity( + db_mod.types.GraphPath, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.paths, + builder.paths.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + graphPathRetainedBytes(path), + ); + const owned = try cloneGraphPath(self.alloc, path); + builder.paths.appendAssumeCapacity(owned); + } + for (graph_result.matches) |match| { + try validateGraphPatternMatchPayload(match); + const limit = graphQueryReturnLimit(self.queries, graph_result.name); + const effective_limit = if (limit > 0) + @min(@as(usize, limit), public_limits.max_graph_result_items) + else + public_limits.max_graph_result_items; + if (builder.matches.items.len >= effective_limit) { + builder.truncated = true; + continue; + } + try ensureGraphMergeListCapacity( + db_mod.types.GraphPatternMatch, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.matches, + builder.matches.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + graphPatternMatchRetainedBytes(match), + ); + const owned = try cloneGraphPatternMatch(self.alloc, match); + builder.matches.appendAssumeCapacity(owned); + } + for (graph_result.aggregates) |aggregate| { + const distinct = graphAggregateIsDistinct(self.queries, graph_result.name, aggregate.name); + var found = false; + for (builder.aggregates.items) |*existing| { + if (!std.mem.eql(u8, existing.name, aggregate.name)) continue; + if (existing.distinct != distinct) return error.InvalidRemoteResponse; + existing.exact = existing.exact and aggregate.exact; + if (distinct) { + existing.appendDistinctValues(self.alloc, aggregate) catch |err| { + if (err == error.GraphDistinctBudgetExceeded) { + graph_distinct_budget_diagnostic.recordBudget(graph_result.name, distinct_budget); + } + return err; + }; + } else { + existing.value = std.math.add(u128, existing.value, aggregate.value) catch return error.Overflow; + } + found = true; + break; + } + if (!found) { + try ensureGraphMergeListCapacity( + GraphAggregateResultBuilder, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.aggregates, + builder.aggregates.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + aggregate.name.len, + ); + const name = try self.alloc.dupe(u8, aggregate.name); + var owned = GraphAggregateResultBuilder{ + .name = name, + .value = if (distinct) 0 else aggregate.value, + .exact = aggregate.exact, + .distinct = distinct, + .distinct_budget = distinct_budget, + }; + var owned_active = true; + errdefer if (owned_active) owned.deinit(self.alloc); + if (distinct) owned.appendDistinctValues(self.alloc, aggregate) catch |err| { + if (err == error.GraphDistinctBudgetExceeded) { + graph_distinct_budget_diagnostic.recordBudget(graph_result.name, distinct_budget); + } + return err; + }; + builder.aggregates.appendAssumeCapacity(owned); + owned_active = false; + } + } + for (graph_result.hits) |hit| { + try validateGraphSearchHitPayload(hit); + const identity = graph_node_identity.Ref{ .table = hit.source_table, .key = hit.id }; + if (builder.hit_identities.contains(identity)) return error.UnsupportedQueryRequest; + if (builder.hits.items.len >= public_limits.max_graph_hydrated_bindings) + return error.QueryCandidateBudgetExceeded; + try ensureGraphIdentityIndexCapacity( + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.hit_identities, + builder.hits.items.len + 1, + ); + try ensureGraphMergeListCapacity( + db_mod.types.SearchHit, + self.alloc, + merge_work_budget, + graph_result.name, + merge_query, + &builder.hits, + builder.hits.items.len + 1, + ); + try retainGraphMergeBytes( + merge_work_budget, + graph_result.name, + merge_query, + graphHitRetainedBytes(hit), + ); + const owned = try hit.clone(self.alloc); + builder.hits.appendAssumeCapacity(owned); + const stored = &builder.hits.items[builder.hits.items.len - 1]; + builder.hit_identities.putAssumeCapacityNoClobber(.{ + .table = stored.source_table, + .key = stored.id, + }, {}); + } + for (graph_result.metric_status) |status| { + try mergeGraphSearchMetricStatus(self.alloc, &builder.metric_status, status); + } + } + } + + pub fn toOwned(self: *GraphSearchResultsAccumulator) ![]db_mod.types.GraphSearchResult { + if (self.finished) return error.InvalidRemoteResponse; + self.finished = true; + const merge_work_budget = &self.admission.work_budget; + const distinct_budget = &self.admission.distinct_budget; + for (self.builders.items) |builder| { + const query = graphQueryByName(self.queries, builder.name) orelse + return error.InvalidRemoteResponse; + try retainGraphMergeBytes( + merge_work_budget, + builder.name, + query, + @sizeOf(db_mod.types.GraphSearchResult), + ); + } + const merged = try self.alloc.alloc(db_mod.types.GraphSearchResult, self.builders.items.len); + var initialized: usize = 0; + errdefer { + for (merged[0..initialized]) |*graph_result| graph_result.deinit(self.alloc); + self.alloc.free(merged); + } + for (self.builders.items, 0..) |*builder, i| { + const query = graphQueryByName(self.queries, builder.name) orelse + return error.InvalidRemoteResponse; + merged[i] = builder.toOwned(self.alloc, merge_work_budget, query) catch |err| { + if (err == error.GraphDistinctBudgetExceeded) { + graph_distinct_budget_diagnostic.recordBudget(builder.name, distinct_budget); + } + return err; + }; + if (self.shard_count > 1) clearMergedDocOrdinals(merged[i].hits); + initialized += 1; + } + return merged; + } +}; + +/// A shard graph response is operation-keyed just like the public contract. +/// Empty operations are represented by an empty result, never by omitting the +/// operation, so version skew or a broken worker cannot silently turn an exact +/// graph request into a partial success. +fn validateGraphQueriesForShard( + queries: []const db_mod.types.NamedGraphQuery, + graph_results: []const db_mod.types.GraphSearchResult, +) !void { + for (queries) |named| { + var occurrences: usize = 0; + for (graph_results) |graph_result| { + if (std.mem.eql(u8, named.name, graph_result.name)) occurrences += 1; + } + if (occurrences != 1) return error.InvalidRemoteResponse; + } + + for (graph_results) |graph_result| { + var requested = false; + for (queries) |named| { + if (!std.mem.eql(u8, named.name, graph_result.name)) continue; + requested = true; + break; + } + if (!requested) return error.InvalidRemoteResponse; + } +} + +fn validateGraphAggregateShard( + queries: []const db_mod.types.NamedGraphQuery, + graph_result: db_mod.types.GraphSearchResult, +) !void { + const query = blk: { + for (queries) |named| { + if (std.mem.eql(u8, named.name, graph_result.name)) break :blk named.query; + } + return error.InvalidRemoteResponse; + }; + if (query.aggregates.len == 0) { + if (graph_result.aggregates.len != 0) return error.InvalidRemoteResponse; + return; + } + if (graph_result.aggregates.len != query.aggregates.len) return error.InvalidRemoteResponse; + for (query.aggregates) |requested| { + var occurrences: usize = 0; + for (graph_result.aggregates) |aggregate| { + if (!std.mem.eql(u8, requested.name, aggregate.name)) continue; + if (!aggregate.exact) return error.QueryCandidateBudgetExceeded; + if (requested.distinct) { + if (aggregate.value != aggregate.distinct_values.len) + return error.InvalidRemoteResponse; + } else if (aggregate.distinct_values.len != 0) { + return error.InvalidRemoteResponse; + } + occurrences += 1; + } + if (occurrences != 1) return error.InvalidRemoteResponse; + } +} + +fn graphAggregateIsDistinct( + queries: []const db_mod.types.NamedGraphQuery, + query_name: []const u8, + aggregate_name: []const u8, +) bool { + for (queries) |named| { + if (!std.mem.eql(u8, named.name, query_name)) continue; + for (named.query.aggregates) |aggregate| { + if (std.mem.eql(u8, aggregate.name, aggregate_name)) return aggregate.distinct; + } + return false; + } + return false; +} + +fn graphQueryReturnLimit( + queries: []const db_mod.types.NamedGraphQuery, + query_name: []const u8, +) u32 { + for (queries) |named| { + if (std.mem.eql(u8, named.name, query_name)) return named.query.return_limit; + } + return 0; +} + +fn cloneGraphNodeRefs(alloc: std.mem.Allocator, values: []const graph_node_identity.Ref) ![]graph_node_identity.Ref { + var out = std.ArrayListUnmanaged(graph_node_identity.Ref).empty; + errdefer { + for (out.items) |value| { + if (value.table) |table| alloc.free(table); + alloc.free(value.key); + } + out.deinit(alloc); + } + try out.ensureTotalCapacity(alloc, values.len); + for (values) |value| try appendClonedGraphNodeRef(alloc, &out, value); + return try out.toOwnedSlice(alloc); +} + +fn appendClonedGraphNodeRef( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged(graph_node_identity.Ref), + value: graph_node_identity.Ref, +) !void { + const table = if (value.table) |table_name| try alloc.dupe(u8, table_name) else null; + errdefer if (table) |table_name| alloc.free(table_name); + const key = try alloc.dupe(u8, value.key); + errdefer alloc.free(key); + try out.append(alloc, .{ .table = table, .key = key }); +} + +fn freeGraphNodeRefs(alloc: std.mem.Allocator, values: []const graph_node_identity.Ref) void { + for (values) |value| { + if (value.table) |table| alloc.free(table); + alloc.free(value.key); + } + if (values.len > 0) alloc.free(values); +} + +fn cloneGraphSearchResult( + alloc: std.mem.Allocator, + source: db_mod.types.GraphSearchResult, +) !db_mod.types.GraphSearchResult { + const GraphNode = std.meta.Child(@TypeOf(source.nodes)); + const nodes = try alloc.alloc(GraphNode, source.nodes.len); + var initialized_nodes: usize = 0; + errdefer { + for (nodes[0..initialized_nodes]) |*node| node.deinit(alloc); + if (source.nodes.len > 0) alloc.free(nodes); + } + for (source.nodes, 0..) |node, i| { + nodes[i] = try cloneGraphResultNode(alloc, node); + initialized_nodes += 1; + } + + const GraphPath = std.meta.Child(@TypeOf(source.paths)); + const paths = try alloc.alloc(GraphPath, source.paths.len); + var initialized_paths: usize = 0; + errdefer { + for (paths[0..initialized_paths]) |path| graph_paths.freePath(alloc, path); + if (source.paths.len > 0) alloc.free(paths); + } + for (source.paths, 0..) |path, i| { + paths[i] = try cloneGraphPath(alloc, path); + initialized_paths += 1; + } + + const hits = try alloc.alloc(db_mod.types.SearchHit, source.hits.len); + var initialized_hits: usize = 0; + errdefer { + for (hits[0..initialized_hits]) |*hit| hit.deinit(alloc); + if (source.hits.len > 0) alloc.free(hits); + } + for (source.hits, 0..) |hit, i| { + hits[i] = try hit.clone(alloc); + initialized_hits += 1; + } + + const matches = try alloc.alloc(db_mod.types.GraphPatternMatch, source.matches.len); + var initialized_matches: usize = 0; + errdefer { + for (matches[0..initialized_matches]) |*match| match.deinit(alloc); + if (source.matches.len > 0) alloc.free(matches); + } + for (source.matches, 0..) |match, i| { + matches[i] = try cloneGraphPatternMatch(alloc, match); + initialized_matches += 1; + } + + const aggregates = try alloc.alloc(db_mod.types.GraphAggregateResult, source.aggregates.len); + var initialized_aggregates: usize = 0; + errdefer { + for (aggregates[0..initialized_aggregates]) |*aggregate| aggregate.deinit(alloc); + if (source.aggregates.len > 0) alloc.free(aggregates); + } + for (source.aggregates, 0..) |aggregate, i| { + const name = try alloc.dupe(u8, aggregate.name); + const distinct_values = cloneGraphNodeRefs(alloc, aggregate.distinct_values) catch |err| { + alloc.free(name); + return err; + }; + aggregates[i] = .{ .name = name, .value = aggregate.value, .exact = aggregate.exact, .distinct_values = distinct_values }; + initialized_aggregates += 1; + } + + return .{ + .name = try alloc.dupe(u8, source.name), + .nodes = nodes, + .paths = paths, + .matches = matches, + .aggregates = aggregates, + .hits = hits, + .total_hits = source.total_hits, + .truncated = source.truncated, + }; +} + +pub fn cloneGraphMetricStatus( + alloc: std.mem.Allocator, + source: db_mod.types.GraphMetricStatus, +) !db_mod.types.GraphMetricStatus { + const name = try alloc.dupe(u8, source.name); + errdefer alloc.free(name); + var edge_filter = try source.edge_filter.cloneAlloc(alloc); + errdefer edge_filter.deinit(alloc); + const recent_events = if (source.recent_events.len > 0) + try alloc.dupe(graph_mod.GraphIndex.GraphMetricEvent, source.recent_events) + else + @constCast((&[_]graph_mod.GraphIndex.GraphMetricEvent{})[0..]); + errdefer if (recent_events.len > 0) alloc.free(recent_events); + const last_error = if (source.last_error.len > 0) try alloc.dupe(u8, source.last_error) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + const build_worker_id = if (source.build_worker_id.len > 0) try alloc.dupe(u8, source.build_worker_id) else ""; + errdefer if (build_worker_id.len > 0) alloc.free(build_worker_id); + const build_cursor = if (source.build_cursor.len > 0) try alloc.dupe(u8, source.build_cursor) else ""; + errdefer if (build_cursor.len > 0) alloc.free(build_cursor); + const build_pages = try cloneGraphMetricBuildPageStatuses(alloc, source.build_pages); + errdefer { + for (build_pages) |*page| page.deinit(alloc); + if (build_pages.len > 0) alloc.free(build_pages); + } + return .{ + .name = name, + .state = source.state, + .phase = source.phase, + .edge_filter = edge_filter, + .metadata_version = source.metadata_version, + .config_fingerprint = source.config_fingerprint, + .maintenance_paused = source.maintenance_paused, + .build_queued = source.build_queued, + .published_generation = source.published_generation, + .edge_generation = source.edge_generation, + .target_edge_generation = source.target_edge_generation, + .queued_generation = source.queued_generation, + .building_generation = source.building_generation, + .build_job_id = source.build_job_id, + .build_started_at_ms = source.build_started_at_ms, + .build_iteration = source.build_iteration, + .build_lease_expires_at_ms = source.build_lease_expires_at_ms, + .build_worker_id = build_worker_id, + .build_cursor = build_cursor, + .build_completed_units = source.build_completed_units, + .build_total_units = source.build_total_units, + .build_pages = build_pages, + .build_pages_truncated = source.build_pages_truncated, + .retry_count = source.retry_count, + .last_error = last_error, + .progress = source.progress, + .converged = source.converged, + .iterations_completed = source.iterations_completed, + .delta = source.delta, + .computed_at_ms = source.computed_at_ms, + .last_event = source.last_event, + .recent_events = recent_events, + }; +} + +fn cloneGraphMetricBuildPageStatuses( + alloc: std.mem.Allocator, + source: []const db_mod.types.GraphMetricBuildPageStatus, +) ![]db_mod.types.GraphMetricBuildPageStatus { + if (source.len == 0) return @constCast((&[_]db_mod.types.GraphMetricBuildPageStatus{})[0..]); + const out = try alloc.alloc(db_mod.types.GraphMetricBuildPageStatus, source.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*page| page.deinit(alloc); + alloc.free(out); + } + for (source, 0..) |page, i| { + const worker_id = if (page.worker_id.len > 0) try alloc.dupe(u8, page.worker_id) else ""; + errdefer if (worker_id.len > 0) alloc.free(worker_id); + const cursor = if (page.cursor.len > 0) try alloc.dupe(u8, page.cursor) else ""; + errdefer if (cursor.len > 0) alloc.free(cursor); + const last_error = if (page.last_error.len > 0) try alloc.dupe(u8, page.last_error) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + out[i] = .{ + .phase = page.phase, + .iteration = page.iteration, + .page_id = page.page_id, + .state = page.state, + .range_kind = page.range_kind, + .worker_id = worker_id, + .lease_expires_at_ms = page.lease_expires_at_ms, + .attempt = page.attempt, + .cursor = cursor, + .completed_units = page.completed_units, + .total_units = page.total_units, + .last_error = last_error, + }; + initialized += 1; + } + return out; +} + +fn cloneGraphPatternMatch( + alloc: std.mem.Allocator, + source: db_mod.types.GraphPatternMatch, +) !db_mod.types.GraphPatternMatch { + const bindings = try alloc.alloc(db_mod.types.GraphPatternBinding, source.bindings.len); + var initialized_bindings: usize = 0; + errdefer { + for (bindings[0..initialized_bindings]) |*binding| binding.deinit(alloc); + if (source.bindings.len > 0) alloc.free(bindings); + } + for (source.bindings, 0..) |binding, i| { + const alias = try alloc.dupe(u8, binding.alias); + errdefer alloc.free(alias); + const node = try cloneGraphResultNode(alloc, binding.node); + bindings[i] = .{ .alias = alias, .node = node }; + initialized_bindings += 1; + } + + const path = try alloc.alloc(graph_query_mod.PathEdgeInfo, source.path.len); + var initialized_path: usize = 0; + errdefer { + for (path[0..initialized_path]) |edge| { + alloc.free(edge.source); + alloc.free(edge.target); + alloc.free(edge.edge_type); + if (edge.metadata.len > 0) alloc.free(edge.metadata); + } + if (source.path.len > 0) alloc.free(path); + } + for (source.path, 0..) |edge, i| { + path[i] = try clonePathEdge(graph_query_mod.PathEdgeInfo, alloc, edge); + initialized_path += 1; + } + + const null_aliases = try alloc.alloc([]u8, source.null_aliases.len); + var initialized_null_aliases: usize = 0; + errdefer { + for (null_aliases[0..initialized_null_aliases]) |alias| alloc.free(alias); + if (source.null_aliases.len > 0) alloc.free(null_aliases); + } + for (source.null_aliases, 0..) |alias, i| { + null_aliases[i] = try alloc.dupe(u8, alias); + initialized_null_aliases += 1; + } + + return .{ + .bindings = bindings, + .path = path, + .null_aliases = null_aliases, + }; +} + +fn cloneGraphResultNode( + alloc: std.mem.Allocator, + source: graph_query_mod.GraphResultNode, +) !graph_query_mod.GraphResultNode { + const key = try alloc.dupe(u8, source.key); + errdefer alloc.free(key); + const table = if (source.table) |value| try alloc.dupe(u8, value) else null; + errdefer if (table) |value| alloc.free(value); + const path = if (source.path) |items| try cloneStringSlice(alloc, items) else null; + errdefer if (path) |items| freeStringSlice(alloc, items); + const path_tables = if (source.path_tables) |items| try cloneOptionalStringSlice(alloc, items) else null; + errdefer if (path_tables) |items| freeOptionalStringSlice(alloc, items); + const path_edges = if (source.path_edges) |items| + try clonePathEdges(graph_query_mod.PathEdgeInfo, alloc, items) + else + null; + errdefer if (path_edges) |items| freePathEdges(alloc, items); + const provenance = if (source.provenance) |items| + try cloneStringSlice(alloc, items) + else + null; + errdefer if (provenance) |items| freeStringSlice(alloc, items); + + const metrics = try alloc.alloc(graph_query_mod.GraphMetricValue, source.metrics.len); + var initialized_metrics: usize = 0; + errdefer { + for (metrics[0..initialized_metrics]) |*metric| metric.deinit(alloc); + if (source.metrics.len > 0) alloc.free(metrics); + } + for (source.metrics, 0..) |metric, i| { + metrics[i] = .{ + .name = try alloc.dupe(u8, metric.name), + .score = metric.score, + }; + initialized_metrics += 1; + } + + return .{ + .key = key, + .depth = source.depth, + .distance = source.distance, + .path = path, + .path_tables = path_tables, + .path_edges = path_edges, + .provenance = provenance, + .table = table, + .metrics = metrics, + }; +} + +fn cloneGraphPath( + alloc: std.mem.Allocator, + source: db_mod.types.GraphPath, +) !db_mod.types.GraphPath { + const nodes = try cloneStringSlice(alloc, source.nodes); + errdefer freeStringSlice(alloc, nodes); + const node_tables = try cloneOptionalStringSlice(alloc, source.node_tables); + errdefer freeOptionalStringSlice(alloc, node_tables); + const edges = try clonePathEdges(graph_paths.PathEdge, alloc, source.edges); + errdefer freePathEdges(alloc, edges); + + return .{ + .nodes = nodes, + .node_tables = node_tables, + .edges = edges, + .total_weight = source.total_weight, + .length = source.length, + }; +} + +fn cloneStringSlice( + alloc: std.mem.Allocator, + source: []const []const u8, +) ![][]const u8 { + const out = try alloc.alloc([]const u8, source.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| alloc.free(item); + alloc.free(out); + } + for (source, 0..) |item, i| { + out[i] = try alloc.dupe(u8, item); + initialized += 1; + } + return out; +} + +fn freeStringSlice(alloc: std.mem.Allocator, items: []const []const u8) void { + for (items) |item| alloc.free(item); + alloc.free(items); +} + +fn cloneOptionalStringSlice( + alloc: std.mem.Allocator, + source: []const ?[]const u8, +) ![]?[]const u8 { + if (source.len == 0) return &.{}; + const out = try alloc.alloc(?[]const u8, source.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| if (item) |value| alloc.free(value); + alloc.free(out); + } + for (source, 0..) |item, i| { + out[i] = if (item) |value| try alloc.dupe(u8, value) else null; + initialized += 1; + } + return out; +} + +fn freeOptionalStringSlice( + alloc: std.mem.Allocator, + items: []const ?[]const u8, +) void { + for (items) |item| if (item) |value| alloc.free(value); + if (items.len > 0) alloc.free(items); +} + +fn clonePathEdges( + comptime Edge: type, + alloc: std.mem.Allocator, + source: anytype, +) ![]Edge { + const out = try alloc.alloc(Edge, source.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |edge| freePathEdge(alloc, edge); + alloc.free(out); + } + for (source, 0..) |edge, i| { + out[i] = try clonePathEdge(Edge, alloc, edge); + initialized += 1; + } + return out; +} + +fn clonePathEdge( + comptime Edge: type, + alloc: std.mem.Allocator, + source: anytype, +) !Edge { + const edge_source = try alloc.dupe(u8, source.source); + errdefer alloc.free(edge_source); + const target = try alloc.dupe(u8, source.target); + errdefer alloc.free(target); + const edge_type = try alloc.dupe(u8, source.edge_type); + errdefer alloc.free(edge_type); + const metadata = if (source.metadata.len > 0) + try alloc.dupe(u8, source.metadata) + else + ""; + errdefer if (metadata.len > 0) alloc.free(metadata); + return .{ + .source = edge_source, + .target = target, + .edge_type = edge_type, + .weight = source.weight, + .metadata = metadata, + .traversal_direction = source.traversal_direction, + }; +} + +fn freePathEdges(alloc: std.mem.Allocator, edges: anytype) void { + for (edges) |edge| freePathEdge(alloc, edge); + alloc.free(edges); +} + +fn freePathEdge(alloc: std.mem.Allocator, edge: anytype) void { + alloc.free(edge.source); + alloc.free(edge.target); + alloc.free(edge.edge_type); + if (edge.metadata.len > 0) alloc.free(edge.metadata); +} + +test "query parser accepts full text request subset" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match":{"field":"body","text":"alpha"}},"fields":["title"],"limit":5} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, 5), owned.req.limit); + try std.testing.expectEqual(@as(usize, 1), owned.req.fields.len); + try std.testing.expectEqual(false, owned.req.include_all_fields); +} + +test "query parser accepts generated query request shape" { + const metadata_openapi = @import("antfly_metadata_openapi"); + const full_text = try ant_json.RawValue.init( + \\{"match":{"field":"body","text":"alpha"}} + ); + + const body = try jsonStringifyAlloc(std.testing.allocator, metadata_openapi.QueryRequest{ + .full_text_search = full_text, + .fields = &.{"title"}, + .limit = 5, + .profile = false, + }); + defer std.testing.allocator.free(body); + + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", body); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, 5), owned.req.limit); + try std.testing.expectEqual(@as(usize, 1), owned.req.fields.len); + try std.testing.expectEqualStrings("title", owned.req.fields[0]); +} + +test "query parser defers ordinary stored projection to response encoding" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"fields":["id","title"],"limit":5} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 2), owned.req.fields.len); + try std.testing.expect(owned.req.defer_stored_projection); +} + +test "query parser defaults to stored documents when fields are omitted" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"limit":5} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 0), owned.req.fields.len); + try std.testing.expectEqual(true, owned.req.include_all_fields); + try std.testing.expectEqual(true, owned.req.include_stored); + try std.testing.expectEqual(false, owned.req.defer_stored_projection); +} + +test "query parser keeps special stored projection in db layer" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"fields":["title","_chunks.*"],"limit":5} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 2), owned.req.fields.len); + try std.testing.expect(!owned.req.defer_stored_projection); +} + +test "query parser accepts generated count and profile flags" { + const metadata_openapi = @import("antfly_metadata_openapi"); + const full_text = try ant_json.RawValue.init( + \\{"match":{"field":"body","text":"alpha"}} + ); + + const body = try jsonStringifyAlloc(std.testing.allocator, metadata_openapi.QueryRequest{ + .full_text_search = full_text, + .count = true, + .profile = true, + }); + defer std.testing.allocator.free(body); + + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", body); + defer owned.deinit(std.testing.allocator); + try std.testing.expect(owned.req.count_only); + try std.testing.expect(owned.req.profile); +} + +test "query parser accepts aggregations" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match":{"field":"body","text":"alpha"}},"aggregations":{"price_stats":{"type":"stats","field":"price"},"categories":{"type":"terms","field":"category","size":5}}} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.aggregations_json.len > 0); + try std.testing.expect(std.mem.indexOf(u8, owned.req.aggregations_json, "\"price_stats\"") != null); + try std.testing.expect(std.mem.indexOf(u8, owned.req.aggregations_json, "\"categories\"") != null); +} + +test "query parser accepts bleve match query shape" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match":"alpha","field":"body"},"limit":5} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, 5), owned.req.limit); + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .match); + try std.testing.expectEqualStrings("body", owned.req.full_text.?.match.field); + try std.testing.expectEqualStrings("alpha", owned.req.full_text.?.match.text); +} + +test "query parser accepts bleve match_all query shape" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match_all":{}},"limit":5} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, 5), owned.req.limit); + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .match_all); +} + +test "query parser accepts bleve boolean filter shape" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"filter":{"match_all":{}}}} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .bool_query); +} + +test "query parser preserves filter and exclusion request JSON" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match":"alpha","field":"body"},"filter_query":{"term":"published","field":"status"},"exclusion_query":{"term":"draft","field":"status"}} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .match); + try ant_json.testing.expectEqualJsonText(std.testing.allocator, + \\{"term":{"path":"status","term":"published"}} + , owned.req.filter_query_json); + try ant_json.testing.expectEqualJsonText(std.testing.allocator, + \\{"term":{"path":"status","term":"draft"}} + , owned.req.exclusion_query_json); +} + +test "query parser does not use dense fast path when public filters are present" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"dense_idx":[0.1,0.2]},"indexes":["dense_idx"],"filter_query":{"term":{"status":"published"}},"exclusion_query":{"term":{"status":"draft"}},"limit":5} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); + try ant_json.testing.expectEqualJsonText(std.testing.allocator, + \\{"term":{"path":"status","term":"published"}} + , owned.req.filter_query_json); + try ant_json.testing.expectEqualJsonText(std.testing.allocator, + \\{"term":{"path":"status","term":"draft"}} + , owned.req.exclusion_query_json); +} + +test "query parser accepts typed bleve leaf queries through db full_text" { + var fuzzy = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"term":"alph","field":"body","fuzziness":1}} + ); + defer fuzzy.deinit(std.testing.allocator); + try std.testing.expect(fuzzy.req.full_text != null); + try std.testing.expect(fuzzy.req.full_text.? == .fuzzy or fuzzy.req.full_text.? == .term); + switch (fuzzy.req.full_text.?) { + .fuzzy => |q| try std.testing.expectEqualStrings("alph", q.term), + .term => |q| try std.testing.expectEqualStrings("alph", q.term), + else => return error.TestUnexpectedResult, + } + + var numeric = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"field":"score","min":10,"max":20,"inclusive_max":true}} + ); + defer numeric.deinit(std.testing.allocator); + try std.testing.expect(numeric.req.full_text != null); + try std.testing.expect(numeric.req.full_text.? == .numeric_range); + try std.testing.expectEqual(@as(f64, 10), numeric.req.full_text.?.numeric_range.min.?); + try std.testing.expectEqual(@as(f64, 20), numeric.req.full_text.?.numeric_range.max.?); + try std.testing.expectEqual(true, numeric.req.full_text.?.numeric_range.inclusive_max); + + var date_range = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"field":"created_at","start":"2026-03-01T00:00:00Z","end":"2026-03-31","inclusive_end":true}} + ); + defer date_range.deinit(std.testing.allocator); + try std.testing.expect(date_range.req.full_text != null); + try std.testing.expect(date_range.req.full_text.? == .date_range); + try std.testing.expect(date_range.req.full_text.?.date_range.start_ns != null); + try std.testing.expect(date_range.req.full_text.?.date_range.end_ns != null); + try std.testing.expectEqual(true, date_range.req.full_text.?.date_range.inclusive_end); +} + +test "query parser accepts bleve query string queries" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"body:alpha AND title:\"beta gamma\""},"limit":5} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(u32, 5), owned.req.limit); + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .bool_query); + const root = owned.req.full_text.?.bool_query; + try std.testing.expectEqual(@as(usize, 2), root.must.len); + try std.testing.expect(root.must[0] == .match); + try std.testing.expectEqualStrings("body", root.must[0].match.field); + try std.testing.expectEqualStrings("alpha", root.must[0].match.text); + try std.testing.expect(root.must[1] == .match_phrase); + try std.testing.expectEqualStrings("title", root.must[1].match_phrase.field); + try std.testing.expectEqualStrings("beta gamma", root.must[1].match_phrase.text); +} + +test "query parser accepts bleve query string boosts" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"body:alpha^2 AND title:\"beta gamma\"~3^4"}} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .bool_query); + const root = owned.req.full_text.?.bool_query; + try std.testing.expectEqual(@as(usize, 2), root.must.len); + try std.testing.expect(root.must[0] == .match); + try std.testing.expectApproxEqAbs(@as(f32, 2.0), root.must[0].match.boost, 0.0001); + try std.testing.expect(root.must[1] == .match_phrase); + try std.testing.expectApproxEqAbs(@as(f32, 4.0), root.must[1].match_phrase.boost, 0.0001); +} + +test "query parser accepts bleve query string field groups" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"title:(alpha beta)"}} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .bool_query); + const root = owned.req.full_text.?.bool_query; + try std.testing.expectEqual(@as(usize, 2), root.must.len); + try std.testing.expect(root.must[0] == .match); + try std.testing.expect(root.must[1] == .match); + try std.testing.expectEqualStrings("title", root.must[0].match.field); + try std.testing.expectEqualStrings("alpha", root.must[0].match.text); + try std.testing.expectEqualStrings("title", root.must[1].match.field); + try std.testing.expectEqualStrings("beta", root.must[1].match.text); +} + +test "query parser accepts bleve query string inline ranges" { + var numeric = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"score:[10 TO 20}"}} + ); + defer numeric.deinit(std.testing.allocator); + + try std.testing.expect(numeric.req.full_text != null); + try std.testing.expect(numeric.req.full_text.? == .numeric_range); + try std.testing.expectEqual(@as(f64, 10), numeric.req.full_text.?.numeric_range.min.?); + try std.testing.expectEqual(@as(f64, 20), numeric.req.full_text.?.numeric_range.max.?); + try std.testing.expect(numeric.req.full_text.?.numeric_range.inclusive_min); + try std.testing.expect(!numeric.req.full_text.?.numeric_range.inclusive_max); + + var date = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"created:[2024-01-01T00:00:00Z TO 2024-12-31T00:00:00Z]"}} + ); + defer date.deinit(std.testing.allocator); + + try std.testing.expect(date.req.full_text != null); + try std.testing.expect(date.req.full_text.? == .date_range); + try std.testing.expect(date.req.full_text.?.date_range.start_ns != null); + try std.testing.expect(date.req.full_text.?.date_range.end_ns != null); + + var term = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"title:[alpha TO omega]"}} + ); + defer term.deinit(std.testing.allocator); + + try std.testing.expect(term.req.full_text != null); + try std.testing.expect(term.req.full_text.? == .term_range); + try std.testing.expectEqualStrings("alpha", term.req.full_text.?.term_range.min.?); + try std.testing.expectEqualStrings("omega", term.req.full_text.?.term_range.max.?); +} + +test "query parser accepts bleve query string filters" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"query":"alpha"},"filter_query":{"query":"status:published OR status:review"}} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.full_text != null); + try std.testing.expect(owned.req.full_text.? == .match); + try ant_json.testing.expectEqualJsonText(std.testing.allocator, + \\{"bool":{"should":[{"match":{"path":"status","text":"published"}},{"match":{"path":"status","text":"review"}}],"minimum_should_match":1}} + , owned.req.filter_query_json); +} + +test "query parser rejects invalid bleve date ranges" { + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"field":"created_at","start":"not-a-date"}} + )); +} + +test "query parser resolves semantic search into dense query" { + var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); + try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); + try std.testing.expectEqual(@as(u32, 4), owned.req.dense_queries[0].query.k); + try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); +} + +test "query parser preserves search effort for semantic search" { + var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4,"search_effort":0.3} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.search_effort != null); + try std.testing.expectApproxEqAbs(@as(f32, 0.3), owned.req.search_effort.?, 0.0001); +} + +test "query parser accepts semantic embedding template" { + var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","embedding_template":"{{remotePDF url=this}}","indexes":["semantic_idx"],"limit":4} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); + try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); +} + +test "query parser accepts precomputed embedding payload" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"semantic_idx":[0.5,1.5,2.5]},"limit":6} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); + try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); + try std.testing.expectEqual(@as(u32, 6), owned.req.dense_queries[0].query.k); + try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); + try std.testing.expectEqual(@as(f32, 1.5), owned.req.dense_queries[0].query.vector[1]); +} + +test "query parser accepts packed dense embedding payload" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"semantic_idx":"AAAAPwAAwD8AACBA"},"indexes":["semantic_idx"],"fields":["title"],"search_effort":0.3,"limit":6} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); + try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); + try std.testing.expectEqual(@as(u32, 6), owned.req.dense_queries[0].query.k); + try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); + try std.testing.expectApproxEqAbs(@as(f32, 0.5), owned.req.dense_queries[0].query.vector[0], 0.0001); + try std.testing.expectApproxEqAbs(@as(f32, 1.5), owned.req.dense_queries[0].query.vector[1], 0.0001); + try std.testing.expectApproxEqAbs(@as(f32, 2.5), owned.req.dense_queries[0].query.vector[2], 0.0001); + try std.testing.expectEqual(@as(?f32, 0.3), owned.req.search_effort); + try std.testing.expect(owned.req.defer_stored_projection); +} + +test "query parser rejects packed dense indexes that reference a missing embedding" { + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"semantic_idx":"AAAAPwAAwD8AACBA"},"indexes":["missing_idx"],"limit":6} + )); +} + +test "query parser rejects invalid packed dense embedding payload" { + try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"semantic_idx":"not-base64"},"indexes":["semantic_idx"],"limit":6} + )); +} + +test "query parser accepts sparse embedding payload" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"sparse_idx":{"indices":[1,7],"values":[0.4,0.9]}},"limit":6} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.sparse_queries.len); + try std.testing.expectEqualStrings("sparse_idx", owned.req.sparse_queries[0].index_name); + try std.testing.expectEqual(@as(u32, 6), owned.req.sparse_queries[0].query.k); + try std.testing.expectEqual(@as(usize, 2), owned.req.sparse_queries[0].query.indices.len); + try std.testing.expectEqual(@as(u32, 7), owned.req.sparse_queries[0].query.indices[1]); +} + +test "query parser accepts merge config reranker and pruner" { + var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"full_text_search":{"match":{"field":"body","text":"alpha concept"}},"merge_config":{"strategy":"rsf","window_size":25,"rank_constant":42.0,"weights":{"full_text":0.5,"semantic_idx":1.5}},"reranker":{"provider":"antfly","model":"cross-encoder/ms-marco-MiniLM-L-6-v2","field":"body","top_n":3},"pruner":{"min_score_ratio":0.5,"require_multi_index":true},"limit":6} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.merge_config != null); + try std.testing.expectEqual(.rsf, owned.req.merge_config.?.strategy); + try std.testing.expectEqual(@as(u32, 25), owned.req.merge_config.?.window_size); + try std.testing.expectEqual(@as(usize, 2), owned.req.merge_config.?.weights.len); + try std.testing.expect(owned.req.reranker != null); + try std.testing.expectEqual(.antfly, owned.req.reranker.?.provider); + try std.testing.expectEqualStrings("body", owned.req.reranker.?.field); + try std.testing.expectEqual(@as(?u32, 3), owned.req.reranker.?.top_n); + try std.testing.expectEqualStrings("alpha concept", owned.req.reranker_query_text); + try std.testing.expect(owned.req.include_stored); + try std.testing.expect(owned.req.pruner != null); + try std.testing.expectEqual(@as(f64, 0.5), owned.req.pruner.?.min_score_ratio); + try std.testing.expect(owned.req.pruner.?.require_multi_index); +} + +test "query parser rejects dense reranking without query text" { + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"embeddings":{"dense_idx":[1.0,0.0,0.0]},"indexes":["dense_idx"],"reranker":{"provider":"antfly","model":"cross-encoder/ms-marco-MiniLM-L-6-v2","field":"body","top_n":2},"limit":6} + )); +} + +test "query parser accepts graph queries" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"neighbors":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]},"edge_types":["links"],"max_depth":1}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); + try std.testing.expectEqualStrings("neighbors", owned.req.graph_queries[0].name); + try std.testing.expectEqualStrings("graph_idx", owned.req.graph_queries[0].query.index_name); + try std.testing.expect(owned.req.graph_queries[0].query.query_type == .traverse); + switch (owned.req.graph_queries[0].query.start_nodes) { + .identities => |identities| { + try std.testing.expectEqualStrings("doc:a", identities[0].key); + try std.testing.expect(identities[0].table == null); + }, + else => return error.TestUnexpectedResult, + } +} + +test "query parser preserves exact graph path endpoint identities" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"path":{"index":"graph_idx","shortest_path":{"from":{"key":"shared"},"to":{"key":"shared","table":"companies"}}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + const graph_query = owned.req.graph_queries[0].query; + switch (graph_query.start_nodes) { + .identities => |identities| { + try std.testing.expectEqual(@as(usize, 1), identities.len); + try std.testing.expectEqualStrings("shared", identities[0].key); + try std.testing.expect(identities[0].table == null); + }, + else => return error.TestUnexpectedResult, + } + switch (graph_query.target_nodes.?) { + .identities => |identities| { + try std.testing.expectEqual(@as(usize, 1), identities.len); + try std.testing.expectEqualStrings("shared", identities[0].key); + try std.testing.expectEqualStrings("companies", identities[0].table.?); + }, + else => return error.TestUnexpectedResult, + } +} + +test "query parser adapts deprecated graph searches" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_searches":{"neighbors":{"type":"neighbors","index_name":"graph_idx","start_nodes":{"keys":["doc:a"]},"params":{"edge_types":["links"],"max_depth":1}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); + try std.testing.expectEqualStrings("neighbors", owned.req.graph_queries[0].name); + try std.testing.expect(owned.req.graph_queries[0].query.query_type == .neighbors); + const transport = owned.req.graph_query_transport orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(db_mod.types.GraphQueryWireDialect.legacy, transport.dialect); + try std.testing.expect(std.mem.startsWith(u8, transport.operations_json, "{\"neighbors\":")); +} + +test "query parser rejects graph queries and graph searches together" { + try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"new":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]}}}},"graph_searches":{"old":{"type":"neighbors","index_name":"graph_idx","start_nodes":{"keys":["doc:a"]}}}} + )); +} + +test "query parser accepts graph pattern searches" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"pattern_walk":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{"filter":{"ids":["doc:a"]}},"b":{"table":"entities"}},"edges":[{"from":"a","to":"b","types":["links"],"max_hops":2}]},"return":{"bindings":["b"],"limit":10}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); + try std.testing.expect(owned.req.graph_queries[0].query.query_type == .pattern); + try std.testing.expectEqual(@as(usize, 2), owned.req.graph_queries[0].query.match_pattern.?.nodes.len); + try std.testing.expectEqualStrings("entities", owned.req.graph_queries[0].query.match_pattern.?.nodes[1].table.?); + try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries[0].query.return_aliases.len); + try std.testing.expectEqual(@as(u32, 10), owned.req.graph_queries[0].query.params.max_results); +} + +test "query parser owns graph match anchor through its required node alias" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"escaped":{"index":"graph_idx","match":{"anchor":"a\u0062","nodes":{"a\u0062":{}},"edges":[]},"return":{"bindings":["a\u0062"]}}}} + ); + defer owned.deinit(std.testing.allocator); + + const pattern = owned.req.graph_queries[0].query.match_pattern.?; + try std.testing.expect(pattern.anchor_alias != null); + const anchor_alias = pattern.anchor_alias.?; + try std.testing.expectEqualStrings("ab", anchor_alias); + try std.testing.expectEqual(@intFromPtr(pattern.nodes[0].alias.ptr), @intFromPtr(anchor_alias.ptr)); +} + +test "query parser treats explicit graph document fields as a projection" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"walk":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]},"include_documents":true,"fields":["title"]}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + const graph_query = owned.req.graph_queries[0].query; + try std.testing.expect(graph_query.include_documents); + try std.testing.expect(!graph_query.include_all_fields); + try std.testing.expectEqual(@as(usize, 1), graph_query.fields.len); + try std.testing.expectEqualStrings("title", graph_query.fields[0]); +} + +test "query parser accepts exact graph count aggregates" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"pattern_count":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{}},"edges":[]},"return":{"aggregates":{"count":{"count":"*"}}}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + const graph_query = owned.req.graph_queries[0].query; + try std.testing.expectEqual(@as(usize, 1), graph_query.aggregates.len); + try std.testing.expectEqualStrings("count", graph_query.aggregates[0].name); + try std.testing.expectEqualStrings("*", graph_query.aggregates[0].of); +} + +test "query parser accepts duplicate graph count expressions under different names" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_queries":{"pattern_count":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{}},"edges":[]},"return":{"aggregates":{"first":{"count":"a","distinct":true},"second":{"count":"a","distinct":true}}}}},"limit":10} + ); + defer owned.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 2), owned.req.graph_queries[0].query.aggregates.len); +} + +test "query parser rejects semantic search offsets" { + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4,"offset":1} + )); +} + +test "query parser records approximate source diagnostic for semantic exact sort" { + db_mod.resetLastSortRejectionDiagnostic(); + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"order_by":[{"field":"created_at","desc":true}],"limit":4} + )); + const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("created_at", diagnostic.field); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); +} + +test "query parser rejects semantic cursor-only pagination as approximate source" { + db_mod.resetLastSortRejectionDiagnostic(); + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"search_after":["doc:a"],"limit":4} + )); + const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("_id", diagnostic.field); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); +} + +test "query parser rejects semantic search_before pagination as approximate source" { + db_mod.resetLastSortRejectionDiagnostic(); + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"search_before":["doc:a"],"limit":4} + )); + const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("_id", diagnostic.field); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); +} + +test "query parser rejects semantic score sort as approximate source" { + const alloc = std.testing.allocator; + db_mod.resetLastSortRejectionDiagnostic(); + try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(alloc, FakeSemanticResolver.iface(), "docs", + \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"order_by":[{"field":"_score","desc":true}],"limit":4} + )); + const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("_score", diagnostic.field); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); + try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); +} + +test "query encoder emits antfly-style response envelope" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 1.25, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{}, result); + defer encoded.deinit(alloc); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"responses\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"table\":\"docs\"") != null); +} + +test "query encoder does not expose internal doc ordinals" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .doc_ordinal = 42, + .native_text_doc_id = 7, + .score = 1.25, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{}, result); + defer encoded.deinit(alloc); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "doc_ordinal") == null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "native_text_doc_id") == null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "ordinal") == null); +} + +test "query encoder emits aggregations" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 1.25, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\",\"price\":10,\"category\":\"books\"}"), + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + const aggregation_results = try alloc.alloc(db_mod.aggregations.SearchAggregationResult, 2); + aggregation_results[0] = .{ + .name = "price_stats", + .field = "price", + .type = "stats", + .value_json = try alloc.dupe(u8, "{\"count\":1,\"sum\":10,\"avg\":10,\"min\":10,\"max\":10,\"sum_squares\":100,\"variance\":0,\"std_dev\":0}"), + }; + const buckets = try alloc.alloc(db_mod.aggregations.SearchAggregationBucket, 1); + buckets[0] = .{ + .key_json = try alloc.dupe(u8, "\"books\""), + .count = 1, + }; + aggregation_results[1] = .{ + .name = "categories", + .field = "category", + .type = "terms", + .buckets = buckets, + }; + + var meta: QueryResponseMeta = .{ + .aggregation_results = aggregation_results, + }; + defer meta.deinit(alloc); + + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .aggregations_json = + \\{"price_stats":{"type":"stats","field":"price"},"categories":{"type":"terms","field":"category","size":5}} + , + }, meta, result); + defer encoded.deinit(alloc); + + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"aggregations\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"price_stats\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"sum\":10") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"categories\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"key\":\"books\"") != null); +} + +test "query encoder supports count-only and profile responses" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 1.25, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{ .count_only = true, .profile = true }, .{ + .took_ms = 7, + .shard_count = 3, + .merged = true, + .dense_search = .{ + .resolved_search_width = 128, + .resolved_epsilon = 0.15, + .hbc_reranked_vectors = 42, + .hbc_search_ns = 123456, + }, + }, result); + defer encoded.deinit(alloc); + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"hits":{"total":{"value":1,"relation":"exact"},"hits":[]}}]} + , encoded.json); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"profile\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"took\":7") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"shards\":{\"total\":3,\"successful\":3,\"failed\":0}") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"merge\":{\"strategy\":\"rrf\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"dense_search\":{\"total_ns\":0") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"resolved_search_width\":128") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"resolved_epsilon\":0.15") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"hbc_reranked_vectors\":42") != null); +} + +test "query encoder projects deferred stored fields without round-tripping bytes" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 1.25, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\",\"id\":\"stored-id\",\"body\":\"hello\"}"), + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .fields = &.{ "id", "title" }, + .include_all_fields = false, + .defer_stored_projection = true, + }, .{}, result); + defer encoded.deinit(alloc); + + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_source\":{\"id\":\"stored-id\",\"title\":\"alpha\"}") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"body\"") == null); +} + +test "query encoder omits _source for key-only hits" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:key-only"), + .score = 0.75, + .stored_data = null, + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .include_all_fields = false, + }, .{}, result); + defer encoded.deinit(alloc); + + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:key-only\"") != null); + try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_source\"") == null); +} + +test "query encoder emits graph results" { + const alloc = std.testing.allocator; + const graph_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + graph_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + }; + const graph_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + graph_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 1, + .stored_data = try alloc.dupe(u8, "{\"title\":\"beta\"}"), + }; + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = graph_nodes, + .paths = &.{}, + .hits = graph_hits, + .total_hits = 1, + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, + }; + defer result.deinit(); + + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .include_documents = true, + }, + }}; + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .graph_queries = &graph_queries, + .graph_query_transport = .{ + .dialect = .canonical, + .operations_json = + \\{"neighbors":{"traverse":{"index":"graph_idx","start":{"keys":["doc:a"]},"include_documents":true}}} + , + .admitted_operations_ptr = @ptrCast(graph_queries[0..].ptr), + .admitted_operations_len = graph_queries.len, + }, + }, .{ .took_ms = 4 }, result); + defer encoded.deinit(alloc); + var parsed = try ant_json.parseFromSlice(metadata_test_openapi.QueryResponses, alloc, encoded.json, .{}); + defer parsed.deinit(); + const responses = parsed.value.responses orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(@as(usize, 1), responses.len); + const decoded_graph_results = responses[0].graph_results orelse return error.TestUnexpectedResult; + const result_value = decoded_graph_results.map.get("neighbors") orelse return error.TestUnexpectedResult; + const neighbors = switch (result_value) { + .graph_nodes_result => |value| value, + else => return error.TestUnexpectedResult, + }; + const nodes = neighbors.nodes; + try std.testing.expectEqual(@as(usize, 1), nodes.len); + try std.testing.expectEqualStrings("doc:b", nodes[0].key); + const document = nodes[0].document orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("beta", document.map.get("title").?.string); +} + +test "query merge applies global score ordering and offset" { + const alloc = std.testing.allocator; + + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 2); + left_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .doc_ordinal = 2, + .score = 2.0, + .stored_data = try alloc.dupe(u8, "{\"title\":\"beta\"}"), + }; + left_hits[1] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .doc_ordinal = 1, + .score = 3.0, + .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + }; + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:c"), + .score = 1.0, + .stored_data = try alloc.dupe(u8, "{\"title\":\"gamma\"}"), + }; + + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 2 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + + var merged = try mergeSearchResults(alloc, .{ .full_text = .{ .match = .{ .field = "body", .text = "alpha" } } }, &.{ left, right }, 1, 1); + defer merged.deinit(); + + try std.testing.expectEqual(@as(u32, 3), merged.total_hits); + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); + try std.testing.expectEqualStrings("doc:b", merged.hits[0].id); + try std.testing.expectEqual(@as(?u32, null), merged.hits[0].doc_ordinal); +} + +test "query merge allocation scales with the selected page" { + const large_stored = "x" ** 1024; + var input_hits: [2048]db_mod.types.SearchHit = undefined; + for (&input_hits) |*hit| { + hit.* = .{ + .id = @constCast("doc:a"), + .stored_data = @constCast(large_stored), + }; + } + const input = db_mod.types.SearchResult{ + .alloc = std.testing.allocator, + .hits = &input_hits, + .total_hits = input_hits.len, + }; + + // Enough for a bounded top-one heap plus one cloned page hit, but not for + // an O(candidate count) pointer array or cloned stored candidates. + var backing: [4096]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&backing); + var merged = try mergeSearchResults(fba.allocator(), .{}, &.{input}, 0, 1); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); + try std.testing.expectEqualStrings("doc:a", merged.hits[0].id); + try std.testing.expectEqual(@as(usize, large_stored.len), merged.hits[0].stored_data.?.len); +} + +test "query merge rejects score ordered hits without finite scores" { + const alloc = std.testing.allocator; + + var missing_score_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + missing_score_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:missing"), + }; + var missing_score = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = missing_score_hits, + .total_hits = 1, + }; + defer missing_score.deinit(); + + const scoring_req = db_mod.types.SearchRequest{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + }; + try std.testing.expectError(error.InvalidQueryRequest, mergeSearchResults(alloc, scoring_req, &.{missing_score}, 0, 10)); + + var non_finite_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + non_finite_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:nan"), + .score = std.math.nan(f32), + }; + var non_finite = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = non_finite_hits, + .total_hits = 1, + }; + defer non_finite.deinit(); + + try std.testing.expectError(error.InvalidQueryRequest, mergeSearchResults(alloc, scoring_req, &.{non_finite}, 0, 10)); +} + +test "query merge orders non score bearing hits by id without requiring scores" { + const alloc = std.testing.allocator; + + var hits = try alloc.alloc(db_mod.types.SearchHit, 2); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 100.0, + }; + hits[1] = .{ + .id = try alloc.dupe(u8, "doc:a"), + }; + + var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 2 }; + defer result.deinit(); + + var merged = try mergeSearchResults(alloc, .{ .full_text = .{ .match_all = {} } }, &.{result}, 0, 10); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, 2), merged.hits.len); + try std.testing.expectEqualStrings("doc:a", merged.hits[0].id); + try std.testing.expectEqualStrings("doc:b", merged.hits[1].id); +} + +fn testSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, rank: i64) !db_mod.types.SearchHit { + const sort_values = try alloc.alloc(std.json.Value, 2); + errdefer alloc.free(sort_values); + sort_values[0] = .{ .integer = rank }; + sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); + return .{ + .id = try alloc.dupe(u8, id), + .doc_ordinal = @intCast(@max(rank, 0)), + .sort_values = sort_values, + }; +} + +fn testIdSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8) !db_mod.types.SearchHit { + const sort_values = try alloc.alloc(std.json.Value, 1); + errdefer alloc.free(sort_values); + sort_values[0] = .{ .string = try alloc.dupe(u8, id) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); + return .{ + .id = try alloc.dupe(u8, id), + .sort_values = sort_values, + }; +} + +fn testScoreSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, score: f32) !db_mod.types.SearchHit { + const sort_values = try alloc.alloc(std.json.Value, 2); + errdefer alloc.free(sort_values); + sort_values[0] = .{ .float = @floatCast(score) }; + sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); + return .{ + .id = try alloc.dupe(u8, id), + .score = score, + .sort_values = sort_values, + }; +} + +fn testHierarchyNavigationHitAlloc( + alloc: std.mem.Allocator, + id: []const u8, + position: []const u8, +) !db_mod.types.SearchHit { + const sort_values = try alloc.alloc(std.json.Value, 2); + errdefer alloc.free(sort_values); + sort_values[0] = .{ .string = try alloc.dupe(u8, position) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); + sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); + return .{ + .id = try alloc.dupe(u8, id), + .sort_values = sort_values, + }; +} + +const TestHierarchyUnitChunk = struct { id: []const u8, score: f32 }; + +fn testHierarchyUnitHitForIdAlloc( + alloc: std.mem.Allocator, + unit_id: []const u8, + score: f32, + chunks: []const TestHierarchyUnitChunk, +) !db_mod.types.SearchHit { + const id = try std.fmt.allocPrint( + alloc, + "doc:a/_artifact/asset/document_units_v1/{s}", + .{unit_id}, + ); + defer alloc.free(id); + const chunk_hits = try alloc.alloc(db_mod.types.ChunkHit, chunks.len); + var initialized: usize = 0; + errdefer { + for (chunk_hits[0..initialized]) |*chunk| chunk.deinit(alloc); + alloc.free(chunk_hits); + } + for (chunks, 0..) |chunk, i| { + chunk_hits[i] = .{ + .id = try alloc.dupe(u8, chunk.id), + .score = chunk.score, + }; + initialized += 1; + } + return .{ + .id = try alloc.dupe(u8, id), + .score = score, + .stored_data = try alloc.dupe(u8, "{\"_hierarchy_unit_revision_token\":\"revision-a\"}"), + .artifact_ref = .{ + .document_id = try alloc.dupe(u8, "doc:a"), + .name = try alloc.dupe(u8, "document_units_v1"), + .kind = .asset, + .unit_id = try alloc.dupe(u8, unit_id), + }, + .chunk_hits = chunk_hits, + }; +} + +fn testHierarchyUnitHitAlloc( + alloc: std.mem.Allocator, + score: f32, + chunks: []const TestHierarchyUnitChunk, +) !db_mod.types.SearchHit { + return testHierarchyUnitHitForIdAlloc(alloc, "unit:0", score, chunks); +} + +fn testHierarchyUnitShardResultAlloc( + alloc: std.mem.Allocator, + shard_index: usize, + hit_count: usize, +) !db_mod.types.SearchResult { + const hits = try alloc.alloc(db_mod.types.SearchHit, hit_count); + var initialized: usize = 0; + errdefer { + for (hits[0..initialized]) |*hit| hit.deinit(alloc); + alloc.free(hits); + } + for (hits, 0..) |*hit, hit_index| { + const unit_id = try std.fmt.allocPrint( + alloc, + "unit:{d:0>2}:{d:0>3}", + .{ shard_index, hit_index }, + ); + defer alloc.free(unit_id); + const score: f32 = @floatFromInt(hit_count - hit_index); + hit.* = try testHierarchyUnitHitForIdAlloc(alloc, unit_id, score, &.{}); + initialized += 1; + } + return .{ + .alloc = alloc, + .hits = hits, + .total_hits = @intCast(hit_count), + }; +} + +test "query merge treats hierarchy navigation positions as opaque cursor values" { + const alloc = std.testing.allocator; + const order_by = [_]db_mod.types.SortField{ + .{ .field = "_hierarchy.position" }, + .{ .field = "_id" }, + }; + const cursor = [_]std.json.Value{ + .{ .string = "document_units_v1/00000000000000000007/00000000000000000000" }, + .{ .string = "artifact:page:1" }, + }; + + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyNavigationHitAlloc( + alloc, + "artifact:page:1", + "document_units_v1/00000000000000000007/00000000000000000000", + ); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyNavigationHitAlloc( + alloc, + "artifact:page:2", + "document_units_v1/00000000000000000007/00000000000000000001", + ); + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 2 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 2 }; + defer right.deinit(); + + var merged = try mergeSearchResultsWithRuntimeSchema(alloc, .{ + .hierarchy_children = .{ .parent_id = "doc:a" }, + .order_by = &order_by, + .search_after = &cursor, + .limit = 20, + }, &.{ left, right }, 0, 20, .{}); + defer merged.deinit(); + + // Duplicate parent plans use a logical maximum rather than inflating the + // unit count, and the coordinator applies the opaque tuple cursor. + try std.testing.expectEqual(@as(u32, 2), merged.total_hits); + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); + try std.testing.expectEqualStrings("artifact:page:2", merged.hits[0].id); +} + +test "query merge treats conflicting hierarchy navigation plans as retryable" { + const alloc = std.testing.allocator; + const order_by = [_]db_mod.types.SortField{ + .{ .field = "_hierarchy.position" }, + .{ .field = "_id" }, + }; + + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-a/page/1"); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-b/page/1"); + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + + try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResultsWithRuntimeSchema( + alloc, + .{ + .hierarchy_children = .{ .parent_id = "doc:a" }, + .order_by = &order_by, + .limit = 20, + }, + &.{ left, right }, + 0, + 20, + .{}, + )); +} + +test "query merge treats malformed hierarchy navigation shard tuples as retryable" { + const alloc = std.testing.allocator; + const order_by = [_]db_mod.types.SortField{ + .{ .field = "_hierarchy.position" }, + .{ .field = "_id" }, + }; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-a/page/1"); + alloc.free(hits[0].sort_values[1].string); + hits[0].sort_values[1] = .{ .string = try alloc.dupe(u8, "artifact:wrong-tiebreaker") }; + var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 1 }; + defer result.deinit(); + + try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResultsWithRuntimeSchema( + alloc, + .{ + .hierarchy_children = .{ .parent_id = "doc:a" }, + .order_by = &order_by, + .limit = 20, + }, + &.{result}, + 0, + 20, + .{}, + )); +} + +test "query merge releases hierarchy navigation candidates once at the global budget" { + const alloc = std.testing.allocator; + const order_by = [_]db_mod.types.SortField{ + .{ .field = "_hierarchy.position" }, + .{ .field = "_id" }, + }; + const hit_count = db_mod.types.max_canonical_hierarchy_total_matches + 1; + const hits = try alloc.alloc(db_mod.types.SearchHit, hit_count); + var initialized: usize = 0; + errdefer { + for (hits[0..initialized]) |*hit| hit.deinit(alloc); + alloc.free(hits); + } + for (hits, 0..) |*hit, i| { + const id = try std.fmt.allocPrint(alloc, "unit:{d}", .{i}); + defer alloc.free(id); + const position = try std.fmt.allocPrint(alloc, "position/{d:0>8}", .{i}); + defer alloc.free(position); + hit.* = try testHierarchyNavigationHitAlloc(alloc, id, position); + initialized += 1; + } + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = hit_count, + }; + defer result.deinit(); + + try std.testing.expectError(error.QueryCandidateBudgetExceeded, mergeSearchResultsWithRuntimeSchema( + alloc, + .{ + .hierarchy_children = .{ .parent_id = "doc:a" }, + .order_by = &order_by, + .limit = 20, + }, + &.{result}, + 0, + 20, + .{}, + )); +} + +test "query merge globally coalesces hierarchy unit groups and bounded chunks" { + const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{ + .{ .id = "chunk:a", .score = 0.8 }, + .{ .id = "chunk:shared", .score = 0.4 }, + }); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{ + .{ .id = "chunk:b", .score = 0.9 }, + .{ .id = "chunk:shared", .score = 0.5 }, + }); + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + + var merged = try mergeSearchResults(alloc, .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit_with_chunks, + .hierarchy_group_level = .unit, + .hierarchy_grouped_matches = true, + .max_chunks_per_parent = 2, + .limit = 10, + }, &.{ left, right }, 0, 10); + defer merged.deinit(); + + try std.testing.expectEqual(@as(u32, 1), merged.total_hits); + try std.testing.expectEqual(db_mod.types.TotalHitsRelation.exact, merged.total_hits_relation); + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); + try std.testing.expectEqual(@as(?f32, 0.9), merged.hits[0].score); + try std.testing.expectEqual(@as(usize, 2), merged.hits[0].chunk_hits.len); + try std.testing.expectEqualStrings("chunk:b", merged.hits[0].chunk_hits[0].id); + try std.testing.expectEqualStrings("chunk:a", merged.hits[0].chunk_hits[1].id); +} + +test "query merge composes hierarchy unit groups with canonical graph results" { + const alloc = std.testing.allocator; + const cloneOneGraphResult = struct { + fn run( + allocator: std.mem.Allocator, + source: db_mod.types.GraphSearchResult, + ) ![]db_mod.types.GraphSearchResult { + const out = try allocator.alloc(db_mod.types.GraphSearchResult, 1); + errdefer allocator.free(out); + out[0] = try cloneGraphSearchResult(allocator, source); + return out; + } + }.run; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); + + var left_node = [_]graph_query_mod.GraphResultNode{.{ + .key = "left", + .depth = 1, + .distance = 1, + }}; + var right_node = [_]graph_query_mod.GraphResultNode{.{ + .key = "right", + .depth = 1, + .distance = 1, + }}; + const left_graph = db_mod.types.GraphSearchResult{ + .name = @constCast("walk"), + .nodes = &left_node, + .hits = &.{}, + .total_hits = 1, + }; + const right_graph = db_mod.types.GraphSearchResult{ + .name = @constCast("walk"), + .nodes = &right_node, + .hits = &.{}, + .total_hits = 1, + }; + + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + defer left.deinit(); + left.graph_results = try cloneOneGraphResult(alloc, left_graph); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + right.graph_results = try cloneOneGraphResult(alloc, right_graph); + + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "walk", + .query = .{ + .query_type = .neighbors, + .index_name = "graph", + .start_nodes = .{ .keys = &.{} }, + }, + }}; + var merged = try mergeSearchResults(alloc, .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .graph_queries = &graph_queries, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .limit = 10, + }, &.{ left, right }, 0, 10); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); + try std.testing.expectEqualStrings("walk", merged.graph_results[0].name); + try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("left", merged.graph_results[0].nodes[0].key); + try std.testing.expectEqualStrings("right", merged.graph_results[0].nodes[1].key); +} + +test "query merge treats conflicting hierarchy unit identities as retryable" { + const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); + const right_ref = &right_hits[0].artifact_ref.?; + alloc.free(right_ref.name); + right_ref.name = try alloc.dupe(u8, "document_units_v2"); + + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + + try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResults( + alloc, + .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .limit = 10, + }, + &.{ left, right }, + 0, + 10, + )); +} + +test "query merge treats malformed hierarchy unit shard ranking as retryable" { + const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyUnitHitForIdAlloc(alloc, "unit:0", 0.8, &.{}); + left_hits[0].score = null; + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyUnitHitForIdAlloc(alloc, "unit:1", 0.7, &.{}); + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + defer right.deinit(); + + try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResults( + alloc, + .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .limit = 10, + }, + &.{ left, right }, + 0, + 10, + )); +} + +test "query merge reports an honest lower bound for a partial hierarchy unit union" { + const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); + var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 10 }; + defer left.deinit(); + var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 12 }; + defer right.deinit(); + + var merged = try mergeSearchResults(alloc, .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .limit = 10, + }, &.{ left, right }, 0, 10); + defer merged.deinit(); - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .bool_query); - const root = owned.req.full_text.?.bool_query; - try std.testing.expectEqual(@as(usize, 2), root.must.len); - try std.testing.expect(root.must[0] == .match); - try std.testing.expectApproxEqAbs(@as(f32, 2.0), root.must[0].match.boost, 0.0001); - try std.testing.expect(root.must[1] == .match_phrase); - try std.testing.expectApproxEqAbs(@as(f32, 4.0), root.must[1].match_phrase.boost, 0.0001); + try std.testing.expectEqual(@as(u32, 12), merged.total_hits); + try std.testing.expectEqual(db_mod.types.TotalHitsRelation.gte, merged.total_hits_relation); + try std.testing.expectEqual(@as(usize, 1), merged.hits.len); +} + +test "query merge rejects exact sorting for hierarchy unit groups" { + const alloc = std.testing.allocator; + const order_by = [_]db_mod.types.SortField{ + .{ .field = "_score", .desc = true }, + .{ .field = "_id" }, + }; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); + var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 1 }; + defer result.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .order_by = &order_by, + .limit = 10, + }, &.{result}, 0, 10)); +} + +test "query merge bounds hierarchy unit selection by page instead of shard fanout" { + const alloc = std.testing.allocator; + const shard_count = 11; + const hits_per_shard = 100; + const results = try alloc.alloc(db_mod.types.SearchResult, shard_count); + var initialized: usize = 0; + defer { + for (results[0..initialized]) |*result| result.deinit(); + alloc.free(results); + } + for (results, 0..) |*result, shard_index| { + result.* = try testHierarchyUnitShardResultAlloc(alloc, shard_index, hits_per_shard); + initialized += 1; + } + + var merged = try mergeSearchResults(alloc, .{ + .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, + .return_mode = .unit, + .hierarchy_group_level = .unit, + .limit = hits_per_shard, + }, results, 0, hits_per_shard); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, hits_per_shard), merged.hits.len); + try std.testing.expectEqual(@as(u32, hits_per_shard), merged.total_hits); + try std.testing.expectEqual(db_mod.types.TotalHitsRelation.gte, merged.total_hits_relation); + for (merged.hits, 0..) |hit, i| { + if (i > 0) try std.testing.expect(merged.hits[i - 1].score.? >= hit.score.?); + for (merged.hits[0..i]) |previous| { + try std.testing.expect(!std.mem.eql(u8, previous.id, hit.id)); + } + } +} + +fn testDateSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, created_at_ns: u64) !db_mod.types.SearchHit { + const sort_values = try alloc.alloc(std.json.Value, 2); + errdefer alloc.free(sort_values); + sort_values[0] = .{ .string = try runtime_schema_mod.formatDateTimeNsAlloc(alloc, created_at_ns) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); + sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; + errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); + return .{ + .id = try alloc.dupe(u8, id), + .sort_values = sort_values, + }; +} + +fn testRankRuntimeSchema() runtime_schema_mod.TableSchema { + const templates = struct { + const values = [_]runtime_schema_mod.DynamicTemplate{.{ + .name = "rank", + .path_match = "rank", + .mapping = .{ + .field_type = .numeric, + .doc_values = true, + .sortable = true, + .analyzer = "keyword", + }, + }}; + }.values; + return .{ .dynamic_templates = &templates }; +} + +test "query merge applies deterministic graph metric top-k across shards" { + const alloc = std.testing.allocator; + + const left_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 2); + left_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.8 }; + left_scores[1] = .{ .node = try alloc.dupe(u8, "doc:d"), .score = 0.6 }; + const left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + left_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = left_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .building_generation = 6, + .build_job_id = 12345, + .build_started_at_ms = 1780000000100, + .build_iteration = 2, + .progress = 1.0, + .converged = true, + }, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = left_metrics, + }; + defer left.deinit(); + + const right_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 3); + right_scores[0] = .{ .node = try alloc.dupe(u8, "doc:c"), .score = 0.9 }; + right_scores[1] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + right_scores[2] = .{ .node = try alloc.dupe(u8, "doc:e"), .score = 0.1 }; + const right_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + right_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = right_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .build_queued = true, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .queued_generation = 6, + .building_generation = 6, + .build_job_id = 12345, + .build_started_at_ms = 1780000000100, + .build_iteration = 3, + .progress = 0.0, + .converged = true, + }, + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = right_metrics, + }; + defer right.deinit(); + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + var merged = try mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, right }, 0, 10); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, 1), merged.graph_metric_results.len); + const central = merged.graph_metric_results[0]; + try std.testing.expectEqualStrings("central", central.name); + try std.testing.expectEqual(@as(usize, 3), central.scores.len); + try std.testing.expectEqualStrings("doc:c", central.scores[0].node); + try std.testing.expectEqualStrings("doc:a", central.scores[1].node); + try std.testing.expectEqualStrings("doc:b", central.scores[2].node); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, central.status.state); + try std.testing.expect(central.status.build_queued); + try std.testing.expectEqual(@as(u64, 5), central.status.published_generation); + try std.testing.expectEqual(@as(u64, 6), central.status.edge_generation); + try std.testing.expectEqual(@as(u64, 6), central.status.queued_generation); + try std.testing.expectEqual(@as(u64, 6), central.status.building_generation); + try std.testing.expectEqual(@as(u64, 12345), central.status.build_job_id); + try std.testing.expectEqual(@as(u64, 1780000000100), central.status.build_started_at_ms); + try std.testing.expectEqual(@as(u32, 3), central.status.build_iteration); + + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .profile = true, + .graph_metric_queries = &graph_metric_queries, + }, .{ .took_ms = 4, .shard_count = 2, .merged = true }, merged); + defer encoded.deinit(alloc); + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"profile":{"shards":{"total":2,"successful":2,"failed":0},"graph_metrics":[{"query_name":"central","source":"graph_metric","index_name":"graph_idx","metric_name":"pagerank","freshness":"published","status":{"state":"stale","published_generation":5,"building_generation":6}}]}}]} + , encoded.json); + + const fresh_graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + .freshness = .fresh, + }, + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &fresh_graph_metric_queries }, &.{ left, right }, 0, 10)); + + right_metrics[0].status.published_generation = 4; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, right }, 0, 10)); +} + +test "query merge rejects missing or unpublished graph metric shard results" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const left_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + left_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + const left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + left_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = left_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = left_metrics, + }; + defer left.deinit(); + + var missing = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + }; + defer missing.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, missing }, 0, 10)); + + const unpublished_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + unpublished_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 0, + .edge_generation = 5, + .target_edge_generation = 5, + }, + }; + var unpublished = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = unpublished_metrics, + }; + defer unpublished.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, unpublished }, 0, 10)); +} + +test "query merge rejects duplicate direct graph metric score nodes" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const duplicate_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 2); + duplicate_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + duplicate_scores[1] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.7 }; + const duplicate_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + duplicate_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = duplicate_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var duplicate = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = duplicate_metrics, + }; + defer duplicate.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{duplicate}, 0, 10)); + + const left_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + left_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.8 }; + const left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + left_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = left_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = left_metrics, + }; + defer left.deinit(); + + const right_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + right_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.7 }; + const right_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + right_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = right_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = right_metrics, + }; + defer right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, right }, 0, 10)); +} + +test "query merge rejects non-finite direct graph metric scores" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const nan_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + nan_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = std.math.nan(f64) }; + const nan_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + nan_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = nan_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var nan_result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = nan_metrics, + }; + defer nan_result.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{nan_result}, 0, 10)); + + const inf_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + inf_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = std.math.inf(f64) }; + const inf_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + inf_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = inf_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var inf_result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = inf_metrics, + }; + defer inf_result.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{inf_result}, 0, 10)); +} + +test "query merge rejects duplicate direct graph metric shard results" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const scores_a = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + scores_a[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + const scores_b = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + scores_b[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.7 }; + const duplicate_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 2); + duplicate_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = scores_a, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + duplicate_metrics[1] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = scores_b, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var duplicate = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = duplicate_metrics, + }; + defer duplicate.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{duplicate}, 0, 10)); + + const duplicate_request_queries = [_]db_mod.types.NamedGraphMetricQuery{ + .{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }, + .{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 3, + }, + }, + }; + var valid_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + valid_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + const valid_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + valid_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = valid_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var duplicate_request = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = valid_metrics, + }; + defer duplicate_request.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &duplicate_request_queries }, &.{duplicate_request}, 0, 10)); } -test "query parser accepts bleve query string field groups" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"title:(alpha beta)"}} - ); - defer owned.deinit(std.testing.allocator); +test "query merge rejects mismatched direct graph metric shard identity" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .bool_query); - const root = owned.req.full_text.?.bool_query; - try std.testing.expectEqual(@as(usize, 2), root.must.len); - try std.testing.expect(root.must[0] == .match); - try std.testing.expect(root.must[1] == .match); - try std.testing.expectEqualStrings("title", root.must[0].match.field); - try std.testing.expectEqualStrings("alpha", root.must[0].match.text); - try std.testing.expectEqualStrings("title", root.must[1].match.field); - try std.testing.expectEqualStrings("beta", root.must[1].match.text); + const wrong_index_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + wrong_index_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "other_graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var wrong_index = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = wrong_index_metrics, + }; + defer wrong_index.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{wrong_index}, 0, 10)); + + const wrong_metric_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + wrong_metric_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "degree"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "degree"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var wrong_metric = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = wrong_metric_metrics, + }; + defer wrong_metric.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{wrong_metric}, 0, 10)); + + const wrong_status_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + wrong_status_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "degree"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var wrong_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = wrong_status_metrics, + }; + defer wrong_status.deinit(); + + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{wrong_status}, 0, 10)); } -test "query parser accepts bleve query string inline ranges" { - var numeric = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"score:[10 TO 20}"}} - ); - defer numeric.deinit(std.testing.allocator); +test "query merge rejects inconsistent graph metric fan-in status state" { + const alloc = std.testing.allocator; - try std.testing.expect(numeric.req.full_text != null); - try std.testing.expect(numeric.req.full_text.? == .numeric_range); - try std.testing.expectEqual(@as(f64, 10), numeric.req.full_text.?.numeric_range.min.?); - try std.testing.expectEqual(@as(f64, 20), numeric.req.full_text.?.numeric_range.max.?); - try std.testing.expect(numeric.req.full_text.?.numeric_range.inclusive_min); - try std.testing.expect(!numeric.req.full_text.?.numeric_range.inclusive_max); + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + const direct_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + direct_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + }, + }; + var direct = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = direct_metrics, + }; + defer direct.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{direct}, 0, 10)); + + const direct_future_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + direct_future_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }; + var direct_future = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = direct_future_metrics, + }; + defer direct_future.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{direct_future}, 0, 10)); - var date = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"created:[2024-01-01T00:00:00Z TO 2024-12-31T00:00:00Z]"}} - ); - defer date.deinit(std.testing.allocator); + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + .include_metric_status = true, + }, + }}; + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .disabled, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + }, + }), + }; + var graph = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, + }; + defer graph.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{graph}, 0, 10)); - try std.testing.expect(date.req.full_text != null); - try std.testing.expect(date.req.full_text.? == .date_range); - try std.testing.expect(date.req.full_text.?.date_range.start_ns != null); - try std.testing.expect(date.req.full_text.?.date_range.end_ns != null); + const graph_future_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_future_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }), + }; + var graph_future = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_future_results, + }; + defer graph_future.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{graph_future}, 0, 10)); - var term = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"title:[alpha TO omega]"}} - ); - defer term.deinit(std.testing.allocator); + var rerank = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + }, + }; + defer rerank.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, &.{rerank}, 0, 10)); - try std.testing.expect(term.req.full_text != null); - try std.testing.expect(term.req.full_text.? == .term_range); - try std.testing.expectEqualStrings("alpha", term.req.full_text.?.term_range.min.?); - try std.testing.expectEqualStrings("omega", term.req.full_text.?.term_range.max.?); + var rerank_future = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }; + defer rerank_future.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, &.{rerank_future}, 0, 10)); } -test "query parser accepts bleve query string filters" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"query":"alpha"},"filter_query":{"query":"status:published OR status:review"}} - ); - defer owned.deinit(std.testing.allocator); +test "query merge rejects non-finite graph metric fan-in status numbers" { + const alloc = std.testing.allocator; - try std.testing.expect(owned.req.full_text != null); - try std.testing.expect(owned.req.full_text.? == .match); - try ant_json.testing.expectEqualJsonText(std.testing.allocator, - \\{"bool":{"should":[{"match":{"path":"status","text":"published"}},{"match":{"path":"status","text":"review"}}],"minimum_should_match":1}} - , owned.req.filter_query_json); + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + const direct_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + direct_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = std.math.nan(f64), + .converged = true, + }, + }; + var direct = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = direct_metrics, + }; + defer direct.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{direct}, 0, 10)); + + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .include_metric_status = true, + }, + }}; + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .delta = std.math.inf(f64), + .converged = true, + }, + }), + }; + var graph = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, + }; + defer graph.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{graph}, 0, 10)); + + var rerank = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = std.math.inf(f64), + .converged = true, + }, + }; + defer rerank.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, &.{rerank}, 0, 10)); } -test "query parser rejects invalid bleve date ranges" { - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", - \\{"full_text_search":{"field":"created_at","start":"not-a-date"}} - )); +test "query merge rejects out-of-range graph metric fan-in progress" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const high_progress_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + high_progress_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .building, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.25, + .converged = false, + }, + }; + var high_progress = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = high_progress_metrics, + }; + defer high_progress.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{high_progress}, 0, 10)); + + var negative_progress = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = -0.1, + .converged = true, + }, + }; + defer negative_progress.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, &.{negative_progress}, 0, 10)); } -test "query parser resolves semantic search into dense query" { - var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4} - ); - defer owned.deinit(std.testing.allocator); +test "query merge rejects incompatible graph metric fan-in metadata" { + const alloc = std.testing.allocator; + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + + const left_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + left_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + const left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + left_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = left_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .metadata_version = 1, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = left_metrics, + }; + defer left.deinit(); + + const right_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + right_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.7 }; + const right_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + right_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = right_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .metadata_version = 2, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = right_metrics, + }; + defer right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, right }, 0, 10)); + + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + }, + }}; + const graph_left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var graph_left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_left_results, + }; + defer graph_left.deinit(); - try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); - try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); - try std.testing.expectEqual(@as(u32, 4), owned.req.dense_queries[0].query.k); - try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); -} + const filter_types = try alloc.alloc([]const u8, 1); + filter_types[0] = try alloc.dupe(u8, "cites"); + const graph_right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .edge_filter = .{ .mode = .types, .types = filter_types }, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var graph_right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_right_results, + }; + defer graph_right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{ graph_left, graph_right }, 0, 10)); -test "query parser preserves search effort for semantic search" { - var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4,"search_effort":0.3} - ); - defer owned.deinit(std.testing.allocator); + const hits_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + }, + }, + }; + const hits_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 2); + hits_metrics[0] = .{ + .name = try alloc.dupe(u8, "authority"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_authority"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "hits_authority"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + hits_metrics[1] = .{ + .name = try alloc.dupe(u8, "hub"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_hub"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "hits_hub"), + .state = .fresh, + .published_generation = 6, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }; + var hits_pair_mismatch = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = hits_metrics, + }; + defer hits_pair_mismatch.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &hits_metric_queries }, &.{hits_pair_mismatch}, 0, 10)); - try std.testing.expect(owned.req.search_effort != null); - try std.testing.expectApproxEqAbs(@as(f32, 0.3), owned.req.search_effort.?, 0.0001); + var rerank_left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .metadata_version = 1, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + defer rerank_left.deinit(); + var rerank_right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .metadata_version = 2, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + defer rerank_right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, &.{ rerank_left, rerank_right }, 0, 10)); } -test "query parser accepts semantic embedding template" { - var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","embedding_template":"{{remotePDF url=this}}","indexes":["semantic_idx"],"limit":4} - ); - defer owned.deinit(std.testing.allocator); - - try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); - try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); -} +test "query merge rejects unsolicited graph score surfaces" { + const alloc = std.testing.allocator; -test "query parser accepts precomputed embedding payload" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"semantic_idx":[0.5,1.5,2.5]},"limit":6} - ); - defer owned.deinit(std.testing.allocator); + const direct_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + direct_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var unsolicited_direct = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = direct_metrics, + }; + defer unsolicited_direct.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{}, &.{unsolicited_direct}, 0, 10)); - try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); - try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); - try std.testing.expectEqual(@as(u32, 6), owned.req.dense_queries[0].query.k); - try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); - try std.testing.expectEqual(@as(f32, 1.5), owned.req.dense_queries[0].query.vector[1]); -} + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 3, + }, + }}; + const extra_direct_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 2); + extra_direct_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + extra_direct_metrics[1] = .{ + .name = try alloc.dupe(u8, "extra"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }; + var extra_direct = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = extra_direct_metrics, + }; + defer extra_direct.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{extra_direct}, 0, 10)); -test "query parser accepts packed dense embedding payload" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"semantic_idx":"AAAAPwAAwD8AACBA"},"indexes":["semantic_idx"],"fields":["title"],"search_effort":0.3,"limit":6} - ); - defer owned.deinit(std.testing.allocator); + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = &.{}, + }; + var unsolicited_graph = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, + }; + defer unsolicited_graph.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{}, &.{unsolicited_graph}, 0, 10)); - try std.testing.expectEqual(@as(usize, 1), owned.req.dense_queries.len); - try std.testing.expectEqualStrings("semantic_idx", owned.req.dense_queries[0].index_name); - try std.testing.expectEqual(@as(u32, 6), owned.req.dense_queries[0].query.k); - try std.testing.expectEqual(@as(usize, 3), owned.req.dense_queries[0].query.vector.len); - try std.testing.expectApproxEqAbs(@as(f32, 0.5), owned.req.dense_queries[0].query.vector[0], 0.0001); - try std.testing.expectApproxEqAbs(@as(f32, 1.5), owned.req.dense_queries[0].query.vector[1], 0.0001); - try std.testing.expectApproxEqAbs(@as(f32, 2.5), owned.req.dense_queries[0].query.vector[2], 0.0001); - try std.testing.expectEqual(@as(?f32, 0.3), owned.req.search_effort); - try std.testing.expect(owned.req.defer_stored_projection); + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + }, + }}; + const extra_graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 2); + extra_graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = &.{}, + }; + extra_graph_results[1] = .{ + .name = try alloc.dupe(u8, "extra_neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = &.{}, + }; + var extra_graph = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = extra_graph_results, + }; + defer extra_graph.deinit(); + try std.testing.expectError(error.InvalidRemoteResponse, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{extra_graph}, 0, 10)); } -test "query parser rejects packed dense indexes that reference a missing embedding" { - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"semantic_idx":"AAAAPwAAwD8AACBA"},"indexes":["missing_idx"],"limit":6} - )); -} +test "query merge rejects unsolicited graph search metric status" { + const alloc = std.testing.allocator; -test "query parser rejects invalid packed dense embedding payload" { - try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"semantic_idx":"not-base64"},"indexes":["semantic_idx"],"limit":6} - )); -} + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + }, + }}; -test "query parser accepts sparse embedding payload" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"sparse_idx":{"indices":[1,7],"values":[0.4,0.9]}},"limit":6} - ); - defer owned.deinit(std.testing.allocator); + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var unsolicited_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, + }; + defer unsolicited_status.deinit(); - try std.testing.expectEqual(@as(usize, 1), owned.req.sparse_queries.len); - try std.testing.expectEqualStrings("sparse_idx", owned.req.sparse_queries[0].index_name); - try std.testing.expectEqual(@as(u32, 6), owned.req.sparse_queries[0].query.k); - try std.testing.expectEqual(@as(usize, 2), owned.req.sparse_queries[0].query.indices.len); - try std.testing.expectEqual(@as(u32, 7), owned.req.sparse_queries[0].query.indices[1]); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{unsolicited_status}, 0, 10)); + + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const metric_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + }, + }}; + const extra_status_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + extra_status_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + .{ + .name = try alloc.dupe(u8, "degree"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var extra_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = extra_status_results, + }; + defer extra_status.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &metric_graph_queries }, &.{extra_status}, 0, 10)); } -test "query parser accepts merge config reranker and pruner" { - var owned = try parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"full_text_search":{"match":{"field":"body","text":"alpha concept"}},"merge_config":{"strategy":"rsf","window_size":25,"rank_constant":42.0,"weights":{"full_text":0.5,"semantic_idx":1.5}},"reranker":{"provider":"antfly","model":"cross-encoder/ms-marco-MiniLM-L-6-v2","field":"body","top_n":3},"pruner":{"min_score_ratio":0.5,"require_multi_index":true},"limit":6} - ); - defer owned.deinit(std.testing.allocator); +test "query merge validates included graph search metric status list" { + const alloc = std.testing.allocator; - try std.testing.expect(owned.req.merge_config != null); - try std.testing.expectEqual(.rsf, owned.req.merge_config.?.strategy); - try std.testing.expectEqual(@as(u32, 25), owned.req.merge_config.?.window_size); - try std.testing.expectEqual(@as(usize, 2), owned.req.merge_config.?.weights.len); - try std.testing.expect(owned.req.reranker != null); - try std.testing.expectEqual(.antfly, owned.req.reranker.?.provider); - try std.testing.expectEqualStrings("body", owned.req.reranker.?.field); - try std.testing.expectEqual(@as(?u32, 3), owned.req.reranker.?.top_n); - try std.testing.expectEqualStrings("alpha concept", owned.req.reranker_query_text); - try std.testing.expect(owned.req.include_stored); - try std.testing.expect(owned.req.pruner != null); - try std.testing.expectEqual(@as(f64, 0.5), owned.req.pruner.?.min_score_ratio); - try std.testing.expect(owned.req.pruner.?.require_multi_index); -} + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .include_metric_status = true, + }, + }}; -test "query parser rejects dense reranking without query text" { - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", - \\{"embeddings":{"dense_idx":[1.0,0.0,0.0]},"indexes":["dense_idx"],"reranker":{"provider":"antfly","model":"cross-encoder/ms-marco-MiniLM-L-6-v2","field":"body","top_n":2},"limit":6} - )); -} + const duplicate_status_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_status_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_status_results, + }; + defer duplicate_status.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{duplicate_status}, 0, 10)); -test "query parser accepts graph queries" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"neighbors":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]},"edge_types":["links"],"max_depth":1}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); + const invalid_status_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + invalid_status_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + }, + }), + }; + var invalid_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = invalid_status_results, + }; + defer invalid_status.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{invalid_status}, 0, 10)); - try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); - try std.testing.expectEqualStrings("neighbors", owned.req.graph_queries[0].name); - try std.testing.expectEqualStrings("graph_idx", owned.req.graph_queries[0].query.index_name); - try std.testing.expect(owned.req.graph_queries[0].query.query_type == .traverse); - switch (owned.req.graph_queries[0].query.start_nodes) { - .identities => |identities| { - try std.testing.expectEqualStrings("doc:a", identities[0].key); - try std.testing.expect(identities[0].table == null); - }, - else => return error.TestUnexpectedResult, - } + const hits_pair_status_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + hits_pair_status_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "hits_authority"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + .{ + .name = try alloc.dupe(u8, "hits_hub"), + .state = .fresh, + .published_generation = 6, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }), + }; + var hits_pair_status_mismatch = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = hits_pair_status_results, + }; + defer hits_pair_status_mismatch.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{hits_pair_status_mismatch}, 0, 10)); } -test "query parser preserves exact graph path endpoint identities" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"path":{"index":"graph_idx","shortest_path":{"from":{"key":"shared"},"to":{"key":"shared","table":"companies"}}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); +test "query merge rejects malformed graph search metric payloads" { + const alloc = std.testing.allocator; - const graph_query = owned.req.graph_queries[0].query; - switch (graph_query.start_nodes) { - .identities => |identities| { - try std.testing.expectEqual(@as(usize, 1), identities.len); - try std.testing.expectEqualStrings("shared", identities[0].key); - try std.testing.expect(identities[0].table == null); - }, - else => return error.TestUnexpectedResult, - } - switch (graph_query.target_nodes.?) { - .identities => |identities| { - try std.testing.expectEqual(@as(usize, 1), identities.len); - try std.testing.expectEqualStrings("shared", identities[0].key); - try std.testing.expectEqualStrings("companies", identities[0].table.?); + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, }, - else => return error.TestUnexpectedResult, - } -} + }}; -test "query parser adapts deprecated graph searches" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_searches":{"neighbors":{"type":"neighbors","index_name":"graph_idx","start_nodes":{"keys":["doc:a"]},"params":{"edge_types":["links"],"max_depth":1}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); + const missing_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + missing_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + const missing_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + missing_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = missing_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var missing_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = missing_results, + }; + defer missing_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{missing_payload}, 0, 10)); - try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); - try std.testing.expectEqualStrings("neighbors", owned.req.graph_queries[0].name); - try std.testing.expect(owned.req.graph_queries[0].query.query_type == .neighbors); - const transport = owned.req.graph_query_transport orelse return error.TestUnexpectedResult; - try std.testing.expectEqual(db_mod.types.GraphQueryWireDialect.legacy, transport.dialect); - try std.testing.expect(std.mem.startsWith(u8, transport.operations_json, "{\"neighbors\":")); -} + const duplicate_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + duplicate_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.8 }, + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.7 }, + }), + }; + const duplicate_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = duplicate_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_results, + }; + defer duplicate_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{duplicate_payload}, 0, 10)); -test "query parser rejects graph queries and graph searches together" { - try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"new":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]}}}},"graph_searches":{"old":{"type":"neighbors","index_name":"graph_idx","start_nodes":{"keys":["doc:a"]}}}} - )); -} + const non_finite_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + non_finite_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = std.math.nan(f64) }, + }), + }; + const non_finite_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + non_finite_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = non_finite_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var non_finite_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = non_finite_results, + }; + defer non_finite_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{non_finite_payload}, 0, 10)); -test "query parser accepts graph pattern searches" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"pattern_walk":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{"filter":{"ids":["doc:a"]}},"b":{"table":"entities"}},"edges":[{"from":"a","to":"b","types":["links"],"max_hops":2}]},"return":{"bindings":["b"],"limit":10}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); + const duplicate_projected_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ + .name = "pagerank", + .freshness = .published, + }, + .{ + .name = "pagerank", + .freshness = .published, + }, + }; + const duplicate_projected_metric_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &duplicate_projected_metric_reads, + }, + }}; + const duplicate_projected_metric_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + duplicate_projected_metric_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.8 }, + }), + }; + const duplicate_projected_metric_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_projected_metric_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = duplicate_projected_metric_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_projected_metric_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_projected_metric_results, + }; + defer duplicate_projected_metric_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &duplicate_projected_metric_queries }, &.{duplicate_projected_metric_payload}, 0, 10)); - try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries.len); - try std.testing.expect(owned.req.graph_queries[0].query.query_type == .pattern); - try std.testing.expectEqual(@as(usize, 2), owned.req.graph_queries[0].query.match_pattern.?.nodes.len); - try std.testing.expectEqualStrings("entities", owned.req.graph_queries[0].query.match_pattern.?.nodes[1].table.?); - try std.testing.expectEqual(@as(usize, 1), owned.req.graph_queries[0].query.return_aliases.len); - try std.testing.expectEqual(@as(u32, 10), owned.req.graph_queries[0].query.params.max_results); + const status_only_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .include_metric_status = true, + }, + }}; + const unsolicited_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + unsolicited_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.8 }, + }), + }; + const unsolicited_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + unsolicited_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = unsolicited_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var unsolicited_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = unsolicited_results, + }; + defer unsolicited_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &status_only_queries }, &.{unsolicited_payload}, 0, 10)); } -test "query parser owns graph match anchor through its required node alias" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"escaped":{"index":"graph_idx","match":{"anchor":"a\u0062","nodes":{"a\u0062":{}},"edges":[]},"return":{"bindings":["a\u0062"]}}}} - ); - defer owned.deinit(std.testing.allocator); - - const pattern = owned.req.graph_queries[0].query.match_pattern.?; - try std.testing.expect(pattern.anchor_alias != null); - const anchor_alias = pattern.anchor_alias.?; - try std.testing.expectEqualStrings("ab", anchor_alias); - try std.testing.expectEqual(@intFromPtr(pattern.nodes[0].alias.ptr), @intFromPtr(anchor_alias.ptr)); -} +test "query merge rejects malformed graph search traversal payloads" { + const alloc = std.testing.allocator; -test "query parser treats explicit graph document fields as a projection" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"walk":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]},"include_documents":true,"fields":["title"]}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + }, + }}; - const graph_query = owned.req.graph_queries[0].query; - try std.testing.expect(graph_query.include_documents); - try std.testing.expect(!graph_query.include_all_fields); - try std.testing.expectEqual(@as(usize, 1), graph_query.fields.len); - try std.testing.expectEqualStrings("title", graph_query.fields[0]); -} + const non_finite_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + non_finite_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = std.math.inf(f64), + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + const non_finite_node_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + non_finite_node_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = non_finite_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var non_finite_node = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = non_finite_node_results, + }; + defer non_finite_node.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{non_finite_node}, 0, 10)); + + const node_path_edges = try alloc.alloc(graph_query_mod.PathEdgeInfo, 1); + node_path_edges[0] = .{ + .source = try alloc.dupe(u8, "doc:a"), + .target = try alloc.dupe(u8, "doc:b"), + .edge_type = try alloc.dupe(u8, "links"), + .weight = std.math.nan(f64), + }; + const non_finite_node_edge_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + non_finite_node_edge_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = node_path_edges, + .metrics = &.{}, + }; + const non_finite_node_edge_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + non_finite_node_edge_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = non_finite_node_edge_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var non_finite_node_edge = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = non_finite_node_edge_results, + }; + defer non_finite_node_edge.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{non_finite_node_edge}, 0, 10)); + + const mismatched_path_nodes = try alloc.alloc([]const u8, 1); + mismatched_path_nodes[0] = try alloc.dupe(u8, "doc:a"); + const mismatched_path_edges = try alloc.alloc(graph_query_mod.PathEdgeInfo, 1); + mismatched_path_edges[0] = .{ + .source = try alloc.dupe(u8, "doc:a"), + .target = try alloc.dupe(u8, "doc:b"), + .edge_type = try alloc.dupe(u8, "links"), + .weight = 1.0, + }; + const mismatched_node_path_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + mismatched_node_path_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = mismatched_path_nodes, + .path_edges = mismatched_path_edges, + .metrics = &.{}, + }; + const mismatched_node_path_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + mismatched_node_path_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = mismatched_node_path_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var mismatched_node_path = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = mismatched_node_path_results, + }; + defer mismatched_node_path.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{mismatched_node_path}, 0, 10)); -test "query parser accepts exact graph count aggregates" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"pattern_count":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{}},"edges":[]},"return":{"aggregates":{"count":{"count":"*"}}}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); + const duplicate_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 2); + duplicate_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + duplicate_nodes[1] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + const duplicate_node_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_node_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = duplicate_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 2, + .metric_status = &.{}, + }; + var duplicate_node_payload = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_node_results, + }; + defer duplicate_node_payload.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{duplicate_node_payload}, 0, 10)); - const graph_query = owned.req.graph_queries[0].query; - try std.testing.expectEqual(@as(usize, 1), graph_query.aggregates.len); - try std.testing.expectEqualStrings("count", graph_query.aggregates[0].name); - try std.testing.expectEqualStrings("*", graph_query.aggregates[0].of); -} + const duplicate_shard_left_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + duplicate_shard_left_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:c"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + const duplicate_shard_left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_shard_left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = duplicate_shard_left_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var duplicate_shard_left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_shard_left_results, + }; + defer duplicate_shard_left.deinit(); -test "query parser accepts duplicate graph count expressions under different names" { - var owned = try parseQueryRequest(std.testing.allocator, null, "docs", - \\{"graph_queries":{"pattern_count":{"index":"graph_idx","match":{"anchor":"a","nodes":{"a":{}},"edges":[]},"return":{"aggregates":{"first":{"count":"a","distinct":true},"second":{"count":"a","distinct":true}}}}},"limit":10} - ); - defer owned.deinit(std.testing.allocator); - try std.testing.expectEqual(@as(usize, 2), owned.req.graph_queries[0].query.aggregates.len); + const duplicate_shard_right_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + duplicate_shard_right_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:c"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = &.{}, + }; + const duplicate_shard_right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_shard_right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = duplicate_shard_right_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var duplicate_shard_right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_shard_right_results, + }; + defer duplicate_shard_right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{ duplicate_shard_left, duplicate_shard_right }, 0, 10)); + + const path_nodes = try alloc.alloc([]const u8, 2); + path_nodes[0] = try alloc.dupe(u8, "doc:a"); + path_nodes[1] = try alloc.dupe(u8, "doc:b"); + const path_edges = try alloc.alloc(graph_paths.PathEdge, 1); + path_edges[0] = .{ + .source = try alloc.dupe(u8, "doc:a"), + .target = try alloc.dupe(u8, "doc:b"), + .edge_type = try alloc.dupe(u8, "links"), + .weight = 1.0, + }; + const non_finite_paths = try alloc.alloc(db_mod.types.GraphPath, 1); + non_finite_paths[0] = .{ + .nodes = path_nodes, + .edges = path_edges, + .total_weight = std.math.inf(f64), + .length = 1, + }; + const non_finite_path_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + non_finite_path_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = non_finite_paths, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var non_finite_path = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = non_finite_path_results, + }; + defer non_finite_path.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{non_finite_path}, 0, 10)); + + const bad_length_path_nodes = try alloc.alloc([]const u8, 2); + bad_length_path_nodes[0] = try alloc.dupe(u8, "doc:a"); + bad_length_path_nodes[1] = try alloc.dupe(u8, "doc:b"); + const bad_length_path_edges = try alloc.alloc(graph_paths.PathEdge, 1); + bad_length_path_edges[0] = .{ + .source = try alloc.dupe(u8, "doc:a"), + .target = try alloc.dupe(u8, "doc:b"), + .edge_type = try alloc.dupe(u8, "links"), + .weight = 1.0, + }; + const bad_length_paths = try alloc.alloc(db_mod.types.GraphPath, 1); + bad_length_paths[0] = .{ + .nodes = bad_length_path_nodes, + .edges = bad_length_path_edges, + .total_weight = 1.0, + .length = 2, + }; + const bad_length_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + bad_length_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = bad_length_paths, + .hits = &.{}, + .total_hits = 1, + .metric_status = &.{}, + }; + var bad_length = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = bad_length_results, + }; + defer bad_length.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{bad_length}, 0, 10)); } -test "query parser rejects semantic search offsets" { - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"limit":4,"offset":1} - )); +test "query merge rejects unqualified graph search identity collisions without collapsing qualified identities" { + const alloc = std.testing.allocator; + const queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"root"} }, + }, + }}; + var left_nodes = [_]graph_query_mod.GraphResultNode{.{ + .key = "shared", + .table = "people", + .depth = 1, + .distance = 1, + }}; + var right_nodes = [_]graph_query_mod.GraphResultNode{.{ + .key = "shared", + .table = "companies", + .depth = 1, + .distance = 1, + }}; + var left_hits = [_]db_mod.types.SearchHit{.{ + .id = @constCast("shared"), + .source_table = @constCast("people"), + }}; + var right_hits = [_]db_mod.types.SearchHit{.{ + .id = @constCast("shared"), + .source_table = @constCast("companies"), + }}; + var left_graph = [_]db_mod.types.GraphSearchResult{.{ + .name = @constCast("neighbors"), + .nodes = &left_nodes, + .hits = &left_hits, + .total_hits = 1, + }}; + var right_graph = [_]db_mod.types.GraphSearchResult{.{ + .name = @constCast("neighbors"), + .nodes = &right_nodes, + .hits = &right_hits, + .total_hits = 1, + }}; + const shards = [_]db_mod.types.SearchResult{ + .{ .alloc = alloc, .hits = &.{}, .total_hits = 0, .graph_results = &left_graph }, + .{ .alloc = alloc, .hits = &.{}, .total_hits = 0, .graph_results = &right_graph }, + }; + var merged = try mergeSearchResults(alloc, .{ .graph_queries = &queries }, &shards, 0, 10); + defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].nodes.len); + try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].hits.len); } -test "query parser records approximate source diagnostic for semantic exact sort" { - db_mod.resetLastSortRejectionDiagnostic(); - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"order_by":[{"field":"created_at","desc":true}],"limit":4} - )); - const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; - try std.testing.expectEqualStrings("created_at", diagnostic.field); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); -} +test "query merge rejects malformed graph search hit payloads" { + const alloc = std.testing.allocator; -test "query parser rejects semantic cursor-only pagination as approximate source" { - db_mod.resetLastSortRejectionDiagnostic(); - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"search_after":["doc:a"],"limit":4} - )); - const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; - try std.testing.expectEqualStrings("_id", diagnostic.field); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); -} + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + }, + }}; -test "query parser rejects semantic search_before pagination as approximate source" { - db_mod.resetLastSortRejectionDiagnostic(); - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(std.testing.allocator, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"search_before":["doc:a"],"limit":4} - )); - const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; - try std.testing.expectEqualStrings("_id", diagnostic.field); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); -} + const non_finite_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + non_finite_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = std.math.inf(f32), + }; + const non_finite_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + non_finite_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = non_finite_hits, + .total_hits = 1, + .metric_status = &.{}, + }; + var non_finite = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = non_finite_results, + }; + defer non_finite.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{non_finite}, 0, 10)); -test "query parser rejects semantic score sort as approximate source" { - const alloc = std.testing.allocator; - db_mod.resetLastSortRejectionDiagnostic(); - try std.testing.expectError(error.UnsupportedQueryRequest, parseQueryRequest(alloc, FakeSemanticResolver.iface(), "docs", - \\{"semantic_search":"alpha concept","indexes":["semantic_idx"],"order_by":[{"field":"_score","desc":true}],"limit":4} - )); - const diagnostic = db_mod.takeLastSortRejectionDiagnostic() orelse return error.TestUnexpectedResult; - try std.testing.expectEqualStrings("_score", diagnostic.field); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.reason); - try std.testing.expectEqualStrings("approximate_candidate_source", diagnostic.detail); -} + const duplicate_hits = try alloc.alloc(db_mod.types.SearchHit, 2); + duplicate_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 1.0, + }; + duplicate_hits[1] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 0.9, + }; + const duplicate_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = duplicate_hits, + .total_hits = 2, + .metric_status = &.{}, + }; + var duplicate = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_results, + }; + defer duplicate.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{duplicate}, 0, 10)); -test "query encoder emits antfly-style response envelope" { - const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .score = 1.25, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + const duplicate_shard_left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + duplicate_shard_left_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:c"), + .score = 1.0, }; - var result = db_mod.types.SearchResult{ + const duplicate_shard_left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_shard_left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = duplicate_shard_left_hits, + .total_hits = 1, + .metric_status = &.{}, + }; + var duplicate_shard_left = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_shard_left_results, + }; + defer duplicate_shard_left.deinit(); + + const duplicate_shard_right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + duplicate_shard_right_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:c"), + .score = 0.9, + }; + const duplicate_shard_right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_shard_right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = duplicate_shard_right_hits, .total_hits = 1, + .metric_status = &.{}, }; - defer result.deinit(); + var duplicate_shard_right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_shard_right_results, + }; + defer duplicate_shard_right.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{ duplicate_shard_left, duplicate_shard_right }, 0, 10)); - var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{}, result); - defer encoded.deinit(alloc); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"responses\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"table\":\"docs\"") != null); + const score_detail_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + score_detail_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 2.0, + .score_details = .{ + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .base_score = 1.0, + .base_weight = 1.0, + .metric_score = 0.5, + .metric_score_used = 0.5, + .metric_weight = 2.0, + .final_score = 2.0, + .published_generation = 8, + }, + }; + const score_detail_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + score_detail_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = score_detail_hits, + .total_hits = 1, + .metric_status = &.{}, + }; + var score_detail = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = score_detail_results, + }; + defer score_detail.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &graph_queries }, &.{score_detail}, 0, 10)); } -test "query encoder does not expose internal doc ordinals" { +test "query merge preserves failed graph metric status across shard fan-in" { const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .doc_ordinal = 42, - .native_text_doc_id = 7, - .score = 1.25, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + + const left_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + left_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.8 }; + const left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + left_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = left_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, }; - var result = db_mod.types.SearchResult{ + var left = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = 1, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = left_metrics, }; - defer result.deinit(); - - var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{}, result); - defer encoded.deinit(alloc); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "doc_ordinal") == null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "native_text_doc_id") == null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "ordinal") == null); -} + defer left.deinit(); -test "query encoder emits aggregations" { - const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .score = 1.25, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\",\"price\":10,\"category\":\"books\"}"), + const right_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + right_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.7 }; + const right_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + right_metrics[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = right_scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .failed, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .build_job_id = 99, + .retry_count = 3, + .last_error = try alloc.dupe(u8, "publish verifier rejected generation"), + .progress = 0.4, + .converged = true, + }, }; - var result = db_mod.types.SearchResult{ + var right = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = 1, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = right_metrics, }; - defer result.deinit(); + defer right.deinit(); - const aggregation_results = try alloc.alloc(db_mod.aggregations.SearchAggregationResult, 2); - aggregation_results[0] = .{ - .name = "price_stats", - .field = "price", - .type = "stats", - .value_json = try alloc.dupe(u8, "{\"count\":1,\"sum\":10,\"avg\":10,\"min\":10,\"max\":10,\"sum_squares\":100,\"variance\":0,\"std_dev\":0}"), + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .published, + }, + }}; + var merged = try mergeSearchResults(alloc, .{ .graph_metric_queries = &graph_metric_queries }, &.{ left, right }, 0, 10); + defer merged.deinit(); + + try std.testing.expectEqual(@as(usize, 1), merged.graph_metric_results.len); + try std.testing.expectEqual(@as(usize, 2), merged.graph_metric_results[0].scores.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, merged.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(u64, 5), merged.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(u64, 6), merged.graph_metric_results[0].status.target_edge_generation); + try std.testing.expectEqual(@as(u64, 6), merged.graph_metric_results[0].status.building_generation); + try std.testing.expectEqual(@as(u32, 3), merged.graph_metric_results[0].status.retry_count); + try std.testing.expectEqualStrings("publish verifier rejected generation", merged.graph_metric_results[0].status.last_error); + + const fresh_graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &fresh_graph_metric_queries }, &.{ left, right }, 0, 10)); + + const hits_left_authority_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + hits_left_authority_scores[0] = .{ .node = try alloc.dupe(u8, "doc:a"), .score = 0.9 }; + const hits_left_hub_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + hits_left_hub_scores[0] = .{ .node = try alloc.dupe(u8, "doc:c"), .score = 0.75 }; + const hits_left_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 2); + hits_left_metrics[0] = .{ + .name = try alloc.dupe(u8, "authority"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_authority"), + .scores = hits_left_authority_scores, + .status = .{ + .name = try alloc.dupe(u8, "hits_authority"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, + }, }; - const buckets = try alloc.alloc(db_mod.aggregations.SearchAggregationBucket, 1); - buckets[0] = .{ - .key_json = try alloc.dupe(u8, "\"books\""), - .count = 1, + hits_left_metrics[1] = .{ + .name = try alloc.dupe(u8, "hub"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_hub"), + .scores = hits_left_hub_scores, + .status = .{ + .name = try alloc.dupe(u8, "hits_hub"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, + }, }; - aggregation_results[1] = .{ - .name = "categories", - .field = "category", - .type = "terms", - .buckets = buckets, + var hits_left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = hits_left_metrics, }; - - var meta: QueryResponseMeta = .{ - .aggregation_results = aggregation_results, + defer hits_left.deinit(); + + const hits_right_authority_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + hits_right_authority_scores[0] = .{ .node = try alloc.dupe(u8, "doc:b"), .score = 0.7 }; + const hits_right_hub_scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + hits_right_hub_scores[0] = .{ .node = try alloc.dupe(u8, "doc:d"), .score = 0.65 }; + const hits_right_metrics = try alloc.alloc(db_mod.types.GraphMetricResult, 2); + hits_right_metrics[0] = .{ + .name = try alloc.dupe(u8, "authority"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_authority"), + .scores = hits_right_authority_scores, + .status = .{ + .name = try alloc.dupe(u8, "hits_authority"), + .state = .failed, + .published_generation = 8, + .edge_generation = 9, + .target_edge_generation = 9, + .building_generation = 9, + .retry_count = 2, + .last_error = try alloc.dupe(u8, "authority shard failed"), + .progress = 0.5, + .converged = true, + }, }; - defer meta.deinit(alloc); - - var encoded = try encodeQueryResponses(alloc, "docs", .{ - .aggregations_json = - \\{"price_stats":{"type":"stats","field":"price"},"categories":{"type":"terms","field":"category","size":5}} - , - }, meta, result); - defer encoded.deinit(alloc); - - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"aggregations\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"price_stats\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"sum\":10") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"categories\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"key\":\"books\"") != null); -} - -test "query encoder supports count-only and profile responses" { - const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .score = 1.25, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), + hits_right_metrics[1] = .{ + .name = try alloc.dupe(u8, "hub"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "hits_hub"), + .scores = hits_right_hub_scores, + .status = .{ + .name = try alloc.dupe(u8, "hits_hub"), + .state = .failed, + .published_generation = 8, + .edge_generation = 9, + .target_edge_generation = 9, + .building_generation = 9, + .retry_count = 2, + .last_error = try alloc.dupe(u8, "hub shard failed"), + .progress = 0.5, + .converged = true, + }, }; - var result = db_mod.types.SearchResult{ + var hits_right = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = 1, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = hits_right_metrics, }; - defer result.deinit(); + defer hits_right.deinit(); - var encoded = try encodeQueryResponses(alloc, "docs", .{ .count_only = true, .profile = true }, .{ - .took_ms = 7, - .shard_count = 3, - .merged = true, - .dense_search = .{ - .resolved_search_width = 128, - .resolved_epsilon = 0.15, - .hbc_reranked_vectors = 42, - .hbc_search_ns = 123456, + const hits_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 2, + .freshness = .published, + }, }, - }, result); - defer encoded.deinit(alloc); - try ant_json.testing.expectSubsetJsonText(alloc, - \\{"responses":[{"hits":{"total":{"value":1,"relation":"exact"},"hits":[]}}]} - , encoded.json); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"profile\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"took\":7") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"shards\":{\"total\":3,\"successful\":3,\"failed\":0}") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"merge\":{\"strategy\":\"rrf\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"dense_search\":{\"total_ns\":0") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"resolved_search_width\":128") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"resolved_epsilon\":0.15") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"hbc_reranked_vectors\":42") != null); + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 2, + .freshness = .published, + }, + }, + }; + var hits_merged = try mergeSearchResults(alloc, .{ .graph_metric_queries = &hits_metric_queries }, &.{ hits_left, hits_right }, 0, 10); + defer hits_merged.deinit(); + try std.testing.expectEqual(@as(usize, 2), hits_merged.graph_metric_results.len); + for (hits_merged.graph_metric_results) |metric_result| { + try std.testing.expectEqual(@as(usize, 2), metric_result.scores.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, metric_result.status.state); + try std.testing.expectEqual(@as(u64, 8), metric_result.status.published_generation); + try std.testing.expectEqual(@as(u64, 9), metric_result.status.target_edge_generation); + try std.testing.expectEqual(@as(u64, 9), metric_result.status.building_generation); + try std.testing.expectEqual(@as(u32, 2), metric_result.status.retry_count); + } + + const fresh_hits_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 2, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 2, + .freshness = .fresh, + }, + }, + }; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_metric_queries = &fresh_hits_metric_queries }, &.{ hits_left, hits_right }, 0, 10)); } -test "query encoder projects deferred stored fields without round-tripping bytes" { +test "query merge requires comparable graph search metric generations across shards" { const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .score = 1.25, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\",\"id\":\"stored-id\",\"body\":\"hello\"}"), + + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + .include_metric_status = true, + }, + }}; + const req = db_mod.types.SearchRequest{ .graph_queries = &graph_queries }; + + const left_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + left_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.8 }, + }), }; - var result = db_mod.types.SearchResult{ + const left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = left_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var left = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, + .hits = &.{}, + .total_hits = 0, + .graph_results = left_results, + }; + defer left.deinit(); + + const right_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + right_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:c"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.7 }, + }), + }; + const right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = right_nodes, + .paths = &.{}, + .hits = &.{}, .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }), }; - defer result.deinit(); + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = right_results, + }; + defer right.deinit(); - var encoded = try encodeQueryResponses(alloc, "docs", .{ - .fields = &.{ "id", "title" }, - .include_all_fields = false, - .defer_stored_projection = true, - }, .{}, result); - defer encoded.deinit(alloc); + var merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); + defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].nodes.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, merged.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 5), merged.graph_results[0].metric_status[0].published_generation); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:a\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_source\":{\"id\":\"stored-id\",\"title\":\"alpha\"}") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"body\"") == null); -} + const fresh_graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + const fresh_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &fresh_graph_metric_reads, + }, + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &fresh_graph_queries }, &.{ left, right }, 0, 10)); -test "query encoder omits _source for key-only hits" { - const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:key-only"), - .score = 0.75, - .stored_data = null, - }; - var result = db_mod.types.SearchResult{ + right_results[0].metric_status[0].published_generation = 4; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, right }, 0, 10)); + + right_results[0].metric_status[0].published_generation = 0; + right_results[0].metric_status[0].state = .not_ready; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, right }, 0, 10)); + + var missing = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = 1, + .hits = &.{}, + .total_hits = 0, }; - defer result.deinit(); + defer missing.deinit(); + try std.testing.expectError(error.InvalidRemoteResponse, mergeSearchResults(alloc, req, &.{ left, missing }, 0, 10)); +} - var encoded = try encodeQueryResponses(alloc, "docs", .{ - .include_all_fields = false, - }, .{}, result); - defer encoded.deinit(alloc); +test "query merge allows unpublished projected graph search metric status" { + const alloc = std.testing.allocator; - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_id\":\"doc:key-only\"") != null); - try std.testing.expect(std.mem.indexOf(u8, encoded.json, "\"_source\"") == null); -} + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + }, + }}; + const req = db_mod.types.SearchRequest{ .graph_queries = &graph_queries }; -test "query encoder emits graph results" { - const alloc = std.testing.allocator; - const graph_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); - graph_nodes[0] = .{ + const nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + nodes[0] = .{ .key = try alloc.dupe(u8, "doc:b"), .depth = 1, .distance = 1, .path = null, .path_edges = null, - }; - const graph_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - graph_hits[0] = .{ - .id = try alloc.dupe(u8, "doc:b"), - .score = 1, - .stored_data = try alloc.dupe(u8, "{\"title\":\"beta\"}"), + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = null }, + }), }; const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); graph_results[0] = .{ .name = try alloc.dupe(u8, "neighbors"), - .nodes = graph_nodes, + .nodes = nodes, .paths = &.{}, - .hits = graph_hits, + .hits = &.{}, .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 0, + .edge_generation = 3, + .target_edge_generation = 3, + .progress = 0.0, + .converged = false, + }, + }), }; var result = db_mod.types.SearchResult{ .alloc = alloc, @@ -2974,708 +7512,1147 @@ test "query encoder emits graph results" { }; defer result.deinit(); - const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + var merged = try mergeSearchResults(alloc, req, &.{result}, 0, 10); + defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results[0].nodes.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results[0].nodes[0].metrics.len); + try std.testing.expect(merged.graph_results[0].nodes[0].metrics[0].score == null); + try std.testing.expectEqual(@as(usize, 0), merged.graph_results[0].metric_status.len); + + const include_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "neighbors", .query = .{ - .query_type = .traverse, + .query_type = .neighbors, .index_name = "graph_idx", .start_nodes = .{ .keys = &.{"doc:a"} }, - .include_documents = true, + .metrics = &graph_metric_reads, + .include_metric_status = true, }, }}; - var encoded = try encodeQueryResponses(alloc, "docs", .{ - .graph_queries = &graph_queries, - .graph_query_transport = .{ - .dialect = .canonical, - .operations_json = - \\{"neighbors":{"traverse":{"index":"graph_idx","start":{"keys":["doc:a"]},"include_documents":true}}} - , - .admitted_operations_ptr = @ptrCast(graph_queries[0..].ptr), - .admitted_operations_len = graph_queries.len, + var included = try mergeSearchResults(alloc, .{ .graph_queries = &include_graph_queries }, &.{result}, 0, 10); + defer included.deinit(); + try std.testing.expectEqual(@as(usize, 1), included.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, included.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 0), included.graph_results[0].metric_status[0].published_generation); + + const fresh_graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + const fresh_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &fresh_graph_metric_reads, }, - }, .{ .took_ms = 4 }, result); - defer encoded.deinit(alloc); - var parsed = try ant_json.parseFromSlice(metadata_test_openapi.QueryResponses, alloc, encoded.json, .{}); - defer parsed.deinit(); - const responses = parsed.value.responses orelse return error.TestUnexpectedResult; - try std.testing.expectEqual(@as(usize, 1), responses.len); - const decoded_graph_results = responses[0].graph_results orelse return error.TestUnexpectedResult; - const result_value = decoded_graph_results.map.get("neighbors") orelse return error.TestUnexpectedResult; - const neighbors = switch (result_value) { - .graph_nodes_result => |value| value, - else => return error.TestUnexpectedResult, - }; - const nodes = neighbors.nodes; - try std.testing.expectEqual(@as(usize, 1), nodes.len); - try std.testing.expectEqualStrings("doc:b", nodes[0].key); - const document = nodes[0].document orelse return error.TestUnexpectedResult; - try std.testing.expectEqualStrings("beta", document.map.get("title").?.string); -} - -test "query merge applies global score ordering and offset" { - const alloc = std.testing.allocator; - - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 2); - left_hits[0] = .{ - .id = try alloc.dupe(u8, "doc:b"), - .doc_ordinal = 2, - .score = 2.0, - .stored_data = try alloc.dupe(u8, "{\"title\":\"beta\"}"), - }; - left_hits[1] = .{ - .id = try alloc.dupe(u8, "doc:a"), - .doc_ordinal = 1, - .score = 3.0, - .stored_data = try alloc.dupe(u8, "{\"title\":\"alpha\"}"), - }; - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = .{ - .id = try alloc.dupe(u8, "doc:c"), - .score = 1.0, - .stored_data = try alloc.dupe(u8, "{\"title\":\"gamma\"}"), - }; - - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 2 }; - defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; - defer right.deinit(); - - var merged = try mergeSearchResults(alloc, .{ .full_text = .{ .match = .{ .field = "body", .text = "alpha" } } }, &.{ left, right }, 1, 1); - defer merged.deinit(); - - try std.testing.expectEqual(@as(u32, 3), merged.total_hits); - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); - try std.testing.expectEqualStrings("doc:b", merged.hits[0].id); - try std.testing.expectEqual(@as(?u32, null), merged.hits[0].doc_ordinal); -} - -test "query merge allocation scales with the selected page" { - const large_stored = "x" ** 1024; - var input_hits: [2048]db_mod.types.SearchHit = undefined; - for (&input_hits) |*hit| { - hit.* = .{ - .id = @constCast("doc:a"), - .stored_data = @constCast(large_stored), - }; - } - const input = db_mod.types.SearchResult{ - .alloc = std.testing.allocator, - .hits = &input_hits, - .total_hits = input_hits.len, - }; - - // Enough for a bounded top-one heap plus one cloned page hit, but not for - // an O(candidate count) pointer array or cloned stored candidates. - var backing: [4096]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&backing); - var merged = try mergeSearchResults(fba.allocator(), .{}, &.{input}, 0, 1); - defer merged.deinit(); - - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); - try std.testing.expectEqualStrings("doc:a", merged.hits[0].id); - try std.testing.expectEqual(@as(usize, large_stored.len), merged.hits[0].stored_data.?.len); -} - -test "query merge rejects score ordered hits without finite scores" { - const alloc = std.testing.allocator; - - var missing_score_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - missing_score_hits[0] = .{ - .id = try alloc.dupe(u8, "doc:missing"), - }; - var missing_score = db_mod.types.SearchResult{ - .alloc = alloc, - .hits = missing_score_hits, - .total_hits = 1, - }; - defer missing_score.deinit(); - - const scoring_req = db_mod.types.SearchRequest{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - }; - try std.testing.expectError(error.InvalidQueryRequest, mergeSearchResults(alloc, scoring_req, &.{missing_score}, 0, 10)); + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &fresh_graph_queries }, &.{result}, 0, 10)); - var non_finite_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - non_finite_hits[0] = .{ - .id = try alloc.dupe(u8, "doc:nan"), - .score = std.math.nan(f32), + const graph_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "pagerank", + .freshness = .published, + }}; + const order_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &graph_metric_orders, + }, + }}; + const order_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + order_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 0, + .edge_generation = 3, + .target_edge_generation = 3, + .progress = 0.0, + .converged = false, + }, + }), }; - var non_finite = db_mod.types.SearchResult{ + var order_result = db_mod.types.SearchResult{ .alloc = alloc, - .hits = non_finite_hits, - .total_hits = 1, + .hits = &.{}, + .total_hits = 0, + .graph_results = order_results, }; - defer non_finite.deinit(); - - try std.testing.expectError(error.InvalidQueryRequest, mergeSearchResults(alloc, scoring_req, &.{non_finite}, 0, 10)); + defer order_result.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &order_graph_queries }, &.{order_result}, 0, 10)); } -test "query merge orders non score bearing hits by id without requiring scores" { +test "query merge rejects ambiguous graph search fan-in metric status" { const alloc = std.testing.allocator; - var hits = try alloc.alloc(db_mod.types.SearchHit, 2); - hits[0] = .{ - .id = try alloc.dupe(u8, "doc:b"), - .score = 100.0, - }; - hits[1] = .{ - .id = try alloc.dupe(u8, "doc:a"), - }; - - var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 2 }; - defer result.deinit(); - - var merged = try mergeSearchResults(alloc, .{ .full_text = .{ .match_all = {} } }, &.{result}, 0, 10); - defer merged.deinit(); - - try std.testing.expectEqual(@as(usize, 2), merged.hits.len); - try std.testing.expectEqualStrings("doc:a", merged.hits[0].id); - try std.testing.expectEqualStrings("doc:b", merged.hits[1].id); -} + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + }, + }}; + const req = db_mod.types.SearchRequest{ .graph_queries = &graph_queries }; -fn testSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, rank: i64) !db_mod.types.SearchHit { - const sort_values = try alloc.alloc(std.json.Value, 2); - errdefer alloc.free(sort_values); - sort_values[0] = .{ .integer = rank }; - sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); - return .{ - .id = try alloc.dupe(u8, id), - .doc_ordinal = @intCast(@max(rank, 0)), - .sort_values = sort_values, + const duplicate_status_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_status_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .progress = 1.0, + .converged = true, + }, + }), }; -} - -fn testIdSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8) !db_mod.types.SearchHit { - const sort_values = try alloc.alloc(std.json.Value, 1); - errdefer alloc.free(sort_values); - sort_values[0] = .{ .string = try alloc.dupe(u8, id) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); - return .{ - .id = try alloc.dupe(u8, id), - .sort_values = sort_values, + var duplicate_status = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_status_results, }; -} + defer duplicate_status.deinit(); -fn testScoreSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, score: f32) !db_mod.types.SearchHit { - const sort_values = try alloc.alloc(std.json.Value, 2); - errdefer alloc.free(sort_values); - sort_values[0] = .{ .float = @floatCast(score) }; - sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); - return .{ - .id = try alloc.dupe(u8, id), - .score = score, - .sort_values = sort_values, - }; -} + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{duplicate_status}, 0, 10)); -fn testHierarchyNavigationHitAlloc( - alloc: std.mem.Allocator, - id: []const u8, - position: []const u8, -) !db_mod.types.SearchHit { - const sort_values = try alloc.alloc(std.json.Value, 2); - errdefer alloc.free(sort_values); - sort_values[0] = .{ .string = try alloc.dupe(u8, position) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); - sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); - return .{ - .id = try alloc.dupe(u8, id), - .sort_values = sort_values, + const duplicate_query_results = try alloc.alloc(db_mod.types.GraphSearchResult, 2); + duplicate_query_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), }; -} + duplicate_query_results[1] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_query = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_query_results, + }; + defer duplicate_query.deinit(); -const TestHierarchyUnitChunk = struct { id: []const u8, score: f32 }; + try std.testing.expectError(error.InvalidRemoteResponse, mergeSearchResults(alloc, req, &.{duplicate_query}, 0, 10)); -fn testHierarchyUnitHitForIdAlloc( - alloc: std.mem.Allocator, - unit_id: []const u8, - score: f32, - chunks: []const TestHierarchyUnitChunk, -) !db_mod.types.SearchHit { - const id = try std.fmt.allocPrint( - alloc, - "doc:a/_artifact/asset/document_units_v1/{s}", - .{unit_id}, - ); - defer alloc.free(id); - const chunk_hits = try alloc.alloc(db_mod.types.ChunkHit, chunks.len); - var initialized: usize = 0; - errdefer { - for (chunk_hits[0..initialized]) |*chunk| chunk.deinit(alloc); - alloc.free(chunk_hits); - } - for (chunks, 0..) |chunk, i| { - chunk_hits[i] = .{ - .id = try alloc.dupe(u8, chunk.id), - .score = chunk.score, - }; - initialized += 1; - } - return .{ - .id = try alloc.dupe(u8, id), - .score = score, - .stored_data = try alloc.dupe(u8, "{\"_hierarchy_unit_revision_token\":\"revision-a\"}"), - .artifact_ref = .{ - .document_id = try alloc.dupe(u8, "doc:a"), - .name = try alloc.dupe(u8, "document_units_v1"), - .kind = .asset, - .unit_id = try alloc.dupe(u8, unit_id), + const duplicate_request_queries = [_]db_mod.types.NamedGraphQuery{ + .{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + }, + }, + .{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:b"} }, + .metrics = &graph_metric_reads, + }, }, - .chunk_hits = chunk_hits, }; -} - -fn testHierarchyUnitHitAlloc( - alloc: std.mem.Allocator, - score: f32, - chunks: []const TestHierarchyUnitChunk, -) !db_mod.types.SearchHit { - return testHierarchyUnitHitForIdAlloc(alloc, "unit:0", score, chunks); -} + const valid_query_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + valid_query_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_request = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = valid_query_results, + }; + defer duplicate_request.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &duplicate_request_queries }, &.{duplicate_request}, 0, 10)); -fn testHierarchyUnitShardResultAlloc( - alloc: std.mem.Allocator, - shard_index: usize, - hit_count: usize, -) !db_mod.types.SearchResult { - const hits = try alloc.alloc(db_mod.types.SearchHit, hit_count); - var initialized: usize = 0; - errdefer { - for (hits[0..initialized]) |*hit| hit.deinit(alloc); - alloc.free(hits); - } - for (hits, 0..) |*hit, hit_index| { - const unit_id = try std.fmt.allocPrint( - alloc, - "unit:{d:0>2}:{d:0>3}", - .{ shard_index, hit_index }, - ); - defer alloc.free(unit_id); - const score: f32 = @floatFromInt(hit_count - hit_index); - hit.* = try testHierarchyUnitHitForIdAlloc(alloc, unit_id, score, &.{}); - initialized += 1; - } - return .{ + const duplicate_order_metrics = [_]graph_query_mod.GraphMetricOrder{ + .{ + .name = "pagerank", + .freshness = .published, + }, + .{ + .name = "pagerank", + .direction = .asc, + .freshness = .published, + }, + }; + const duplicate_order_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &duplicate_order_metrics, + }, + }}; + const duplicate_order_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + duplicate_order_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var duplicate_order = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = @intCast(hit_count), + .hits = &.{}, + .total_hits = 0, + .graph_results = duplicate_order_results, }; + defer duplicate_order.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &duplicate_order_queries }, &.{duplicate_order}, 0, 10)); } -test "query merge treats hierarchy navigation positions as opaque cursor values" { +test "query merge preserves failed graph search metric status across shards" { const alloc = std.testing.allocator; - const order_by = [_]db_mod.types.SortField{ - .{ .field = "_hierarchy.position" }, - .{ .field = "_id" }, + + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + .include_metric_status = true, + }, + }}; + const req = db_mod.types.SearchRequest{ .graph_queries = &graph_queries }; + + const left_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + left_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:b"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.8 }, + }), }; - const cursor = [_]std.json.Value{ - .{ .string = "document_units_v1/00000000000000000007/00000000000000000000" }, - .{ .string = "artifact:page:1" }, + const left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = left_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = left_results, }; - - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyNavigationHitAlloc( - alloc, - "artifact:page:1", - "document_units_v1/00000000000000000007/00000000000000000000", - ); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyNavigationHitAlloc( - alloc, - "artifact:page:2", - "document_units_v1/00000000000000000007/00000000000000000001", - ); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 2 }; defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 2 }; + + const right_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + right_nodes[0] = .{ + .key = try alloc.dupe(u8, "doc:c"), + .depth = 1, + .distance = 1, + .path = null, + .path_edges = null, + .metrics = try alloc.dupe(graph_query_mod.GraphMetricValue, &.{ + .{ .name = try alloc.dupe(u8, "pagerank"), .score = 0.7 }, + }), + }; + const right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = right_nodes, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .failed, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .retry_count = 2, + .last_error = try alloc.dupe(u8, "metric rebuild failed"), + .progress = 0.35, + .converged = true, + }, + }), + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = right_results, + }; defer right.deinit(); - var merged = try mergeSearchResultsWithRuntimeSchema(alloc, .{ - .hierarchy_children = .{ .parent_id = "doc:a" }, - .order_by = &order_by, - .search_after = &cursor, - .limit = 20, - }, &.{ left, right }, 0, 20, .{}); + var merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); defer merged.deinit(); - - // Duplicate parent plans use a logical maximum rather than inflating the - // unit count, and the coordinator applies the opaque tuple cursor. - try std.testing.expectEqual(@as(u32, 2), merged.total_hits); - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); - try std.testing.expectEqualStrings("artifact:page:2", merged.hits[0].id); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].nodes.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, merged.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 5), merged.graph_results[0].metric_status[0].published_generation); + try std.testing.expectEqual(@as(u64, 6), merged.graph_results[0].metric_status[0].target_edge_generation); + try std.testing.expectEqual(@as(u64, 6), merged.graph_results[0].metric_status[0].building_generation); + try std.testing.expectEqual(@as(u32, 2), merged.graph_results[0].metric_status[0].retry_count); + try std.testing.expectEqualStrings("metric rebuild failed", merged.graph_results[0].metric_status[0].last_error); + + const fresh_graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + const fresh_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &fresh_graph_metric_reads, + }, + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &fresh_graph_queries }, &.{ left, right }, 0, 10)); } -test "query merge treats conflicting hierarchy navigation plans as retryable" { +test "query merge enforces graph search order and filter metric generations across shards" { const alloc = std.testing.allocator; - const order_by = [_]db_mod.types.SortField{ - .{ .field = "_hierarchy.position" }, - .{ .field = "_id" }, - }; - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-a/page/1"); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-b/page/1"); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + const graph_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "pagerank", + .op = .gte, + .value = 0.5, + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &graph_metric_orders, + .where_metric = &graph_metric_filters, + .include_metric_status = true, + }, + }}; + const req = db_mod.types.SearchRequest{ .graph_queries = &graph_queries }; + + const left_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + left_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }, + }), + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = left_results, + }; defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + + const right_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + right_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 1, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .progress = 0.25, + .converged = true, + }, + }), + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_results = right_results, + }; defer right.deinit(); - try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResultsWithRuntimeSchema( - alloc, - .{ - .hierarchy_children = .{ .parent_id = "doc:a" }, - .order_by = &order_by, - .limit = 20, + var merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); + defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), merged.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, merged.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 5), merged.graph_results[0].metric_status[0].published_generation); + try std.testing.expectEqual(@as(u64, 6), merged.graph_results[0].metric_status[0].building_generation); + + const no_status_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &graph_metric_orders, + .where_metric = &graph_metric_filters, }, - &.{ left, right }, - 0, - 20, - .{}, - )); + }}; + var no_status_merged = try mergeSearchResults(alloc, .{ .graph_queries = &no_status_graph_queries }, &.{ left, right }, 0, 10); + defer no_status_merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), no_status_merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 0), no_status_merged.graph_results[0].metric_status.len); + + right_results[0].metric_status[0].state = .failed; + right_results[0].metric_status[0].retry_count = 2; + right_results[0].metric_status[0].last_error = try alloc.dupe(u8, "order/filter metric rebuild failed"); + + var failed_merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); + defer failed_merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), failed_merged.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), failed_merged.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed_merged.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 5), failed_merged.graph_results[0].metric_status[0].published_generation); + try std.testing.expectEqual(@as(u64, 6), failed_merged.graph_results[0].metric_status[0].building_generation); + try std.testing.expectEqual(@as(u32, 2), failed_merged.graph_results[0].metric_status[0].retry_count); + try std.testing.expectEqualStrings("order/filter metric rebuild failed", failed_merged.graph_results[0].metric_status[0].last_error); + + const fresh_graph_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + const fresh_graph_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "pagerank", + .op = .gte, + .value = 0.5, + .freshness = .fresh, + }}; + const fresh_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &fresh_graph_metric_orders, + .where_metric = &fresh_graph_metric_filters, + }, + }}; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ .graph_queries = &fresh_graph_queries }, &.{ left, right }, 0, 10)); + + right_results[0].metric_status[0].published_generation = 4; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, right }, 0, 10)); + + right_results[0].metric_status[0].published_generation = 0; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, right }, 0, 10)); } -test "query merge treats malformed hierarchy navigation shard tuples as retryable" { +test "query profile reports merged graph search metric generation" { const alloc = std.testing.allocator; - const order_by = [_]db_mod.types.SortField{ - .{ .field = "_hierarchy.position" }, - .{ .field = "_id" }, - }; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = try testHierarchyNavigationHitAlloc(alloc, "artifact:page:1", "hn3/revision-a/page/1"); - alloc.free(hits[0].sort_values[1].string); - hits[0].sort_values[1] = .{ .string = try alloc.dupe(u8, "artifact:wrong-tiebreaker") }; - var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 1 }; - defer result.deinit(); - try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResultsWithRuntimeSchema( - alloc, - .{ - .hierarchy_children = .{ .parent_id = "doc:a" }, - .order_by = &order_by, - .limit = 20, + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, }, - &.{result}, - 0, - 20, - .{}, - )); -} + }}; -test "query merge releases hierarchy navigation candidates once at the global budget" { - const alloc = std.testing.allocator; - const order_by = [_]db_mod.types.SortField{ - .{ .field = "_hierarchy.position" }, - .{ .field = "_id" }, + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 2, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .progress = 0.5, + .converged = true, + }, + }), }; - const hit_count = db_mod.types.max_canonical_hierarchy_total_matches + 1; - const hits = try alloc.alloc(db_mod.types.SearchHit, hit_count); - var initialized: usize = 0; - errdefer { - for (hits[0..initialized]) |*hit| hit.deinit(alloc); - alloc.free(hits); - } - for (hits, 0..) |*hit, i| { - const id = try std.fmt.allocPrint(alloc, "unit:{d}", .{i}); - defer alloc.free(id); - const position = try std.fmt.allocPrint(alloc, "position/{d:0>8}", .{i}); - defer alloc.free(position); - hit.* = try testHierarchyNavigationHitAlloc(alloc, id, position); - initialized += 1; - } var result = db_mod.types.SearchResult{ .alloc = alloc, - .hits = hits, - .total_hits = hit_count, + .hits = &.{}, + .total_hits = 0, + .graph_results = graph_results, }; defer result.deinit(); - try std.testing.expectError(error.QueryCandidateBudgetExceeded, mergeSearchResultsWithRuntimeSchema( - alloc, - .{ - .hierarchy_children = .{ .parent_id = "doc:a" }, - .order_by = &order_by, - .limit = 20, + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .profile = true, + .graph_queries = &graph_queries, + .graph_query_transport = .{ + .dialect = .legacy, + .operations_json = "{}", + .admitted_operations_ptr = @ptrCast(graph_queries[0..].ptr), + .admitted_operations_len = graph_queries.len, }, - &.{result}, - 0, - 20, - .{}, - )); + }, .{ .took_ms = 2 }, result); + defer encoded.deinit(alloc); + + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"profile":{"graph_metrics":[{"query_name":"neighbors","source":"graph_query","index_name":"graph_idx","metric_name":"pagerank","freshness":"published","status":{"state":"stale","published_generation":5,"building_generation":6}}]}}]} + , encoded.json); } -test "query merge globally coalesces hierarchy unit groups and bounded chunks" { +test "query merge requires comparable graph metric rerank generations across shards" { const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{ - .{ .id = "chunk:a", .score = 0.8 }, - .{ .id = "chunk:shared", .score = 0.4 }, - }); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{ - .{ .id = "chunk:b", .score = 0.9 }, - .{ .id = "chunk:shared", .score = 0.5 }, - }); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + left_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 3.0, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = left_hits, + .total_hits = 1, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, + }, + }; defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; + + var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + right_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:b"), + .score = 2.0, + }; + var right = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = right_hits, + .total_hits = 1, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .stale, + .published_generation = 8, + .edge_generation = 9, + .target_edge_generation = 9, + .progress = 1.0, + .converged = true, + }, + }; defer right.deinit(); - var merged = try mergeSearchResults(alloc, .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit_with_chunks, - .hierarchy_group_level = .unit, - .hierarchy_grouped_matches = true, - .max_chunks_per_parent = 2, - .limit = 10, - }, &.{ left, right }, 0, 10); + const req = db_mod.types.SearchRequest{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 2.0, + }, + }; + + var merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); defer merged.deinit(); + try std.testing.expect(merged.graph_metric_rerank_status != null); + try std.testing.expectEqual(@as(u64, 8), merged.graph_metric_rerank_status.?.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, merged.graph_metric_rerank_status.?.state); + try std.testing.expectEqualStrings("doc:a", merged.hits[0].id); - try std.testing.expectEqual(@as(u32, 1), merged.total_hits); - try std.testing.expectEqual(db_mod.types.TotalHitsRelation.exact, merged.total_hits_relation); - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); - try std.testing.expectEqual(@as(?f32, 0.9), merged.hits[0].score); - try std.testing.expectEqual(@as(usize, 2), merged.hits[0].chunk_hits.len); - try std.testing.expectEqualStrings("chunk:b", merged.hits[0].chunk_hits[0].id); - try std.testing.expectEqualStrings("chunk:a", merged.hits[0].chunk_hits[1].id); + const old_right_error = right.graph_metric_rerank_status.?.last_error; + right.graph_metric_rerank_status.?.state = .failed; + right.graph_metric_rerank_status.?.target_edge_generation = 9; + right.graph_metric_rerank_status.?.building_generation = 9; + right.graph_metric_rerank_status.?.retry_count = 2; + right.graph_metric_rerank_status.?.last_error = try alloc.dupe(u8, "metric rebuild failed"); + if (old_right_error.len > 0) alloc.free(old_right_error); + + var failed_merged = try mergeSearchResults(alloc, req, &.{ left, right }, 0, 10); + defer failed_merged.deinit(); + try std.testing.expect(failed_merged.graph_metric_rerank_status != null); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed_merged.graph_metric_rerank_status.?.state); + try std.testing.expectEqual(@as(u64, 8), failed_merged.graph_metric_rerank_status.?.published_generation); + try std.testing.expectEqual(@as(u64, 9), failed_merged.graph_metric_rerank_status.?.building_generation); + try std.testing.expectEqual(@as(u32, 2), failed_merged.graph_metric_rerank_status.?.retry_count); + try std.testing.expectEqualStrings("metric rebuild failed", failed_merged.graph_metric_rerank_status.?.last_error); + + var encoded_failed = try encodeQueryResponses(alloc, "docs", .{ + .profile = true, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, .{ .took_ms = 5, .shard_count = 2, .merged = true }, failed_merged); + defer encoded_failed.deinit(alloc); + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"profile":{"shards":{"total":2,"successful":2,"failed":0},"graph_metrics":[{"query_name":"graph_metric_rerank","source":"graph_metric_rerank","index_name":"graph_idx","metric_name":"pagerank","freshness":"published","status":{"state":"failed","published_generation":8,"building_generation":9,"retry_count":2,"last_error":"metric rebuild failed"}}]}}]} + , encoded_failed.json); + + const fresh_req = db_mod.types.SearchRequest{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .fresh, + .weight = 1.0, + }, + }; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, fresh_req, &.{ left, right }, 0, 10)); + + right.graph_metric_rerank_status.?.published_generation = 7; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, right }, 0, 10)); } -test "query merge composes hierarchy unit groups with canonical graph results" { +test "query merge rejects malformed graph metric rerank score details" { const alloc = std.testing.allocator; - const cloneOneGraphResult = struct { - fn run( + + const Make = struct { + fn result( allocator: std.mem.Allocator, - source: db_mod.types.GraphSearchResult, - ) ![]db_mod.types.GraphSearchResult { - const out = try allocator.alloc(db_mod.types.GraphSearchResult, 1); - errdefer allocator.free(out); - out[0] = try cloneGraphSearchResult(allocator, source); - return out; + details_index_name: []const u8, + details_metric_name: []const u8, + details_generation: u64, + details_final_score: f64, + ) !db_mod.types.SearchResult { + var hits = try allocator.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try allocator.dupe(u8, "doc:a"), + .score = 2.0, + .score_details = .{ + .index_name = try allocator.dupe(u8, details_index_name), + .metric_name = try allocator.dupe(u8, details_metric_name), + .base_score = 1.0, + .base_weight = 1.0, + .metric_score = 0.5, + .metric_score_used = 0.5, + .metric_weight = 2.0, + .final_score = details_final_score, + .published_generation = details_generation, + }, + }; + return .{ + .alloc = allocator, + .hits = hits, + .total_hits = 1, + .graph_metric_rerank_status = .{ + .name = try allocator.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, + }, + }; } - }.run; - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); + }; - var left_node = [_]graph_query_mod.GraphResultNode{.{ - .key = "left", - .depth = 1, - .distance = 1, - }}; - var right_node = [_]graph_query_mod.GraphResultNode{.{ - .key = "right", - .depth = 1, - .distance = 1, - }}; - const left_graph = db_mod.types.GraphSearchResult{ - .name = @constCast("walk"), - .nodes = &left_node, - .hits = &.{}, - .total_hits = 1, + var unsolicited_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + unsolicited_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 2.0, + .score_details = .{ + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .base_score = 1.0, + .base_weight = 1.0, + .metric_score = 0.5, + .metric_score_used = 0.5, + .metric_weight = 2.0, + .final_score = 2.0, + .published_generation = 8, + }, }; - const right_graph = db_mod.types.GraphSearchResult{ - .name = @constCast("walk"), - .nodes = &right_node, - .hits = &.{}, + var unsolicited = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = unsolicited_hits, .total_hits = 1, }; + defer unsolicited.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{}, &.{unsolicited}, 0, 10)); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; - defer left.deinit(); - left.graph_results = try cloneOneGraphResult(alloc, left_graph); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; - defer right.deinit(); - right.graph_results = try cloneOneGraphResult(alloc, right_graph); - - const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ - .name = "walk", - .query = .{ - .query_type = .neighbors, - .index_name = "graph", - .start_nodes = .{ .keys = &.{} }, + const req = db_mod.types.SearchRequest{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, }, - }}; - var merged = try mergeSearchResults(alloc, .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .graph_queries = &graph_queries, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .limit = 10, - }, &.{ left, right }, 0, 10); - defer merged.deinit(); + }; - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); - try std.testing.expectEqual(@as(usize, 1), merged.graph_results.len); - try std.testing.expectEqualStrings("walk", merged.graph_results[0].name); - try std.testing.expectEqual(@as(usize, 2), merged.graph_results[0].nodes.len); - try std.testing.expectEqualStrings("left", merged.graph_results[0].nodes[0].key); - try std.testing.expectEqualStrings("right", merged.graph_results[0].nodes[1].key); -} + var wrong_generation = try Make.result(alloc, "graph_idx", "pagerank", 7, 2.0); + defer wrong_generation.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{wrong_generation}, 0, 10)); + + var wrong_identity = try Make.result(alloc, "other_idx", "pagerank", 8, 2.0); + defer wrong_identity.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{wrong_identity}, 0, 10)); -test "query merge treats conflicting hierarchy unit identities as retryable" { - const alloc = std.testing.allocator; - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); - const right_ref = &right_hits[0].artifact_ref.?; - alloc.free(right_ref.name); - right_ref.name = try alloc.dupe(u8, "document_units_v2"); + var mismatched_final_score = try Make.result(alloc, "graph_idx", "pagerank", 8, 1.5); + defer mismatched_final_score.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{mismatched_final_score}, 0, 10)); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; - defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; - defer right.deinit(); + var mismatched_weight = try Make.result(alloc, "graph_idx", "pagerank", 8, 2.0); + defer mismatched_weight.deinit(); + mismatched_weight.hits[0].score_details.?.metric_weight = 1.0; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{mismatched_weight}, 0, 10)); - try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResults( - alloc, - .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .limit = 10, + var mismatched_metric_score_used = try Make.result(alloc, "graph_idx", "pagerank", 8, 2.0); + defer mismatched_metric_score_used.deinit(); + mismatched_metric_score_used.hits[0].score_details.?.metric_score_used = 0.25; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{mismatched_metric_score_used}, 0, 10)); + + var mismatched_missing_score = try Make.result(alloc, "graph_idx", "pagerank", 8, 2.0); + defer mismatched_missing_score.deinit(); + mismatched_missing_score.hits[0].score_details.?.metric_score = null; + mismatched_missing_score.hits[0].score_details.?.missing_score_used = true; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{mismatched_missing_score}, 0, 10)); + + var mismatched_formula = try Make.result(alloc, "graph_idx", "pagerank", 8, 2.0); + defer mismatched_formula.deinit(); + mismatched_formula.hits[0].score_details.?.base_score = 0.5; + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{mismatched_formula}, 0, 10)); + + var non_finite = try Make.result(alloc, "graph_idx", "pagerank", 8, std.math.nan(f64)); + defer non_finite.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{non_finite}, 0, 10)); + + var non_finite_hits = try alloc.alloc(db_mod.types.SearchHit, 1); + non_finite_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = std.math.inf(f32), + }; + var non_finite_without_details = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = non_finite_hits, + .total_hits = 1, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, }, - &.{ left, right }, - 0, - 10, - )); + }; + defer non_finite_without_details.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{non_finite_without_details}, 0, 10)); } -test "query merge treats malformed hierarchy unit shard ranking as retryable" { +test "query merge rejects missing or unpublished graph metric rerank shard status" { const alloc = std.testing.allocator; + var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyUnitHitForIdAlloc(alloc, "unit:0", 0.8, &.{}); - left_hits[0].score = null; - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyUnitHitForIdAlloc(alloc, "unit:1", 0.7, &.{}); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 1 }; + left_hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 3.0, + }; + var left = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = left_hits, + .total_hits = 1, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 8, + .edge_generation = 8, + .target_edge_generation = 8, + .progress = 1.0, + .converged = true, + }, + }; defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 1 }; - defer right.deinit(); - try std.testing.expectError(error.StorageReadTemporarilyUnavailable, mergeSearchResults( - alloc, - .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .limit = 10, + const req = db_mod.types.SearchRequest{ + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, }, - &.{ left, right }, - 0, - 10, - )); + }; + + var missing = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + }; + defer missing.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, missing }, 0, 10)); + + var unpublished = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .not_ready, + .published_generation = 0, + .edge_generation = 8, + .target_edge_generation = 8, + }, + }; + defer unpublished.deinit(); + try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, req, &.{ left, unpublished }, 0, 10)); } -test "query merge reports an honest lower bound for a partial hierarchy unit union" { - const alloc = std.testing.allocator; - var left_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - left_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); - var right_hits = try alloc.alloc(db_mod.types.SearchHit, 1); - right_hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.9, &.{}); - var left = db_mod.types.SearchResult{ .alloc = alloc, .hits = left_hits, .total_hits = 10 }; - defer left.deinit(); - var right = db_mod.types.SearchResult{ .alloc = alloc, .hits = right_hits, .total_hits = 12 }; - defer right.deinit(); +test "query parser accepts direct graph metric reads" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_metric":{"name":"central","index":"graph_idx","metric":"pagerank","top_k":25,"metric_freshness":"fresh"}} + ); + defer owned.deinit(std.testing.allocator); - var merged = try mergeSearchResults(alloc, .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .limit = 10, - }, &.{ left, right }, 0, 10); - defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), owned.req.graph_metric_queries.len); + try std.testing.expectEqualStrings("central", owned.req.graph_metric_queries[0].name); + try std.testing.expectEqualStrings("graph_idx", owned.req.graph_metric_queries[0].query.index_name); + try std.testing.expectEqualStrings("pagerank", owned.req.graph_metric_queries[0].query.metric_name); + try std.testing.expectEqual(@as(u32, 25), owned.req.graph_metric_queries[0].query.top_k); + try std.testing.expectEqual(db_mod.types.GraphMetricFreshness.fresh, owned.req.graph_metric_queries[0].query.freshness); + try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank","top_k":0}} + )); +} - try std.testing.expectEqual(@as(u32, 12), merged.total_hits); - try std.testing.expectEqual(db_mod.types.TotalHitsRelation.gte, merged.total_hits_relation); - try std.testing.expectEqual(@as(usize, 1), merged.hits.len); +test "query parser accepts graph metric rerank" { + var owned = try parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match_all":{}},"graph_metric_rerank":{"index":"graph_idx","metric":"pagerank","candidate_count":80,"base_weight":0.5,"weight":2.5,"missing_score":-0.25,"metric_freshness":"fresh"},"offset":5,"limit":10} + ); + defer owned.deinit(std.testing.allocator); + + try std.testing.expect(owned.req.graph_metric_rerank != null); + try std.testing.expectEqualStrings("graph_idx", owned.req.graph_metric_rerank.?.index_name); + try std.testing.expectEqualStrings("pagerank", owned.req.graph_metric_rerank.?.metric_name); + try std.testing.expectEqual(db_mod.types.GraphMetricFreshness.fresh, owned.req.graph_metric_rerank.?.freshness); + try std.testing.expectEqual(@as(?u32, 80), owned.req.graph_metric_rerank.?.candidate_count); + try std.testing.expectEqual(@as(u32, 80), db_mod.types.graphMetricRerankCandidateCount(owned.req.graph_metric_rerank.?, owned.req.offset, owned.req.limit)); + try std.testing.expectApproxEqAbs(@as(f64, 0.5), owned.req.graph_metric_rerank.?.base_weight, 0.000001); + try std.testing.expectApproxEqAbs(@as(f64, 2.5), owned.req.graph_metric_rerank.?.weight, 0.000001); + try std.testing.expectApproxEqAbs(@as(f64, -0.25), owned.req.graph_metric_rerank.?.missing_score, 0.000001); + const adaptive = db_mod.types.GraphMetricRerank{ .index_name = "graph_idx", .metric_name = "pagerank" }; + try std.testing.expectEqual(@as(u32, 45), db_mod.types.graphMetricRerankCandidateCount(adaptive, 5, 10)); + try std.testing.expectError(error.QueryCandidateBudgetExceeded, db_mod.types.validateGraphMetricRerankWindow(adaptive, 9_999, 2)); + try std.testing.expectError(error.InvalidQueryRequest, parseQueryRequest(std.testing.allocator, null, "docs", + \\{"full_text_search":{"match_all":{}},"graph_metric_rerank":{"index":"graph_idx","metric":"pagerank","candidate_count":10},"offset":5,"limit":10} + )); } -test "query merge rejects exact sorting for hierarchy unit groups" { +test "query encoder emits graph metric results" { const alloc = std.testing.allocator; - const order_by = [_]db_mod.types.SortField{ - .{ .field = "_score", .desc = true }, - .{ .field = "_id" }, + var scores = try alloc.alloc(db_mod.types.GraphMetricScore, 1); + scores[0] = .{ + .node = try alloc.dupe(u8, "doc:b"), + .score = 0.75, + }; + var metric_results = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + metric_results[0] = .{ + .name = try alloc.dupe(u8, "pagerank"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = scores, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 3, + .edge_generation = 3, + .converged = true, + .iterations_completed = 12, + .delta = 0.00001, + .computed_at_ms = 1780000000000, + }, + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = metric_results, }; - var hits = try alloc.alloc(db_mod.types.SearchHit, 1); - hits[0] = try testHierarchyUnitHitAlloc(alloc, 0.8, &.{}); - var result = db_mod.types.SearchResult{ .alloc = alloc, .hits = hits, .total_hits = 1 }; defer result.deinit(); - try std.testing.expectError(error.UnsupportedQueryRequest, mergeSearchResults(alloc, .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .order_by = &order_by, - .limit = 10, - }, &.{result}, 0, 10)); + var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{}, result); + defer encoded.deinit(alloc); + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"graph_metric_results":{"pagerank":{"metric":"pagerank","status":{"state":"fresh"},"scores":[{"node":"doc:b","score":0.75}]}}}]} + , encoded.json); } -test "query merge bounds hierarchy unit selection by page instead of shard fanout" { +test "query profile reports failed graph metric status across read surfaces" { const alloc = std.testing.allocator; - const shard_count = 11; - const hits_per_shard = 100; - const results = try alloc.alloc(db_mod.types.SearchResult, shard_count); - var initialized: usize = 0; - defer { - for (results[0..initialized]) |*result| result.deinit(); - alloc.free(results); - } - for (results, 0..) |*result, shard_index| { - result.* = try testHierarchyUnitShardResultAlloc(alloc, shard_index, hits_per_shard); - initialized += 1; - } - var merged = try mergeSearchResults(alloc, .{ - .full_text = .{ .match = .{ .field = "body", .text = "alpha" } }, - .return_mode = .unit, - .hierarchy_group_level = .unit, - .limit = hits_per_shard, - }, results, 0, hits_per_shard); - defer merged.deinit(); + const graph_metric_results = try alloc.alloc(db_mod.types.GraphMetricResult, 1); + graph_metric_results[0] = .{ + .name = try alloc.dupe(u8, "central"), + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .scores = &.{}, + .status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .failed, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .retry_count = 2, + .last_error = try alloc.dupe(u8, "direct metric rebuild failed"), + .progress = 0.4, + .converged = true, + }, + }; - try std.testing.expectEqual(@as(usize, hits_per_shard), merged.hits.len); - try std.testing.expectEqual(@as(u32, hits_per_shard), merged.total_hits); - try std.testing.expectEqual(db_mod.types.TotalHitsRelation.gte, merged.total_hits_relation); - for (merged.hits, 0..) |hit, i| { - if (i > 0) try std.testing.expect(merged.hits[i - 1].score.? >= hit.score.?); - for (merged.hits[0..i]) |previous| { - try std.testing.expect(!std.mem.eql(u8, previous.id, hit.id)); - } - } -} + const graph_results = try alloc.alloc(db_mod.types.GraphSearchResult, 1); + graph_results[0] = .{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = &.{}, + .paths = &.{}, + .hits = &.{}, + .total_hits = 0, + .metric_status = try alloc.dupe(db_mod.types.GraphMetricStatus, &.{ + .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .failed, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .retry_count = 3, + .last_error = try alloc.dupe(u8, "graph query metric rebuild failed"), + .progress = 0.3, + .converged = true, + }, + }), + }; -fn testDateSortedQueryHitAlloc(alloc: std.mem.Allocator, id: []const u8, created_at_ns: u64) !db_mod.types.SearchHit { - const sort_values = try alloc.alloc(std.json.Value, 2); - errdefer alloc.free(sort_values); - sort_values[0] = .{ .string = try runtime_schema_mod.formatDateTimeNsAlloc(alloc, created_at_ns) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[0]); - sort_values[1] = .{ .string = try alloc.dupe(u8, id) }; - errdefer db_mod.types.deinitJsonValue(alloc, &sort_values[1]); - return .{ - .id = try alloc.dupe(u8, id), - .sort_values = sort_values, + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = &.{}, + .total_hits = 0, + .graph_metric_results = graph_metric_results, + .graph_results = graph_results, + .graph_metric_rerank_status = .{ + .name = try alloc.dupe(u8, "pagerank"), + .state = .failed, + .published_generation = 5, + .edge_generation = 6, + .target_edge_generation = 6, + .building_generation = 6, + .retry_count = 4, + .last_error = try alloc.dupe(u8, "rerank metric rebuild failed"), + .progress = 0.2, + .converged = true, + }, }; + defer result.deinit(); + + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + }, + }}; + const graph_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "neighbors", + .query = .{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &graph_metric_reads, + .include_metric_status = true, + }, + }}; + + var encoded = try encodeQueryResponses(alloc, "docs", .{ + .profile = true, + .graph_metric_queries = &graph_metric_queries, + .graph_queries = &graph_queries, + .graph_query_transport = .{ + .dialect = .legacy, + .operations_json = "{}", + .admitted_operations_ptr = @ptrCast(graph_queries[0..].ptr), + .admitted_operations_len = graph_queries.len, + }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + }, .{ .took_ms = 2 }, result); + defer encoded.deinit(alloc); + + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"profile":{"graph_metrics":[{"source":"graph_metric","freshness":"published","status":{"state":"failed","published_generation":5,"building_generation":6,"retry_count":2,"last_error":"direct metric rebuild failed"}},{"source":"graph_query","freshness":"published","status":{"state":"failed","published_generation":5,"building_generation":6,"retry_count":3,"last_error":"graph query metric rebuild failed"}},{"source":"graph_metric_rerank","freshness":"published","status":{"state":"failed","published_generation":5,"building_generation":6,"retry_count":4,"last_error":"rerank metric rebuild failed"}}]}}]} + , encoded.json); } -fn testRankRuntimeSchema() runtime_schema_mod.TableSchema { - const templates = struct { - const values = [_]runtime_schema_mod.DynamicTemplate{.{ - .name = "rank", - .path_match = "rank", - .mapping = .{ - .field_type = .numeric, - .doc_values = true, - .sortable = true, - .analyzer = "keyword", - }, - }}; - }.values; - return .{ .dynamic_templates = &templates }; +test "query encoder emits graph metric rerank score details" { + const alloc = std.testing.allocator; + var hits = try alloc.alloc(db_mod.types.SearchHit, 1); + hits[0] = .{ + .id = try alloc.dupe(u8, "doc:a"), + .score = 4.0, + .score_details = .{ + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .base_score = 1.0, + .base_weight = 0.5, + .metric_score = 0.7, + .metric_score_used = 0.7, + .metric_weight = 5.0, + .missing_score_used = false, + .final_score = 4.0, + .published_generation = 11, + }, + }; + var result = db_mod.types.SearchResult{ + .alloc = alloc, + .hits = hits, + .total_hits = 1, + }; + defer result.deinit(); + + var encoded = try encodeQueryResponses(alloc, "docs", .{}, .{ .took_ms = 2 }, result); + defer encoded.deinit(alloc); + + try ant_json.testing.expectSubsetJsonText(alloc, + \\{"responses":[{"hits":{"hits":[{"_id":"doc:a","_score_details":{"graph_metric_rerank":{"index_name":"graph_idx","metric_name":"pagerank","base_score":1,"base_weight":0.5,"metric_score":0.7,"metric_score_used":0.7,"metric_weight":5,"missing_score_used":false,"final_score":4,"published_generation":11}}}]}}]} + , encoded.json); } test "query merge rejects explicit score sort without score-bearing source" { @@ -4144,9 +9121,9 @@ test "query merge preserves graph table provenance under allocation failure" { } fn expectGraphMergeRowLimitAndDistinctIdentity(alloc: std.mem.Allocator) !void { - var binding_a = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("shared"), .table = @constCast("people"), .depth = 0, .distance = 0, .path = &.{}, .path_edges = &.{} } }}; - var binding_b = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("other"), .table = @constCast("people"), .depth = 0, .distance = 0, .path = &.{}, .path_edges = &.{} } }}; - var binding_c = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("shared"), .table = @constCast("companies"), .depth = 0, .distance = 0, .path = &.{}, .path_edges = &.{} } }}; + var binding_a = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("shared"), .table = @constCast("people"), .depth = 0, .distance = 0 } }}; + var binding_b = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("other"), .table = @constCast("people"), .depth = 0, .distance = 0 } }}; + var binding_c = [_]db_mod.types.GraphPatternBinding{.{ .alias = @constCast("n"), .node = .{ .key = @constCast("shared"), .table = @constCast("companies"), .depth = 0, .distance = 0 } }}; var shard_one_matches = [_]db_mod.types.GraphPatternMatch{ .{ .bindings = &binding_a, .path = &.{} }, .{ .bindings = &binding_b, .path = &.{} } }; var shard_two_matches = [_]db_mod.types.GraphPatternMatch{.{ .bindings = &binding_c, .path = &.{} }}; var shard_one_distinct = [_]graph_node_identity.Ref{.{ .table = "people", .key = "shared" }}; @@ -4409,7 +9386,12 @@ test "graph coordinator rejects merged node collections above the public cap" { const first_count = public_limits.max_graph_result_items; const first_nodes = try alloc.alloc(graph_query_mod.GraphResultNode, first_count); defer alloc.free(first_nodes); - for (first_nodes) |*node| node.* = .{ .key = @constCast("node"), .depth = 0, .distance = 0 }; + const first_keys = try alloc.alloc([@sizeOf(usize)]u8, first_count); + defer alloc.free(first_keys); + for (first_nodes, first_keys, 0..) |*node, *key, i| { + key.* = @bitCast(i); + node.* = .{ .key = key[0..], .depth = 0, .distance = 0 }; + } const overflow_nodes = [_]graph_query_mod.GraphResultNode{.{ .key = @constCast("overflow"), .depth = 0, diff --git a/zig/pkg/antfly/src/api/query_contract.zig b/zig/pkg/antfly/src/api/query_contract.zig index bc596d8487..0e8cdbcb6e 100644 --- a/zig/pkg/antfly/src/api/query_contract.zig +++ b/zig/pkg/antfly/src/api/query_contract.zig @@ -2445,6 +2445,7 @@ const QueryBodyContractFields = struct { has_public_doc_filter_bindings: bool, has_public_hierarchy_controls: bool, has_query_timeout: bool, + has_graph_metric: bool, }; const RawGraphValueEntry = struct { @@ -2624,6 +2625,7 @@ fn queryBodyContractFields(alloc: std.mem.Allocator, body: []const u8) !QueryBod .has_public_doc_filter_bindings = parsed.value.object.get("with") != null, .has_public_hierarchy_controls = objectHasNonNullField(parsed.value.object, "hierarchy"), .has_query_timeout = parsed.value.object.get("timeout_ms") != null, + .has_graph_metric = objectHasNonNullField(parsed.value.object, "graph_metric"), }; } @@ -2724,6 +2726,7 @@ pub fn parseQueryRequestWithDeadline( // Admission interprets this extension semantically, so escaped JSON // member names and the canonical spelling have identical behavior. .strip_query_timeout = contract_fields.has_query_timeout, + .strip_graph_metric = contract_fields.has_graph_metric, }); defer if (contract_body) |owned| alloc.free(owned); try ensureQueryDeadline(execution_deadline_ns); @@ -2816,6 +2819,11 @@ pub fn parseQueryRequestWithDeadline( req.dense_queries = vector_queries.dense; req.sparse_queries = vector_queries.sparse; req.graph_queries = try buildGraphQueries(alloc, request); + req.graph_metric_queries = try parseGraphMetricQueriesAlloc(alloc, effective_body); + req.graph_metric_rerank = try parseGraphMetricRerankAlloc(alloc, request.graph_metric_rerank); + if (req.graph_metric_rerank) |rerank| { + try db_mod.types.validateGraphMetricRerankWindow(rerank, req.offset, req.limit); + } if (req.graph_queries.len > 0) { req.graph_query_transport = try captureGraphQueryTransportAlloc(alloc, effective_body, req.graph_queries); } @@ -3559,6 +3567,10 @@ pub fn encodeQueryResponses( null; const profile = if (req.profile) try buildProfileValue(arena, req, meta, result) else null; + const graph_metric_results = if (result.graph_metric_results.len > 0) + try buildGraphMetricResults(arena, result.graph_metric_results) + else + null; const response_json = switch (graph_dialect orelse .canonical) { .canonical => blk: { const graph_results = if (graph_dialect != null) @@ -3573,6 +3585,7 @@ pub fn encodeQueryResponses( .max_score = computeMaxScore(emitted_hits), }, .aggregations = aggregations, + .graph_metric_results = graph_metric_results, .graph_results = graph_results, .profile = profile, .took = meta.took_ms, @@ -3602,6 +3615,7 @@ pub fn encodeQueryResponses( .max_score = computeMaxScore(emitted_hits), }, .aggregations = aggregations, + .graph_metric_results = graph_metric_results, .graph_results = graph_results, .profile = profile, .took = meta.took_ms, @@ -3632,6 +3646,7 @@ fn toOpenApiHit(alloc: std.mem.Allocator, req: db_mod.types.SearchRequest, hit: return .{ ._id = hit.id, ._score = if (hit.score) |score| finiteScoreOrZero(score) else 0, + ._score_details = toOpenApiScoreDetails(hit.score_details), ._distance = if (hit.distance) |distance| finiteScoreOrZero(distance) else null, ._index_scores = try indexScoresJsonValue(alloc, hit.index_scores), ._sort = if (hit.sort_values.len > 0) hit.sort_values else null, @@ -3668,6 +3683,22 @@ fn takeOpenApiObjectMap( return error.InvalidRemoteResponse; } +fn toOpenApiScoreDetails(details: ?db_mod.types.GraphMetricRerankScoreDetails) ?metadata_openapi.QueryScoreDetails { + const rerank = details orelse return null; + return .{ .graph_metric_rerank = .{ + .index_name = rerank.index_name, + .metric_name = rerank.metric_name, + .base_score = rerank.base_score, + .base_weight = rerank.base_weight, + .metric_score = metadata_openapi.types.OpenApiOptionalNullable(f64).fromNullable(rerank.metric_score), + .metric_score_used = rerank.metric_score_used, + .metric_weight = rerank.metric_weight, + .missing_score_used = rerank.missing_score_used, + .final_score = rerank.final_score, + .published_generation = saturatingI64(rerank.published_generation), + } }; +} + fn searchHitHierarchyOpenApiValue( alloc: std.mem.Allocator, req: db_mod.types.SearchRequest, @@ -5558,6 +5589,129 @@ fn jsonValueToI64(value: std.json.Value) !?i64 { }; } +fn buildGraphMetricResults( + alloc: std.mem.Allocator, + results: []const db_mod.types.GraphMetricResult, +) !std.json.ArrayHashMap(indexes_openapi.GraphMetricResult) { + var out: std.json.ArrayHashMap(indexes_openapi.GraphMetricResult) = .{}; + errdefer out.deinit(alloc); + for (results) |result| { + const scores = try alloc.alloc(indexes_openapi.GraphMetricScore, result.scores.len); + for (result.scores, 0..) |score, i| scores[i] = .{ .node = score.node, .score = score.score }; + try out.map.put(alloc, result.name, .{ + .index_name = result.index_name, + .metric = result.metric_name, + .scores = scores, + .status = try toOpenApiGraphMetricStatus(alloc, result.status), + }); + } + return out; +} + +fn graphMetricStateName(state: graph_mod.GraphIndex.GraphMetricState) []const u8 { + return @tagName(state); +} + +fn graphMetricPhaseName(phase: graph_mod.GraphIndex.GraphMetricBuildPhase) []const u8 { + return @tagName(phase); +} + +fn saturatingI64(value: u64) i64 { + return std.math.cast(i64, value) orelse std.math.maxInt(i64); +} + +fn toOpenApiGraphMetricBuildPages( + alloc: std.mem.Allocator, + pages: []const db_mod.types.GraphMetricBuildPageStatus, +) !?[]const indexes_openapi.GraphMetricBuildPageStatus { + if (pages.len == 0) return null; + const out = try alloc.alloc(indexes_openapi.GraphMetricBuildPageStatus, pages.len); + for (pages, 0..) |page, i| out[i] = .{ + .phase = @tagName(page.phase), + .iteration = @intCast(page.iteration), + .page_id = saturatingI64(page.page_id), + .state = @tagName(page.state), + .range_kind = @tagName(page.range_kind), + .worker_id = if (page.worker_id.len > 0) page.worker_id else null, + .lease_expires_at_ms = saturatingI64(page.lease_expires_at_ms), + .attempt = saturatingI64(page.attempt), + .cursor = if (page.cursor.len > 0) page.cursor else null, + .completed_units = saturatingI64(page.completed_units), + .total_units = saturatingI64(page.total_units), + .last_error = if (page.last_error.len > 0) page.last_error else null, + }; + return out; +} + +pub fn toOpenApiGraphMetricStatus( + alloc: std.mem.Allocator, + status: db_mod.types.GraphMetricStatus, +) !indexes_openapi.GraphMetricStatus { + const events = if (status.recent_events.len > 0) blk: { + const out = try alloc.alloc(indexes_openapi.GraphMetricEvent, status.recent_events.len); + for (status.recent_events, 0..) |event, i| out[i] = toOpenApiGraphMetricEvent(event); + break :blk out; + } else null; + return .{ + .state = graphMetricStateName(status.state), + .phase = graphMetricPhaseName(status.phase), + .edge_filter = .{ .mode = @tagName(status.edge_filter.mode), .types = if (status.edge_filter.types.len > 0) status.edge_filter.types else null }, + .metadata_version = @intCast(status.metadata_version), + .config_fingerprint = try std.fmt.allocPrint(alloc, "{x:0>16}", .{status.config_fingerprint}), + .maintenance_paused = status.maintenance_paused, + .build_queued = status.build_queued, + .published_generation = saturatingI64(status.published_generation), + .edge_generation = saturatingI64(status.edge_generation), + .target_edge_generation = saturatingI64(status.target_edge_generation), + .queued_generation = saturatingI64(status.queued_generation), + .building_generation = saturatingI64(status.building_generation), + .build_job_id = saturatingI64(status.build_job_id), + .build_started_at_ms = saturatingI64(status.build_started_at_ms), + .build_iteration = @intCast(status.build_iteration), + .build_lease_expires_at_ms = saturatingI64(status.build_lease_expires_at_ms), + .build_worker_id = if (status.build_worker_id.len > 0) status.build_worker_id else null, + .build_cursor = if (status.build_cursor.len > 0) status.build_cursor else null, + .build_completed_units = saturatingI64(status.build_completed_units), + .build_total_units = saturatingI64(status.build_total_units), + .build_pages = try toOpenApiGraphMetricBuildPages(alloc, status.build_pages), + .build_pages_truncated = status.build_pages_truncated, + .retry_count = saturatingI64(status.retry_count), + .last_error = if (status.last_error.len > 0) status.last_error else null, + .progress = status.progress, + .converged = status.converged, + .iterations_completed = @intCast(status.iterations_completed), + .delta = status.delta, + .computed_at_ms = saturatingI64(status.computed_at_ms), + .last_event = if (status.last_event) |event| toOpenApiGraphMetricEvent(event) else null, + .recent_events = events, + }; +} + +fn toOpenApiGraphMetricEvent(event: graph_mod.GraphIndex.GraphMetricEvent) indexes_openapi.GraphMetricEvent { + return .{ + .sequence = saturatingI64(event.sequence), + .kind = @tagName(event.kind), + .at_ms = saturatingI64(event.at_ms), + .target_edge_generation = saturatingI64(event.target_edge_generation), + .published_generation = saturatingI64(event.published_generation), + .score_count = saturatingI64(event.score_count), + }; +} + +fn toOpenApiGraphMetricStatusMap( + alloc: std.mem.Allocator, + statuses: []const db_mod.types.GraphMetricStatus, +) !?std.json.ArrayHashMap(indexes_openapi.GraphMetricStatus) { + if (statuses.len == 0) return null; + var out: std.json.ArrayHashMap(indexes_openapi.GraphMetricStatus) = .{}; + errdefer out.deinit(alloc); + for (statuses) |status| { + if (out.map.contains(status.name)) return error.InvalidRemoteResponse; + try out.map.put(alloc, status.name, try toOpenApiGraphMetricStatus(alloc, status)); + } + return out; +} + fn buildGraphQueryResults( comptime Result: type, alloc: std.mem.Allocator, @@ -5720,6 +5874,7 @@ fn toOpenApiStatefulGraphResultWithFormat( ), .total = @intCast(graph_result.total_hits), .took = meta.took_ms, + .metric_status = try toOpenApiGraphMetricStatusMap(alloc, graph_result.metric_status), }; return .{ .legacy_graph_search_result = response }; } @@ -5816,6 +5971,7 @@ fn toOpenApiStatefulGraphResultWithFormat( response.* = .{ .kind = "nodes", .nodes = nodes, + .metric_status = try toOpenApiGraphMetricStatusMap(alloc, graph_result.metric_status), .stats = .{ .returned_items = @intCast(nodes.len), .truncated = graph_result.truncated, @@ -7289,6 +7445,64 @@ test "graph document lookup enforces its defensive hydration budget" { ); } +fn buildGraphMetricProfiles( + alloc: std.mem.Allocator, + req: db_mod.types.SearchRequest, + result: db_mod.types.SearchResult, +) !?[]metadata_openapi.GraphMetricProfile { + var count: usize = result.graph_metric_results.len; + for (result.graph_results) |graph_result| count += graph_result.metric_status.len; + if (result.graph_metric_rerank_status != null) count += 1; + if (count == 0) return null; + const profiles = try alloc.alloc(metadata_openapi.GraphMetricProfile, count); + var out: usize = 0; + for (result.graph_metric_results) |metric_result| { + var freshness: db_mod.types.GraphMetricFreshness = .published; + for (req.graph_metric_queries) |query| { + if (std.mem.eql(u8, query.name, metric_result.name)) freshness = query.query.freshness; + } + profiles[out] = .{ + .query_name = metric_result.name, + .source = "graph_metric", + .index_name = metric_result.index_name, + .metric_name = metric_result.metric_name, + .freshness = @tagName(freshness), + .status = try toOpenApiGraphMetricStatus(alloc, metric_result.status), + }; + out += 1; + } + for (result.graph_results) |graph_result| { + var index_name: []const u8 = ""; + for (req.graph_queries) |query| if (std.mem.eql(u8, query.name, graph_result.name)) { + index_name = query.query.index_name; + break; + }; + for (graph_result.metric_status) |status| { + profiles[out] = .{ + .query_name = graph_result.name, + .source = "graph_query", + .index_name = index_name, + .metric_name = status.name, + .freshness = "published", + .status = try toOpenApiGraphMetricStatus(alloc, status), + }; + out += 1; + } + } + if (result.graph_metric_rerank_status) |status| { + const rerank = req.graph_metric_rerank orelse return error.UnsupportedQueryRequest; + profiles[out] = .{ + .query_name = "graph_metric_rerank", + .source = "graph_metric_rerank", + .index_name = rerank.index_name, + .metric_name = rerank.metric_name, + .freshness = @tagName(rerank.freshness), + .status = try toOpenApiGraphMetricStatus(alloc, status), + }; + } + return profiles; +} + fn buildProfileValue( alloc: std.mem.Allocator, req: db_mod.types.SearchRequest, @@ -7321,6 +7535,7 @@ fn buildProfileValue( .semantic_hits = if (req.dense_queries.len > 0 or req.sparse_queries.len > 0) result.total_hits else 0, .duration_ms = meta.took_ms, } else null, + .graph_metrics = try buildGraphMetricProfiles(alloc, req, result), }; const encoded = try jsonStringifyAlloc(alloc, profile); defer alloc.free(encoded); @@ -10205,6 +10420,25 @@ pub fn parseLegacyGraphQuery( else @constCast((&[_][]const u8{})[0..]); errdefer freeOwnedStringSlice(alloc, fields); + const metric_freshness = if (query.metric_freshness) |freshness| + try parseGraphMetricFreshnessString(freshness) + else + graph_query_mod.GraphMetricFreshness.published; + const metrics = if (query.metrics) |values| + try parseGraphMetricReads(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricRead{})[0..]); + errdefer freeGraphMetricReads(alloc, metrics); + const metric_order = if (query.order_by) |values| + try parseGraphMetricOrders(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricOrder{})[0..]); + errdefer freeGraphMetricOrders(alloc, metric_order); + const metric_filters = if (query.where_metric) |values| + try parseGraphMetricFilters(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricFilter{})[0..]); + errdefer freeGraphMetricFilters(alloc, metric_filters); if (query.type == .pattern) { if (pattern.len == 0) return error.UnsupportedQueryRequest; @@ -10221,7 +10455,7 @@ pub fn parseLegacyGraphQuery( else 1; - return .{ + const parsed_query: graph_query_mod.GraphQuery = .{ .query_type = switch (query.type) { .traverse => .traverse, .neighbors => .neighbors, @@ -10239,9 +10473,253 @@ pub fn parseLegacyGraphQuery( .include_documents = query.include_documents orelse false, .fields = fields, .include_all_fields = query.fields == null, + .metrics = metrics, + .order_by = metric_order, + .where_metric = metric_filters, + .include_metric_status = query.include_metric_status orelse false, + }; + try graph_query_mod.validateGraphMetricQueryShape(parsed_query); + return parsed_query; +} + +fn parseGraphMetricQueriesAlloc( + alloc: std.mem.Allocator, + body: []const u8, +) ![]const db_mod.types.NamedGraphMetricQuery { + var parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch return error.InvalidQueryRequest; + defer parsed.deinit(); + if (parsed.value != .object) return error.InvalidQueryRequest; + if (parsed.value.object.get("_graph_metric_queries")) |internal_queries| { + if (internal_queries != .array or internal_queries.array.items.len == 0 or + internal_queries.array.items.len > graph_query_mod.graph_metric_projection_limit) + return error.InvalidQueryRequest; + const items = try alloc.alloc(db_mod.types.NamedGraphMetricQuery, internal_queries.array.items.len); + var initialized: usize = 0; + errdefer { + for (items[0..initialized]) |item| { + alloc.free(item.name); + alloc.free(item.query.index_name); + alloc.free(item.query.metric_name); + } + alloc.free(items); + } + for (internal_queries.array.items, 0..) |value, i| { + if (value != .object) return error.InvalidQueryRequest; + const index_name = try parseRequiredStringField(value.object, "index"); + const metric_name = try parseRequiredStringField(value.object, "metric"); + const result_name = try parseRequiredStringField(value.object, "name"); + if (index_name.len == 0 or metric_name.len == 0 or result_name.len == 0) + return error.InvalidQueryRequest; + for (items[0..initialized]) |prior| { + if (std.mem.eql(u8, prior.name, result_name)) return error.InvalidQueryRequest; + } + const top_k = if (value.object.get("top_k")) |raw| try parseOptionalU32FieldValue(raw) else 10; + if (top_k == 0 or top_k > 10_000) return error.InvalidQueryRequest; + const freshness = if (value.object.get("metric_freshness")) |raw| + try parseGraphMetricFreshness(raw) + else + db_mod.types.GraphMetricFreshness.published; + const owned_name = try alloc.dupe(u8, result_name); + errdefer alloc.free(owned_name); + const owned_index_name = try alloc.dupe(u8, index_name); + errdefer alloc.free(owned_index_name); + const owned_metric_name = try alloc.dupe(u8, metric_name); + errdefer alloc.free(owned_metric_name); + items[i] = .{ + .name = owned_name, + .query = .{ + .index_name = owned_index_name, + .metric_name = owned_metric_name, + .top_k = top_k, + .freshness = freshness, + }, + }; + initialized += 1; + } + return items; + } + const metric_value = parsed.value.object.get("graph_metric") orelse return &.{}; + if (metric_value == .null) return &.{}; + if (metric_value != .object) return error.InvalidQueryRequest; + const index_name = try parseRequiredStringField(metric_value.object, "index"); + const metric_name = try parseRequiredStringField(metric_value.object, "metric"); + if (index_name.len == 0 or metric_name.len == 0) return error.InvalidQueryRequest; + const result_name = if (metric_value.object.get("name") != null) + try parseRequiredStringField(metric_value.object, "name") + else + metric_name; + if (result_name.len == 0) return error.InvalidQueryRequest; + const top_k = if (metric_value.object.get("top_k")) |value| try parseOptionalU32FieldValue(value) else 10; + if (top_k == 0 or top_k > 10_000) return error.InvalidQueryRequest; + const freshness = if (metric_value.object.get("metric_freshness")) |value| + try parseGraphMetricFreshness(value) + else if (metric_value.object.get("freshness")) |value| + try parseGraphMetricFreshness(value) + else + db_mod.types.GraphMetricFreshness.published; + const items = try alloc.alloc(db_mod.types.NamedGraphMetricQuery, 1); + errdefer alloc.free(items); + const owned_name = try alloc.dupe(u8, result_name); + errdefer alloc.free(owned_name); + const owned_index_name = try alloc.dupe(u8, index_name); + errdefer alloc.free(owned_index_name); + const owned_metric_name = try alloc.dupe(u8, metric_name); + errdefer alloc.free(owned_metric_name); + items[0] = .{ + .name = owned_name, + .query = .{ + .index_name = owned_index_name, + .metric_name = owned_metric_name, + .top_k = top_k, + .freshness = freshness, + }, + }; + return items; +} + +fn parseGraphMetricRerankAlloc( + alloc: std.mem.Allocator, + maybe_rerank: ?indexes_openapi.GraphMetricRerank, +) !?db_mod.types.GraphMetricRerank { + const rerank = maybe_rerank orelse return null; + if (rerank.index.len == 0 or rerank.metric.len == 0) return error.InvalidQueryRequest; + const base_weight = rerank.base_weight orelse 1.0; + const weight = rerank.weight orelse 1.0; + const missing_score = rerank.missing_score orelse 0.0; + if (!std.math.isFinite(base_weight) or !std.math.isFinite(weight) or !std.math.isFinite(missing_score)) return error.InvalidQueryRequest; + const index_name = try alloc.dupe(u8, rerank.index); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, rerank.metric); + errdefer alloc.free(metric_name); + const freshness = if (rerank.metric_freshness) |value| try parseGraphMetricFreshnessStringForRequest(value) else .published; + return .{ + .index_name = index_name, + .metric_name = metric_name, + .freshness = freshness, + .candidate_count = if (rerank.candidate_count) |count| + std.math.cast(u32, count) orelse return error.InvalidQueryRequest + else + null, + .base_weight = base_weight, + .weight = weight, + .missing_score = missing_score, }; } +/// Parse only graph-metric extensions without invoking semantic embedding or +/// the general search normalizer. Serverless serving uses this to compose +/// immutable metric artifacts with its independent lake search planner. +pub const OwnedGraphMetricRequests = struct { + queries: []const db_mod.types.NamedGraphMetricQuery = &.{}, + rerank: ?db_mod.types.GraphMetricRerank = null, + + pub fn deinit(self: *OwnedGraphMetricRequests, alloc: std.mem.Allocator) void { + freeNamedGraphMetricQueries(alloc, self.queries); + if (self.rerank) |rerank| { + alloc.free(@constCast(rerank.index_name)); + alloc.free(@constCast(rerank.metric_name)); + } + self.* = undefined; + } +}; + +pub fn parseGraphMetricRequestsAlloc(alloc: std.mem.Allocator, body: []const u8) !OwnedGraphMetricRequests { + if (body.len == 0) return error.InvalidQueryRequest; + var parsed = ant_json.parseFromSlice(metadata_openapi.QueryRequest, alloc, body, .{ + .ignore_unknown_fields = true, + .allocate = .alloc_always, + }) catch return error.InvalidQueryRequest; + defer parsed.deinit(); + const queries = try parseGraphMetricQueriesAlloc(alloc, body); + errdefer freeNamedGraphMetricQueries(alloc, queries); + return .{ + .queries = queries, + .rerank = try parseGraphMetricRerankAlloc(alloc, parsed.value.graph_metric_rerank), + }; +} + +fn parseRequiredStringField(object: std.json.ObjectMap, field: []const u8) ![]const u8 { + const value = object.get(field) orelse return error.InvalidQueryRequest; + if (value != .string) return error.InvalidQueryRequest; + return value.string; +} + +fn parseOptionalU32FieldValue(value: std.json.Value) !u32 { + if (value != .integer) return error.InvalidQueryRequest; + return std.math.cast(u32, value.integer) orelse return error.InvalidQueryRequest; +} + +fn parseGraphMetricFreshness(value: std.json.Value) !db_mod.types.GraphMetricFreshness { + if (value != .string) return error.InvalidQueryRequest; + return try parseGraphMetricFreshnessStringForRequest(value.string); +} + +fn parseGraphMetricFreshnessStringForRequest(value: []const u8) !db_mod.types.GraphMetricFreshness { + if (std.mem.eql(u8, value, "published")) return .published; + if (std.mem.eql(u8, value, "fresh")) return .fresh; + return error.InvalidQueryRequest; +} + +fn parseGraphMetricFreshnessString(value: []const u8) !graph_query_mod.GraphMetricFreshness { + if (std.mem.eql(u8, value, "published")) return .published; + if (std.mem.eql(u8, value, "fresh")) return .fresh; + return error.InvalidQueryRequest; +} + +fn parseGraphMetricReads(alloc: std.mem.Allocator, values: []const []const u8, freshness: graph_query_mod.GraphMetricFreshness) ![]const graph_query_mod.GraphMetricRead { + if (values.len > graph_query_mod.graph_metric_projection_limit) return error.InvalidQueryRequest; + const out = try alloc.alloc(graph_query_mod.GraphMetricRead, values.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| alloc.free(item.name); + alloc.free(out); + } + for (values, 0..) |value, i| { + if (value.len == 0) return error.InvalidQueryRequest; + out[i] = .{ .name = try alloc.dupe(u8, value), .freshness = freshness }; + initialized += 1; + } + return out; +} + +fn parseGraphMetricOrders(alloc: std.mem.Allocator, values: []const indexes_openapi.GraphMetricOrder, freshness: graph_query_mod.GraphMetricFreshness) ![]const graph_query_mod.GraphMetricOrder { + if (values.len > graph_query_mod.graph_metric_order_limit) return error.InvalidQueryRequest; + const out = try alloc.alloc(graph_query_mod.GraphMetricOrder, values.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| alloc.free(item.name); + alloc.free(out); + } + for (values, 0..) |value, i| { + if (value.metric.len == 0) return error.InvalidQueryRequest; + out[i] = .{ + .name = try alloc.dupe(u8, value.metric), + .direction = if (std.mem.eql(u8, value.direction orelse "desc", "asc")) .asc else if (std.mem.eql(u8, value.direction orelse "desc", "desc")) .desc else return error.InvalidQueryRequest, + .nulls = if (std.mem.eql(u8, value.nulls orelse "last", "first") or std.mem.eql(u8, value.nulls orelse "last", "nulls_first")) .first else if (std.mem.eql(u8, value.nulls orelse "last", "last") or std.mem.eql(u8, value.nulls orelse "last", "nulls_last")) .last else return error.InvalidQueryRequest, + .freshness = freshness, + }; + initialized += 1; + } + return out; +} + +fn parseGraphMetricFilters(alloc: std.mem.Allocator, values: []const indexes_openapi.GraphMetricFilter, freshness: graph_query_mod.GraphMetricFreshness) ![]const graph_query_mod.GraphMetricFilter { + if (values.len > graph_query_mod.graph_metric_filter_limit) return error.InvalidQueryRequest; + const out = try alloc.alloc(graph_query_mod.GraphMetricFilter, values.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| alloc.free(item.name); + alloc.free(out); + } + for (values, 0..) |value, i| { + if (value.metric.len == 0 or !std.math.isFinite(value.value)) return error.InvalidQueryRequest; + const op = std.meta.stringToEnum(graph_query_mod.GraphMetricFilterOp, value.op) orelse return error.InvalidQueryRequest; + out[i] = .{ .name = try alloc.dupe(u8, value.metric), .op = op, .value = value.value, .freshness = freshness }; + initialized += 1; + } + return out; +} + fn parseLegacyPatternSteps( alloc: std.mem.Allocator, value: []const indexes_openapi.PatternStep, @@ -10364,7 +10842,26 @@ fn parseGraphTraverseQuery(alloc: std.mem.Allocator, value: indexes_openapi.Grap errdefer freePatternNodeFilter(alloc, filter); const fields = if (traversal.fields) |items| try cloneFields(alloc, items) else &.{}; errdefer freeOwnedStringSlice(alloc, fields); - return .{ + const metric_freshness = if (traversal.metric_freshness) |freshness| + try parseGraphMetricFreshnessString(freshness) + else + graph_query_mod.GraphMetricFreshness.published; + const metrics = if (traversal.metrics) |values| + try parseGraphMetricReads(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricRead{})[0..]); + errdefer freeGraphMetricReads(alloc, metrics); + const metric_order = if (traversal.order_by) |values| + try parseGraphMetricOrders(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricOrder{})[0..]); + errdefer freeGraphMetricOrders(alloc, metric_order); + const metric_filters = if (traversal.where_metric) |values| + try parseGraphMetricFilters(alloc, values, metric_freshness) + else + @constCast((&[_]graph_query_mod.GraphMetricFilter{})[0..]); + errdefer freeGraphMetricFilters(alloc, metric_filters); + const parsed: graph_query_mod.GraphQuery = .{ .query_type = .traverse, .index_name = index, .start_nodes = start, @@ -10382,7 +10879,13 @@ fn parseGraphTraverseQuery(alloc: std.mem.Allocator, value: indexes_openapi.Grap .include_documents = traversal.include_documents orelse false, .fields = fields, .include_all_fields = traversal.fields == null, + .metrics = metrics, + .order_by = metric_order, + .where_metric = metric_filters, + .include_metric_status = traversal.include_metric_status orelse false, }; + try graph_query_mod.validateGraphMetricQueryShape(parsed); + return parsed; } fn parseGraphPathQuery( @@ -11457,6 +11960,11 @@ fn freeSearchRequest(alloc: std.mem.Allocator, req: *db_mod.types.SearchRequest) freeNamedDenseQueries(alloc, req.dense_queries); freeNamedSparseQueries(alloc, req.sparse_queries); freeNamedGraphQueries(alloc, req.graph_queries); + freeNamedGraphMetricQueries(alloc, req.graph_metric_queries); + if (req.graph_metric_rerank) |rerank| { + alloc.free(@constCast(rerank.index_name)); + alloc.free(@constCast(rerank.metric_name)); + } if (req.graph_query_transport) |*transport| transport.deinit(alloc); freeNamedDocFilterBindings(alloc, req.doc_filter_bindings); if (req.sparse) |sparse| { @@ -12399,6 +12907,7 @@ const QueryContractStripOptions = struct { strip_public_doc_filter_bindings: bool = false, strip_public_hierarchy_controls: bool = false, strip_query_timeout: bool = false, + strip_graph_metric: bool = false, }; fn queryBodyForGeneratedContractAlloc( @@ -12406,7 +12915,7 @@ fn queryBodyForGeneratedContractAlloc( body: []const u8, options: QueryContractStripOptions, ) !?[]u8 { - if (!options.strip_internal_shard_fields and !options.strip_public_doc_filter_bindings and !options.strip_public_hierarchy_controls and !options.strip_query_timeout) return null; + if (!options.strip_internal_shard_fields and !options.strip_public_doc_filter_bindings and !options.strip_public_hierarchy_controls and !options.strip_query_timeout and !options.strip_graph_metric) return null; var parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch return error.InvalidQueryRequest; defer parsed.deinit(); @@ -12424,6 +12933,9 @@ fn queryBodyForGeneratedContractAlloc( if (options.strip_query_timeout) { _ = parsed.value.object.orderedRemove("timeout_ms"); } + if (options.strip_graph_metric) { + _ = parsed.value.object.orderedRemove("graph_metric"); + } return try std.json.Stringify.valueAlloc(alloc, parsed.value, .{}); } @@ -13299,6 +13811,30 @@ fn freeNamedGraphQueries(alloc: std.mem.Allocator, items: []const db_mod.types.N if (items.len > 0) alloc.free(items); } +fn freeNamedGraphMetricQueries(alloc: std.mem.Allocator, items: []const db_mod.types.NamedGraphMetricQuery) void { + for (items) |item| { + alloc.free(item.name); + alloc.free(item.query.index_name); + alloc.free(item.query.metric_name); + } + if (items.len > 0) alloc.free(items); +} + +fn freeGraphMetricReads(alloc: std.mem.Allocator, metrics: []const graph_query_mod.GraphMetricRead) void { + for (metrics) |metric| alloc.free(metric.name); + if (metrics.len > 0) alloc.free(metrics); +} + +fn freeGraphMetricOrders(alloc: std.mem.Allocator, orders: []const graph_query_mod.GraphMetricOrder) void { + for (orders) |order| alloc.free(order.name); + if (orders.len > 0) alloc.free(orders); +} + +fn freeGraphMetricFilters(alloc: std.mem.Allocator, filters: []const graph_query_mod.GraphMetricFilter) void { + for (filters) |filter| alloc.free(filter.name); + if (filters.len > 0) alloc.free(filters); +} + pub fn freeGraphQuery(alloc: std.mem.Allocator, query: graph_query_mod.GraphQuery) void { alloc.free(query.index_name); freeGraphNodeSelector(alloc, query.start_nodes); @@ -13316,6 +13852,9 @@ pub fn freeGraphQuery(alloc: std.mem.Allocator, query: graph_query_mod.GraphQuer freeGraphCountAggregates(alloc, query.aggregates); for (query.fields) |field| alloc.free(field); if (query.fields.len > 0) alloc.free(query.fields); + freeGraphMetricReads(alloc, query.metrics); + freeGraphMetricOrders(alloc, query.order_by); + freeGraphMetricFilters(alloc, query.where_metric); } fn freeGraphMatchNodes(alloc: std.mem.Allocator, nodes: []const graph_pattern_mod.MatchNode) void { @@ -17107,6 +17646,98 @@ test "api query contract rejects invalid timeout_ms" { )); } +test "api query contract bounds graph metric top k" { + const alloc = std.testing.allocator; + const accepted = try parseGraphMetricQueriesAlloc(alloc, + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank","top_k":10000}} + ); + defer freeNamedGraphMetricQueries(alloc, accepted); + try std.testing.expectEqual(@as(usize, 1), accepted.len); + try std.testing.expectEqual(@as(u32, 10_000), accepted[0].query.top_k); + + try std.testing.expectError(error.InvalidQueryRequest, parseGraphMetricQueriesAlloc(alloc, + \\{"graph_metric":{"index":"graph_idx","metric":"pagerank","top_k":10001}} + )); +} + +test "api query contract uses portable graph metric filter operators" { + const cases = [_]struct { wire: []const u8, expected: graph_query_mod.GraphMetricFilterOp }{ + .{ .wire = "gt", .expected = .gt }, + .{ .wire = "gte", .expected = .gte }, + .{ .wire = "lt", .expected = .lt }, + .{ .wire = "lte", .expected = .lte }, + .{ .wire = "eq", .expected = .eq }, + .{ .wire = "neq", .expected = .neq }, + }; + for (cases) |case| { + const filters = [_]indexes_openapi.GraphMetricFilter{.{ + .metric = "pagerank", + .op = case.wire, + .value = 0.5, + }}; + const parsed = try parseLegacyGraphQuery(std.testing.allocator, .{ + .type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .where_metric = &filters, + }); + defer freeGraphQuery(std.testing.allocator, parsed); + try std.testing.expectEqual(case.expected, parsed.where_metric[0].op); + } + + const legacy = [_]indexes_openapi.GraphMetricFilter{.{ + .metric = "pagerank", + .op = ">=", + .value = 0.5, + }}; + try std.testing.expectError(error.InvalidQueryRequest, parseLegacyGraphQuery(std.testing.allocator, .{ + .type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .where_metric = &legacy, + })); +} + +test "api query contract rejects oversized and duplicate graph metric clauses" { + var metric_names: [graph_query_mod.graph_metric_projection_limit + 1][]const u8 = undefined; + @memset(&metric_names, "pagerank"); + try std.testing.expectError(error.InvalidQueryRequest, parseGraphMetricReads( + std.testing.allocator, + &metric_names, + .published, + )); + + const duplicate_metrics = [_][]const u8{ "pagerank", "pagerank" }; + try std.testing.expectError(error.InvalidQueryRequest, parseLegacyGraphQuery(std.testing.allocator, .{ + .type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &duplicate_metrics, + })); +} + +test "api query contract preserves graph metric fingerprint precision" { + const alloc = std.testing.allocator; + const converted = try toOpenApiGraphMetricStatus(alloc, .{ + .name = @constCast("pagerank"), + .config_fingerprint = std.math.maxInt(u64), + }); + defer alloc.free(converted.config_fingerprint.?); + try std.testing.expectEqualStrings("ffffffffffffffff", converted.config_fingerprint.?); +} + +test "api query contract treats nullable graph metric extensions as absent" { + const alloc = std.testing.allocator; + var parsed = try parsePublicQueryRequest(alloc, null, "docs", + \\{"full_text_search":{"match":"needle","field":"body"},"graph_metric":null,"graph_metric_rerank":null} + ); + defer parsed.deinit(alloc); + + try std.testing.expect(parsed.req.full_text != null); + try std.testing.expectEqual(@as(usize, 0), parsed.req.graph_metric_queries.len); + try std.testing.expect(parsed.req.graph_metric_rerank == null); +} + test "api query contract rejects legacy native doc id constraint fields" { const alloc = std.testing.allocator; const body = diff --git a/zig/pkg/antfly/src/api/request_admission_policy.zig b/zig/pkg/antfly/src/api/request_admission_policy.zig index 170af346dd..d913368dbb 100644 --- a/zig/pkg/antfly/src/api/request_admission_policy.zig +++ b/zig/pkg/antfly/src/api/request_admission_policy.zig @@ -84,6 +84,7 @@ pub const public_operation_policies = [_]PublicOperationPolicy{ .{ .operation_id = "getIndex", .class = .none }, .{ .operation_id = "createIndex", .class = .none }, .{ .operation_id = "dropIndex", .class = .none }, + .{ .operation_id = "executeGraphMetricAction", .class = .none }, .{ .operation_id = "linearMerge", .class = .write }, .{ .operation_id = "queryTable", .class = .query }, .{ .operation_id = "listTableRepairIssues", .class = .none }, diff --git a/zig/pkg/antfly/src/api/runtime_status.zig b/zig/pkg/antfly/src/api/runtime_status.zig index 479d3a90d7..192e48df65 100644 --- a/zig/pkg/antfly/src/api/runtime_status.zig +++ b/zig/pkg/antfly/src/api/runtime_status.zig @@ -4536,6 +4536,7 @@ fn statusStatsHaveRuntimeFacts(stats: db_mod.types.DBStats) bool { if (stats.async_indexing.startup.active or stats.async_indexing.dense_catch_up.active) return true; if (stats.enrichment.enabled and (stats.enrichment.processed_requests > 0 or stats.enrichment.applied_sequence > 0 or stats.enrichment.target_sequence > 0 or stats.enrichment.retrying or stats.enrichment.worker_failed)) return true; if (stats.text_merge.pending_segments > 0 or stats.text_merge.in_flight_merges > 0 or stats.text_merge.completed_merges > 0 or stats.text_merge.failed_merges > 0) return true; + if (stats.graph_metric_runtime.hasRuntimeFacts()) return true; for (stats.indexes) |index| { if (indexHasArtifactVisibilityFacts(index)) return true; if (index.repair_degraded or index.repair_issue_count != 0) return true; @@ -5336,58 +5337,49 @@ fn cloneResolverReplayDiagnostics(alloc: std.mem.Allocator, stats: db_mod.types. }; } +test "graph metric cached index stats clone retains owned progress and survives allocation failures" { + try testGraphMetricStatsClone(std.testing.allocator); + try std.testing.checkAllAllocationFailures(std.testing.allocator, testGraphMetricStatsClone, .{}); +} + +fn testGraphMetricStatsClone(alloc: std.mem.Allocator) !void { + var pages = [_]db_mod.types.GraphMetricBuildPageStatus{.{ + .worker_id = "page-worker", + .cursor = "page-cursor", + .last_error = "page-error", + }}; + var metrics = [_]db_mod.types.GraphMetricStatus{.{ + .name = @constCast("pagerank"), + .state = .building, + .published_generation = 7, + .build_worker_id = "worker", + .build_cursor = "cursor", + .last_error = "diagnostic", + .build_pages = &pages, + }}; + var indexes = [_]db_mod.types.DBIndexStats{.{ .name = "graph_idx", .kind = .graph, .graph_metric_status = &metrics }}; + const cloned = try cloneDBStats(alloc, .{ .indexes = &indexes, .index_count = 1 }); + defer db_mod.types.freeDBStats(alloc, cloned); + try std.testing.expectEqual(@as(usize, 1), cloned.indexes[0].graph_metric_status.len); + const status = cloned.indexes[0].graph_metric_status[0]; + try std.testing.expectEqualStrings("pagerank", status.name); + try std.testing.expect(status.name.ptr != metrics[0].name.ptr); + try std.testing.expectEqual(@as(u64, 7), status.published_generation); + try std.testing.expectEqualStrings("cursor", status.build_cursor); + try std.testing.expect(status.build_cursor.ptr != metrics[0].build_cursor.ptr); + try std.testing.expectEqualStrings("page-cursor", status.build_pages[0].cursor); + try std.testing.expect(status.build_pages[0].cursor.ptr != pages[0].cursor.ptr); + try std.testing.expectEqualStrings("page-worker", status.build_pages[0].worker_id); + try std.testing.expectEqualStrings("page-error", status.build_pages[0].last_error); +} + pub fn cloneDBStats(alloc: std.mem.Allocator, stats: db_mod.types.DBStats) !db_mod.types.DBStats { const resolver_replay = try cloneResolverReplayDiagnostics(alloc, stats.resolver_replay); errdefer db_mod.types.freeResolverReplayDiagnostics(alloc, resolver_replay); const indexes = try alloc.alloc(db_mod.types.DBIndexStats, stats.indexes.len); var initialized: usize = 0; errdefer { - for (indexes[0..initialized]) |item| { - alloc.free(item.name); - if (item.load_error) |value| alloc.free(value); - if (item.index_repair_last_error) |value| alloc.free(value); - if (item.algebraic_last_error_doc_key) |value| alloc.free(value); - if (item.algebraic_last_error_reason) |value| alloc.free(value); - if (item.algebraic_capability_fingerprint) |value| alloc.free(value); - if (item.algebraic_capability_lifecycle_status) |value| alloc.free(value); - if (item.algebraic_planner_last_decision) |value| alloc.free(value); - if (item.algebraic_planner_last_fallback_reason) |value| alloc.free(value); - if (item.algebraic_planner_lifecycle_blocking_reason) |value| alloc.free(value); - if (item.algebraic_last_observed_query_shape) |value| alloc.free(value); - if (item.algebraic_last_recommended_materialization) |value| alloc.free(value); - if (item.algebraic_top_candidate) |candidate| { - alloc.free(candidate.recommendation); - alloc.free(candidate.materialization_id); - alloc.free(candidate.lifecycle); - alloc.free(candidate.decision); - } - if (item.algebraic_active_progress) |progress| { - alloc.free(progress.recommendation); - alloc.free(progress.materialization_id); - alloc.free(progress.lifecycle); - } - for (item.algebraic_candidates) |candidate| { - alloc.free(candidate.recommendation); - alloc.free(candidate.materialization_id); - alloc.free(candidate.lifecycle); - alloc.free(candidate.decision); - } - if (item.algebraic_candidates.len > 0) alloc.free(item.algebraic_candidates); - for (item.algebraic_candidate_decision_history) |entry| { - alloc.free(entry.recommendation); - alloc.free(entry.materialization_id); - alloc.free(entry.lifecycle); - alloc.free(entry.previous_decision); - alloc.free(entry.decision); - } - if (item.algebraic_candidate_decision_history.len > 0) alloc.free(item.algebraic_candidate_decision_history); - for (item.algebraic_progress) |progress| { - alloc.free(progress.recommendation); - alloc.free(progress.materialization_id); - alloc.free(progress.lifecycle); - } - if (item.algebraic_progress.len > 0) alloc.free(item.algebraic_progress); - } + for (indexes[0..initialized]) |item| db_mod.types.freeDBIndexStatsItem(alloc, item); alloc.free(indexes); } @@ -5488,7 +5480,10 @@ pub fn cloneDBStats(alloc: std.mem.Allocator, stats: db_mod.types.DBStats) !db_m errdefer freeAlgebraicCandidateDecisionStatuses(alloc, algebraic_candidate_decision_history); const algebraic_progress = try cloneAlgebraicProgressStatuses(alloc, item.algebraic_progress); errdefer freeAlgebraicProgressStatuses(alloc, algebraic_progress); + const graph_metric_status = try db_mod.types.cloneGraphMetricStatuses(alloc, item.graph_metric_status); + errdefer db_mod.types.freeGraphMetricStatuses(alloc, graph_metric_status); indexes[i] = .{ + .graph_metric_status = graph_metric_status, .name = try alloc.dupe(u8, item.name), .kind = item.kind, .runtime_observation_stale = item.runtime_observation_stale, diff --git a/zig/pkg/antfly/src/api/table_contract.zig b/zig/pkg/antfly/src/api/table_contract.zig index 199bf581f2..074d106731 100644 --- a/zig/pkg/antfly/src/api/table_contract.zig +++ b/zig/pkg/antfly/src/api/table_contract.zig @@ -613,6 +613,7 @@ fn validatePublicNestedIndexFields(object: anytype, index_type: public_index_con const graph_shapes = .{ .{ "algebraic_planning", public_index_contract.CreatedObjectShape.graph_algebraic_planning }, + .{ "metrics", public_index_contract.CreatedObjectShape.graph_metrics }, }; inline for (graph_shapes) |field_shape| { const value = if (@hasField(Object, "map")) object.map.get(field_shape[0]) else object.get(field_shape[0]); @@ -647,6 +648,8 @@ fn validatePublicArtifactSources(value: std.json.Value, shape: public_index_cont } fn validatePublicCreatedShape(value: std.json.Value, shape: public_index_contract.CreatedObjectShape) !void { + if (shape == .graph_metric_filter and value == .object and + publicRelationshipFieldActive(value.object, "mode") and publicRelationshipFieldActive(value.object, "types")) return error.InvalidCreateIndexRequest; if (!public_index_contract.createdValueMatchesShape(shape, value)) return error.InvalidCreateIndexRequest; switch (value) { .object => |object| { @@ -1287,6 +1290,31 @@ test "table contract canonicalizes generated optional null fields" { ); } +test "table contract preserves graph metric configuration and rejects malformed nested values" { + const alloc = std.testing.allocator; + const body = + \\{"type":"graph","metrics":{"rank":{"kind":"pagerank","max_iterations":20,"edge_filter":{"mode":null,"types":["selected"]}}}} + ; + const config = try parseCreateIndexRequest(alloc, "graph_idx", body); + defer alloc.free(config); + try ant_json.testing.expectEqualJsonText(alloc, + \\{"name":"graph_idx","type":"graph","metrics":{"rank":{"kind":"pagerank","max_iterations":20,"edge_filter":{"types":["selected"]}}}} + , config); + for ([_][]const u8{ + "{\"metrics\":[]}", + "{\"metrics\":{\"rank\":{\"kind\":\"bogus\"}}}", + "{\"metrics\":{\"rank\":{\"max_iterations\":0}}}", + "{\"metrics\":{\"rank\":{\"damping\":1}}}", + "{\"metrics\":{\"rank\":{\"secret\":true}}}", + "{\"metrics\":{\"rank\":{\"edge_filter\":{\"types\":[]}}}}", + "{\"metrics\":{\"rank\":{\"edge_filter\":{\"mode\":\"all\",\"types\":[\"selected\"]}}}}", + }) |fields| { + const invalid = try std.fmt.allocPrint(alloc, "{{\"type\":\"graph\",{s}", .{fields[1..]}); + defer alloc.free(invalid); + try std.testing.expectError(error.InvalidCreateIndexRequest, parseCreateIndexRequest(alloc, "graph_idx", invalid)); + } +} + test "table contract preserves typed artifact-backed graph configuration" { const body = \\{"type":"graph","source":{"artifact":"relations_v1","path":"$.relations[*]","format":"extraction_relation","mention_edge_type":"mentions","nodes":{"model":"document","target":"{{ _item.target.text }}"},"edge":{"type":"{{ _item.predicate }}","weight":0.75,"metadata":{"source":"{{ _item.source }}"}},"context":{"doc_fields":["title","body"]}},"artifact":{"name":"relations_v1","kind":"asset","source":{"type":"template","value":"{{ body }}"},"content_type":"application/json","producer_json":{"type":"document_extraction","api_key":"write-only"},"execution":{"batch_items":8,"batch_bytes":262144}},"algebraic_planning":{"bounded_traversal":{"law":"provenance_semiring"}},"edge_types":[{"name":"mentions"}],"resolvers":[{"name":"kg","table":"entities","source_artifact":"relations_v1","resolution_artifact":"resolution_v1","key_template":"{{ lower _entity.label }}/{{ slug _entity.text }}","candidate_search":"prefix","config_generation":1}]} diff --git a/zig/pkg/antfly/src/api/table_reads.zig b/zig/pkg/antfly/src/api/table_reads.zig index 4a876671b6..676d3a7dc2 100644 --- a/zig/pkg/antfly/src/api/table_reads.zig +++ b/zig/pkg/antfly/src/api/table_reads.zig @@ -68,12 +68,38 @@ const public_limits = @import("public_limits.zig"); const distributed_graph = @import("distributed_graph.zig"); const runtime_status = @import("runtime_status.zig"); const table_read_source = @import("table_read_source.zig"); +const table_read_graph = @import("table_reads/graph.zig"); +const http_route_helpers = @import("http_route_helpers.zig"); fn earliestDeadline(a: ?u64, b: ?u64) ?u64 { if (a) |left| return if (b) |right| @min(left, right) else left; return b; } -const http_route_helpers = @import("http_route_helpers.zig"); + +const GraphMetricFanInShardRequest = table_read_graph.GraphMetricFanInShardRequest; +const graphSearchQueryNeedsInternalMetricStatus = table_read_graph.graphSearchQueryNeedsInternalMetricStatus; +const rejectNonGlobalGraphMetricFanout = table_read_graph.rejectNonGlobalGraphMetricFanout; +const prepareGraphMetricFanInShardRequest = table_read_graph.prepareGraphMetricFanInShardRequest; + +fn ownedIdentityReadGenerationHeaderForTest( + alloc: std.mem.Allocator, + value: []const u8, +) ![]http_common.Header { + const headers = try alloc.alloc(http_common.Header, 2); + errdefer alloc.free(headers); + const name = try alloc.dupe(u8, query_api.QueryResponse.identity_read_generation_header); + errdefer alloc.free(name); + const owned_value = try alloc.dupe(u8, value); + errdefer alloc.free(owned_value); + headers[0] = .{ .name = name, .value = owned_value }; + const ack_name = try alloc.dupe(u8, metadata_api.catalog_route_fence_ack_header); + errdefer alloc.free(ack_name); + headers[1] = .{ + .name = ack_name, + .value = try alloc.dupe(u8, metadata_api.catalog_route_fence_ack_value), + }; + return headers; +} fn publishRuntimeStatusGroupForTest( cache: *runtime_status.TableRuntimeSnapshotCache, @@ -3879,6 +3905,7 @@ pub const ProvisionedTableReadSource = struct { defer prepared.deinit(); const group_ids = prepared.group_ids; if (group_ids.len == 0) return null; + try rejectNonGlobalGraphMetricFanout(group_ids.len, req); try tableReadsValidateDocIdentityReadyForMultiGroup(alloc, routed.catalog, table_name, group_ids.len); try rejectUnsupportedGraphQueryMode(group_ids.len, req); const start_ns = self.monotonicNs(); @@ -5002,6 +5029,8 @@ pub const HostedProvisionedTableReadSource = struct { graph_read_barrier: ?GraphReadBarrier = null, local_source: ?TableReadSource = null, incoming_graph_routes: ?*distributed_graph.IncomingSourceGroupCache = null, + // Private fixture capability; production rejects non-global metric fanout. + testing_allow_non_global_graph_metric_fanout: bool = false, pub fn init( replica_root_dir: []const u8, @@ -5775,6 +5804,9 @@ pub const HostedProvisionedTableReadSource = struct { defer route_snapshot.deinit(alloc); const group_ids = route_snapshot.group_ids; if (group_ids.len == 0) return null; + if (!(@import("builtin").is_test and self.testing_allow_non_global_graph_metric_fanout)) { + try rejectNonGlobalGraphMetricFanout(group_ids.len, req); + } try tableReadsValidateDocIdentityReadyForMultiGroup(alloc, self.catalog, table_name, group_ids.len); try rejectUnsupportedGraphQueryMode(group_ids.len, req); const start_ns = self.monotonicNs(); @@ -6813,6 +6845,9 @@ fn queryHostedAcrossGroupsParallel( } fn distributedSearchShardLimit(req: db_mod.types.SearchRequest) u32 { + if (req.graph_metric_rerank) |rerank| { + return db_mod.types.graphMetricRerankCandidateCount(rerank, req.offset, req.limit); + } if (req.reranker) |reranker| { if (reranker.candidate_count) |candidate_count| return candidate_count; const output_limit = reranker.top_n orelse req.limit; @@ -6828,9 +6863,9 @@ const DistributedCoordinatorPaging = struct { limit: u32, }; -/// Reranking and distributed pruning are coordinator transforms. Retain the -/// global retrieval window here and apply the caller's offset/final limit only -/// after final-score processing. +/// Provider reranking and distributed pruning are coordinator transforms. +/// Graph-metric reranking is computed against shard-local published vectors, +/// then merged by final score using the caller's page at the coordinator. fn distributedCoordinatorPaging(req: db_mod.types.SearchRequest) DistributedCoordinatorPaging { if (req.reranker != null or req.pruner != null) return .{ .offset = 0, @@ -6861,6 +6896,20 @@ test "distributed reranking widens retrieval and stays coordinator owned" { const coordinator = distributedCoordinatorPaging(req); try std.testing.expectEqual(@as(u32, 0), coordinator.offset); try std.testing.expectEqual(@as(u32, 50), coordinator.limit); + + const graph_req = db_mod.types.SearchRequest{ + .limit = 10, + .offset = 5, + .graph_metric_rerank = .{ .index_name = "graph", .metric_name = "pagerank" }, + }; + const graph_shard = distributedSearchShardRequest(graph_req, &.{}, false); + try std.testing.expectEqual(@as(u32, 45), graph_shard.limit); + try std.testing.expectEqual(@as(u32, 0), graph_shard.offset); + try std.testing.expectEqual(@as(?u32, 45), graph_shard.graph_metric_rerank.?.candidate_count); + + const graph_coordinator = distributedCoordinatorPaging(graph_req); + try std.testing.expectEqual(@as(u32, 5), graph_coordinator.offset); + try std.testing.expectEqual(@as(u32, 10), graph_coordinator.limit); } const complete_match_anchor_order = [_]db_mod.types.SortField{.{ .field = "_id" }}; @@ -7136,6 +7185,11 @@ fn distributedSearchShardRequest( // the global merge. copy.reranker = null; copy.reranker_query_text = ""; + // Graph-metric scoring remains shard-local because its published vector is + // shard-local. Pin the already-expanded global candidate window explicitly + // so each shard neither applies the caller offset nor expands it a second + // time; the coordinator merges final scores and applies the public page. + if (copy.graph_metric_rerank) |*rerank| rerank.candidate_count = copy.limit; // Pruning is score-domain-sensitive. Applying it independently on shards // would produce topology-dependent results and, with a reranker, would use // retrieval scores instead of the provider's final scores. @@ -8723,6 +8777,8 @@ fn graphHydrateOnPreparedDb( alloc, req.incoming_index_name, req.keys, + .{ .generation = req.incoming_index_identity.incarnation, .config_fingerprint = req.incoming_index_identity.config_hash }, + req.identity_read_generation, ) else @constCast((&[_]bool{})[0..]), @@ -9877,11 +9933,13 @@ fn queryProvisionedAcrossGroupsPhase( ) !db_mod.types.SearchResult { try validateDistributedPhaseIdentityGenerations(group_ids.len, required_identity_generations, result_identity_generations); const shard_req = distributedSearchShardRequest(req, distributed_text_stats, expand_selected_groups); + var fan_in_shard_req = try prepareGraphMetricFanInShardRequest(alloc, shard_req); + defer fan_in_shard_req.deinit(alloc); const plan = planQueryFanout(self.io_impl, group_ids.len, req); recordFanoutPlan(.query, plan); if (plan.parallel) { - return try queryProvisionedAcrossGroupsParallel(self, alloc, self.io_impl.?.io(), plan.width, group_ids, &shard_req, req, table_name, consistency, required_identity_generations, result_identity_generations); + return try queryProvisionedAcrossGroupsParallel(self, alloc, self.io_impl.?.io(), plan.width, group_ids, &fan_in_shard_req.req, req, table_name, consistency, required_identity_generations, result_identity_generations); } if (plan.reason == .no_io) recordParallelFanoutFallback(.query); @@ -9898,7 +9956,7 @@ fn queryProvisionedAcrossGroupsPhase( } for (group_ids, 0..) |group_id, i| { - var group_req = shard_req; + var group_req = fan_in_shard_req.req; if (required_identity_generations) |generations| group_req.identity_read_generation = generations[i].?; shard_results[i] = try queryHostedLocal(self.resident_db, self.cache, self.replica_root_dir, self.catalog, self.read_safety_barrier, alloc, group_id, self.visibleRootGeneration(group_id), self.managedReadRuntimeConfig(), table_name, group_req, consistency, self.prepare_for_read != null); initialized += 1; @@ -9934,11 +9992,13 @@ fn queryHostedAcrossGroupsPhase( ) !db_mod.types.SearchResult { try validateDistributedPhaseIdentityGenerations(group_ids.len, required_identity_generations, result_identity_generations); const shard_req = distributedSearchShardRequest(req, distributed_text_stats, expand_selected_groups); + var fan_in_shard_req = try prepareGraphMetricFanInShardRequest(alloc, shard_req); + defer fan_in_shard_req.deinit(alloc); const plan = planQueryFanout(self.io_impl, group_ids.len, req); recordFanoutPlan(.query, plan); if (plan.parallel) { - return try queryHostedAcrossGroupsParallel(self, alloc, self.io_impl.?.io(), plan.width, group_ids, &shard_req, req, table_name, consistency, required_identity_generations, result_identity_generations); + return try queryHostedAcrossGroupsParallel(self, alloc, self.io_impl.?.io(), plan.width, group_ids, &fan_in_shard_req.req, req, table_name, consistency, required_identity_generations, result_identity_generations); } if (plan.reason == .no_io) recordParallelFanoutFallback(.query); @@ -9955,7 +10015,7 @@ fn queryHostedAcrossGroupsPhase( } for (group_ids, 0..) |group_id, i| { - var group_req = shard_req; + var group_req = fan_in_shard_req.req; if (required_identity_generations) |generations| group_req.identity_read_generation = generations[i].?; var route = (try table_router.resolveGroupRoute(alloc, self.catalog, self.router, group_id, routePolicyForConsistency(consistency))) orelse return error.TableNotFound; defer route.deinit(alloc); @@ -11117,9 +11177,10 @@ fn profiledDenseQuery(req: db_mod.types.SearchRequest) ?ProfiledDenseQuery { if (req.full_text_queries.len > 0) return null; if (req.sparse != null or req.sparse_queries.len > 0) return null; if (req.graph_queries.len > 0) return null; + if (req.graph_metric_queries.len > 0 or req.graph_metric_rerank != null) return null; if (req.dense_queries.len > 1) return null; if (req.merge_config != null) return null; - if (req.reranker != null) return null; + if (req.reranker != null or req.pruner != null) return null; if (req.dense_queries.len == 1) { var dense_req = req; dense_req.index_name = req.dense_queries[0].index_name; @@ -11209,6 +11270,8 @@ fn isDenseOnlyQuery(req: db_mod.types.SearchRequest) bool { if (req.filter_text != null or req.exclusion_text != null) return false; if (req.sparse != null or req.sparse_queries.len > 0) return false; if (req.graph_queries.len > 0) return false; + if (req.graph_metric_queries.len > 0 or req.graph_metric_rerank != null) return false; + if (req.reranker != null or req.pruner != null) return false; if (req.filter_query_json.len > 0 or req.exclusion_query_json.len > 0) return false; const query_is_dense_or_neutral = switch (req.query) { @@ -19265,6 +19328,12 @@ fn encodeQueryRequest(alloc: std.mem.Allocator, req: db_mod.types.SearchRequest) if (req.graph_queries.len > 0) { try appendGraphQueriesField(alloc, &out, &first, req.graph_queries, req.graph_query_transport); } + if (req.graph_metric_queries.len > 0) { + try appendGraphMetricQueryField(alloc, &out, &first, req.graph_metric_queries); + } + if (req.graph_metric_rerank) |rerank| { + try appendGraphMetricRerankField(alloc, &out, &first, rerank); + } if (req.expand_strategy) |expand_strategy| { try appendJsonFieldString(alloc, &out, &first, "expand_strategy", switch (expand_strategy) { .@"union" => "union", @@ -19416,6 +19485,76 @@ fn appendGraphQueriesField( try out.appendSlice(alloc, transport.operations_json); } +fn appendGraphMetricQueryField( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged(u8), + first: *bool, + queries: []const db_mod.types.NamedGraphMetricQuery, +) !void { + if (queries.len > 1) { + // The public contract remains the ergonomic singular graph_metric + // request. Internal fan-out needs a lossless envelope for coupled + // metrics such as HITS authority/hub, so encode the bounded admitted + // list explicitly instead of dropping one member. + try appendJsonFieldName(alloc, out, first, "_graph_metric_queries"); + try out.append(alloc, '['); + for (queries, 0..) |named, i| { + if (i > 0) try out.append(alloc, ','); + try out.append(alloc, '{'); + var metric_first = true; + try appendJsonFieldString(alloc, out, &metric_first, "name", named.name); + try appendJsonFieldString(alloc, out, &metric_first, "index", named.query.index_name); + try appendJsonFieldString(alloc, out, &metric_first, "metric", named.query.metric_name); + try appendJsonFieldU32(alloc, out, &metric_first, "top_k", named.query.top_k); + try appendJsonFieldString(alloc, out, &metric_first, "metric_freshness", switch (named.query.freshness) { + .published => "published", + .fresh => "fresh", + }); + try out.append(alloc, '}'); + } + try out.append(alloc, ']'); + return; + } + const named = queries[0]; + + try appendJsonFieldName(alloc, out, first, "graph_metric"); + try out.append(alloc, '{'); + var metric_first = true; + try appendJsonFieldString(alloc, out, &metric_first, "name", named.name); + try appendJsonFieldString(alloc, out, &metric_first, "index", named.query.index_name); + try appendJsonFieldString(alloc, out, &metric_first, "metric", named.query.metric_name); + try appendJsonFieldU32(alloc, out, &metric_first, "top_k", named.query.top_k); + try appendJsonFieldString(alloc, out, &metric_first, "metric_freshness", switch (named.query.freshness) { + .published => "published", + .fresh => "fresh", + }); + try out.append(alloc, '}'); +} + +fn appendGraphMetricRerankField( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged(u8), + first: *bool, + rerank: db_mod.types.GraphMetricRerank, +) !void { + try appendJsonFieldName(alloc, out, first, "graph_metric_rerank"); + try out.append(alloc, '{'); + var rerank_first = true; + try appendJsonFieldString(alloc, out, &rerank_first, "index", rerank.index_name); + try appendJsonFieldString(alloc, out, &rerank_first, "metric", rerank.metric_name); + if (rerank.candidate_count) |candidate_count| { + try appendJsonFieldU32(alloc, out, &rerank_first, "candidate_count", candidate_count); + } + try appendJsonFieldF64(alloc, out, &rerank_first, "base_weight", rerank.base_weight); + try appendJsonFieldF64(alloc, out, &rerank_first, "weight", rerank.weight); + try appendJsonFieldF64(alloc, out, &rerank_first, "missing_score", rerank.missing_score); + try appendJsonFieldString(alloc, out, &rerank_first, "metric_freshness", switch (rerank.freshness) { + .published => "published", + .fresh => "fresh", + }); + try out.append(alloc, '}'); +} + fn appendQueryHierarchyField( alloc: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8), @@ -20320,6 +20459,7 @@ fn parseRemoteSearchResultInner(alloc: std.mem.Allocator, body: []const u8) !db_ var hit: db_mod.types.SearchHit = .{ .id = try alloc.dupe(u8, item._id) }; errdefer hit.deinit(alloc); hit.score = item._score; + hit.score_details = try parseRemoteGraphMetricRerankScoreDetails(alloc, item._score_details); hit.distance = item._distance; hit.index_scores = try parseRemoteIndexScoresAlloc(alloc, item._index_scores); hit.sort_values = try db_mod.types.cloneJsonValues(alloc, item._sort orelse &.{}); @@ -20336,6 +20476,20 @@ fn parseRemoteSearchResultInner(alloc: std.mem.Allocator, body: []const u8) !db_ try parseRemoteGraphResults(alloc, graph_results_value) else @constCast((&[_]db_mod.types.GraphSearchResult{})[0..]); + errdefer { + for (graph_results) |*graph_result| graph_result.deinit(alloc); + if (graph_results.len > 0) alloc.free(graph_results); + } + const graph_metric_results: []db_mod.types.GraphMetricResult = if (response.graph_metric_results) |graph_metric_results_value| + try parseRemoteGraphMetricResults(alloc, graph_metric_results_value) + else + @constCast((&[_]db_mod.types.GraphMetricResult{})[0..]); + errdefer { + for (graph_metric_results) |*metric_result| metric_result.deinit(alloc); + if (graph_metric_results.len > 0) alloc.free(graph_metric_results); + } + var graph_metric_rerank_status = try parseRemoteGraphMetricRerankStatus(alloc, response.profile); + errdefer if (graph_metric_rerank_status) |*status| status.deinit(alloc); return .{ .alloc = alloc, @@ -20343,9 +20497,80 @@ fn parseRemoteSearchResultInner(alloc: std.mem.Allocator, body: []const u8) !db_ .total_hits = total_hits, .total_hits_relation = total_hits_relation, .graph_results = graph_results, + .graph_metric_results = graph_metric_results, + .graph_metric_rerank_status = graph_metric_rerank_status, }; } +fn parseRemoteGraphMetricRerankScoreDetails( + alloc: std.mem.Allocator, + maybe_details: ?metadata_openapi.QueryScoreDetails, +) !?db_mod.types.GraphMetricRerankScoreDetails { + const details = (maybe_details orelse return null).graph_metric_rerank orelse return null; + const metric_score = details.metric_score.valueOrNull(); + if (!std.math.isFinite(details.base_score) or + !std.math.isFinite(details.base_weight) or + (metric_score != null and !std.math.isFinite(metric_score.?)) or + !std.math.isFinite(details.metric_score_used) or + !std.math.isFinite(details.metric_weight) or + !std.math.isFinite(details.final_score) or + details.published_generation < 0) + { + return error.InvalidQueryResponse; + } + const index_name = try alloc.dupe(u8, details.index_name); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, details.metric_name); + errdefer alloc.free(metric_name); + return .{ + .index_name = index_name, + .metric_name = metric_name, + .base_score = details.base_score, + .base_weight = details.base_weight, + .metric_score = metric_score, + .metric_score_used = details.metric_score_used, + .metric_weight = details.metric_weight, + .missing_score_used = details.missing_score_used, + .final_score = details.final_score, + .published_generation = @intCast(details.published_generation), + }; +} + +fn parseRemoteGraphMetricRerankStatus( + alloc: std.mem.Allocator, + maybe_profile: ?std.json.Value, +) !?db_mod.types.GraphMetricStatus { + const profile = maybe_profile orelse return null; + if (profile != .object) return error.InvalidQueryResponse; + const graph_metrics_value = profile.object.get("graph_metrics") orelse return null; + // `graph_metrics` is optional in the public profile contract. The typed + // encoder currently preserves absent optional fields as JSON null, so a + // profiled non-metric shard response must be treated exactly like an + // omitted field rather than poisoning the whole fan-in response. + if (graph_metrics_value == .null) return null; + if (graph_metrics_value != .array) return error.InvalidQueryResponse; + + var result: ?db_mod.types.GraphMetricStatus = null; + errdefer if (result) |*status| status.deinit(alloc); + for (graph_metrics_value.array.items) |item| { + if (item != .object) return error.InvalidQueryResponse; + const source_value = item.object.get("source") orelse return error.InvalidQueryResponse; + if (source_value != .string) return error.InvalidQueryResponse; + if (!std.mem.eql(u8, source_value.string, "graph_metric_rerank")) continue; + if (result != null) return error.InvalidQueryResponse; + + const metric_name_value = item.object.get("metric_name") orelse return error.InvalidQueryResponse; + if (metric_name_value != .string or metric_name_value.string.len == 0) return error.InvalidQueryResponse; + const status_value = item.object.get("status") orelse return error.InvalidQueryResponse; + const encoded = try std.json.Stringify.valueAlloc(alloc, status_value, .{}); + defer alloc.free(encoded); + var parsed = try std.json.parseFromSlice(indexes_openapi.GraphMetricStatus, alloc, encoded, .{}); + defer parsed.deinit(); + result = try parseRemoteGraphMetricStatusValue(alloc, metric_name_value.string, parsed.value); + } + return result; +} + fn parseRemoteHierarchyMatchesAlloc( alloc: std.mem.Allocator, hierarchy: ?metadata_openapi.QueryHitHierarchy, @@ -20473,6 +20698,312 @@ fn parseRemoteIndexScoresAlloc( return trimmed; } +fn parseRemoteGraphMetricResults( + alloc: std.mem.Allocator, + value: std.json.ArrayHashMap(indexes_openapi.GraphMetricResult), +) ![]db_mod.types.GraphMetricResult { + const results = try alloc.alloc(db_mod.types.GraphMetricResult, value.map.count()); + var initialized: usize = 0; + errdefer { + for (results[0..initialized]) |*metric_result| metric_result.deinit(alloc); + alloc.free(results); + } + + var it = value.map.iterator(); + while (it.next()) |entry| { + const result_value = entry.value_ptr.*; + if (entry.key_ptr.*.len == 0 or result_value.index_name.len == 0 or result_value.metric.len == 0) { + return error.InvalidQueryResponse; + } + const scores = try alloc.alloc(db_mod.types.GraphMetricScore, result_value.scores.len); + var initialized_scores: usize = 0; + errdefer { + for (scores[0..initialized_scores]) |*score| score.deinit(alloc); + alloc.free(scores); + } + for (result_value.scores, 0..) |score, i| { + if (score.node.len == 0 or !std.math.isFinite(score.score)) return error.InvalidQueryResponse; + scores[i] = .{ + .node = try alloc.dupe(u8, score.node), + .score = score.score, + }; + initialized_scores += 1; + } + const name = try alloc.dupe(u8, entry.key_ptr.*); + errdefer alloc.free(name); + const index_name = try alloc.dupe(u8, result_value.index_name); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, result_value.metric); + errdefer alloc.free(metric_name); + var status = try parseRemoteGraphMetricStatusValue(alloc, result_value.metric, result_value.status); + errdefer status.deinit(alloc); + results[initialized] = .{ + .name = name, + .index_name = index_name, + .metric_name = metric_name, + .scores = scores, + .status = status, + }; + initialized += 1; + } + + return results; +} + +fn parseRemoteGraphMetricStatusMap( + alloc: std.mem.Allocator, + value: ?std.json.ArrayHashMap(indexes_openapi.GraphMetricStatus), +) ![]db_mod.types.GraphMetricStatus { + const statuses = value orelse return &.{}; + const out = try alloc.alloc(db_mod.types.GraphMetricStatus, statuses.map.count()); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*status| status.deinit(alloc); + if (out.len > 0) alloc.free(out); + } + var it = statuses.map.iterator(); + while (it.next()) |entry| { + if (entry.key_ptr.*.len == 0) return error.InvalidQueryResponse; + out[initialized] = try parseRemoteGraphMetricStatusValue(alloc, entry.key_ptr.*, entry.value_ptr.*); + initialized += 1; + } + return out; +} + +fn parseRemoteGraphMetricStatusValue( + alloc: std.mem.Allocator, + metric_name: []const u8, + status: indexes_openapi.GraphMetricStatus, +) !db_mod.types.GraphMetricStatus { + if (metric_name.len == 0 or + !std.math.isFinite(status.progress) or status.progress < 0 or status.progress > 1 or + !std.math.isFinite(status.delta)) + { + return error.InvalidQueryResponse; + } + const name = try alloc.dupe(u8, metric_name); + errdefer alloc.free(name); + var edge_filter = try parseRemoteGraphMetricEdgeFilterStatus(alloc, status.edge_filter); + errdefer edge_filter.deinit(alloc); + const build_worker_id = if (status.build_worker_id) |worker_id| try alloc.dupe(u8, worker_id) else ""; + errdefer if (build_worker_id.len > 0) alloc.free(build_worker_id); + const build_cursor = if (status.build_cursor) |cursor| try alloc.dupe(u8, cursor) else ""; + errdefer if (build_cursor.len > 0) alloc.free(build_cursor); + const build_pages = try parseRemoteGraphMetricBuildPages(alloc, status.build_pages); + errdefer { + for (build_pages) |*page| page.deinit(alloc); + if (build_pages.len > 0) alloc.free(build_pages); + } + const last_error = if (status.last_error) |message| try alloc.dupe(u8, message) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + const recent_events = try parseRemoteGraphMetricEvents(alloc, status.recent_events); + errdefer if (recent_events.len > 0) alloc.free(recent_events); + + return .{ + .name = name, + .state = graphMetricStateFromName(status.state) orelse return error.InvalidQueryResponse, + .phase = graphMetricPhaseFromName(status.phase) orelse return error.InvalidQueryResponse, + .edge_filter = edge_filter, + .metadata_version = try remoteOptionalU32(status.metadata_version), + .config_fingerprint = try remoteOptionalConfigFingerprint(status.config_fingerprint), + .maintenance_paused = status.maintenance_paused orelse false, + .build_queued = status.build_queued, + .published_generation = try remoteU64(status.published_generation), + .edge_generation = try remoteU64(status.edge_generation), + .target_edge_generation = try remoteU64(status.target_edge_generation), + .queued_generation = try remoteOptionalU64(status.queued_generation), + .building_generation = try remoteOptionalU64(status.building_generation), + .build_job_id = try remoteOptionalU64(status.build_job_id), + .build_started_at_ms = try remoteOptionalU64(status.build_started_at_ms), + .build_iteration = try remoteOptionalU32(status.build_iteration), + .build_lease_expires_at_ms = try remoteOptionalU64(status.build_lease_expires_at_ms), + .build_worker_id = build_worker_id, + .build_cursor = build_cursor, + .build_completed_units = try remoteOptionalU64(status.build_completed_units), + .build_total_units = try remoteOptionalU64(status.build_total_units), + .build_pages = build_pages, + .build_pages_truncated = status.build_pages_truncated orelse false, + .retry_count = try remoteOptionalU64(status.retry_count), + .last_error = last_error, + .progress = status.progress, + .converged = status.converged, + .iterations_completed = try remoteU32(status.iterations_completed), + .delta = status.delta, + .computed_at_ms = try remoteU64(status.computed_at_ms), + .last_event = try parseRemoteGraphMetricEvent(status.last_event), + .recent_events = recent_events, + }; +} + +fn parseRemoteGraphMetricEdgeFilterStatus( + alloc: std.mem.Allocator, + maybe_filter: ?indexes_openapi.GraphMetricEdgeFilterStatus, +) !graph_mod.GraphMetricEdgeFilter { + const filter = maybe_filter orelse return .{}; + if (std.mem.eql(u8, filter.mode, "all")) { + if (filter.types != null and filter.types.?.len > 0) return error.InvalidQueryResponse; + return .{}; + } + if (!std.mem.eql(u8, filter.mode, "types")) return error.InvalidQueryResponse; + const raw_types = filter.types orelse return error.InvalidQueryResponse; + if (raw_types.len == 0) return error.InvalidQueryResponse; + const types = try alloc.alloc([]const u8, raw_types.len); + var initialized: usize = 0; + errdefer { + for (types[0..initialized]) |edge_type| alloc.free(edge_type); + alloc.free(types); + } + for (raw_types, 0..) |edge_type, i| { + if (edge_type.len == 0) return error.InvalidQueryResponse; + types[i] = try alloc.dupe(u8, edge_type); + initialized += 1; + } + return .{ .mode = .types, .types = types }; +} + +fn parseRemoteGraphMetricBuildPages( + alloc: std.mem.Allocator, + maybe_pages: ?[]const indexes_openapi.GraphMetricBuildPageStatus, +) ![]db_mod.types.GraphMetricBuildPageStatus { + const pages = maybe_pages orelse return &.{}; + const out = try alloc.alloc(db_mod.types.GraphMetricBuildPageStatus, pages.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*page| page.deinit(alloc); + if (out.len > 0) alloc.free(out); + } + for (pages, 0..) |page, i| { + const worker_id = if (page.worker_id) |value| try alloc.dupe(u8, value) else ""; + errdefer if (worker_id.len > 0) alloc.free(worker_id); + const cursor = if (page.cursor) |value| try alloc.dupe(u8, value) else ""; + errdefer if (cursor.len > 0) alloc.free(cursor); + const last_error = if (page.last_error) |value| try alloc.dupe(u8, value) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + out[i] = .{ + .phase = graphMetricPhaseFromName(page.phase) orelse return error.InvalidQueryResponse, + .iteration = try remoteU32(page.iteration), + .page_id = try remoteU64(page.page_id), + .state = graphMetricBuildPageStateFromName(page.state) orelse return error.InvalidQueryResponse, + .range_kind = graphMetricBuildPageRangeKindFromName(page.range_kind) orelse return error.InvalidQueryResponse, + .worker_id = worker_id, + .lease_expires_at_ms = try remoteOptionalU64(page.lease_expires_at_ms), + .attempt = try remoteOptionalU64(page.attempt), + .cursor = cursor, + .completed_units = try remoteOptionalU64(page.completed_units), + .total_units = try remoteOptionalU64(page.total_units), + .last_error = last_error, + }; + initialized += 1; + } + return out; +} + +fn parseRemoteGraphMetricEvent( + maybe_event: ?indexes_openapi.GraphMetricEvent, +) !?graph_mod.GraphIndex.GraphMetricEvent { + const event = maybe_event orelse return null; + return try parseRemoteGraphMetricEventValue(event); +} + +fn parseRemoteGraphMetricEventValue( + event: indexes_openapi.GraphMetricEvent, +) !graph_mod.GraphIndex.GraphMetricEvent { + return .{ + .sequence = try remoteU64(event.sequence), + .kind = graphMetricEventKindFromName(event.kind) orelse return error.InvalidQueryResponse, + .at_ms = try remoteU64(event.at_ms), + .target_edge_generation = try remoteU64(event.target_edge_generation), + .published_generation = try remoteU64(event.published_generation), + .score_count = try remoteU64(event.score_count), + }; +} + +fn parseRemoteGraphMetricEvents( + alloc: std.mem.Allocator, + maybe_events: ?[]const indexes_openapi.GraphMetricEvent, +) ![]graph_mod.GraphIndex.GraphMetricEvent { + const events = maybe_events orelse return &.{}; + const out = try alloc.alloc(graph_mod.GraphIndex.GraphMetricEvent, events.len); + errdefer alloc.free(out); + for (events, 0..) |event, i| out[i] = try parseRemoteGraphMetricEventValue(event); + return out; +} + +fn graphMetricEventKindFromName(name: []const u8) ?graph_mod.GraphIndex.GraphMetricEventKind { + if (std.mem.eql(u8, name, "publish")) return .publish; + if (std.mem.eql(u8, name, "delete")) return .delete; + if (std.mem.eql(u8, name, "pause")) return .pause; + if (std.mem.eql(u8, name, "resume")) return .@"resume"; + if (std.mem.eql(u8, name, "failed")) return .failed; + return null; +} + +fn graphMetricStateFromName(name: []const u8) ?graph_mod.GraphIndex.GraphMetricState { + if (std.mem.eql(u8, name, "disabled")) return .disabled; + if (std.mem.eql(u8, name, "not_ready")) return .not_ready; + if (std.mem.eql(u8, name, "fresh")) return .fresh; + if (std.mem.eql(u8, name, "stale")) return .stale; + if (std.mem.eql(u8, name, "building")) return .building; + if (std.mem.eql(u8, name, "failed")) return .failed; + return null; +} + +fn graphMetricPhaseFromName(name: []const u8) ?graph_mod.GraphIndex.GraphMetricBuildPhase { + inline for (@typeInfo(graph_mod.GraphIndex.GraphMetricBuildPhase).@"enum".fields) |field| { + if (std.mem.eql(u8, name, field.name)) return @enumFromInt(field.value); + } + return null; +} + +fn graphMetricBuildPageStateFromName(name: []const u8) ?graph_mod.GraphIndex.GraphMetricBuildPageState { + inline for (@typeInfo(graph_mod.GraphIndex.GraphMetricBuildPageState).@"enum".fields) |field| { + if (std.mem.eql(u8, name, field.name)) return @enumFromInt(field.value); + } + return null; +} + +fn graphMetricBuildPageRangeKindFromName(name: []const u8) ?graph_mod.GraphIndex.GraphMetricBuildPageRangeKind { + inline for (@typeInfo(graph_mod.GraphIndex.GraphMetricBuildPageRangeKind).@"enum".fields) |field| { + if (std.mem.eql(u8, name, field.name)) return @enumFromInt(field.value); + } + return null; +} + +fn remoteU64(value: i64) !u64 { + if (value < 0) return error.InvalidQueryResponse; + return @intCast(value); +} + +fn remoteOptionalU64(value: ?i64) !u64 { + return remoteU64(value orelse 0); +} + +fn remoteOptionalConfigFingerprint(value: ?[]const u8) !u64 { + const encoded = value orelse return 0; + if (encoded.len != 16) return error.InvalidQueryResponse; + for (encoded) |char| { + if (!std.ascii.isDigit(char) and !(char >= 'a' and char <= 'f')) return error.InvalidQueryResponse; + } + return std.fmt.parseInt(u64, encoded, 16) catch error.InvalidQueryResponse; +} + +test "remote graph metric fingerprints require exact lowercase hex" { + try std.testing.expectEqual(std.math.maxInt(u64), try remoteOptionalConfigFingerprint("ffffffffffffffff")); + try std.testing.expectEqual(@as(u64, 0), try remoteOptionalConfigFingerprint(null)); + try std.testing.expectError(error.InvalidQueryResponse, remoteOptionalConfigFingerprint("fffffffffffffff")); + try std.testing.expectError(error.InvalidQueryResponse, remoteOptionalConfigFingerprint("FFFFFFFFFFFFFFFF")); + try std.testing.expectError(error.InvalidQueryResponse, remoteOptionalConfigFingerprint("gggggggggggggggg")); +} + +fn remoteU32(value: i64) !u32 { + if (value < 0 or value > std.math.maxInt(u32)) return error.InvalidQueryResponse; + return @intCast(value); +} + +fn remoteOptionalU32(value: ?i64) !u32 { + return remoteU32(value orelse 0); +} + test "parseRemoteSearchResult preserves fused index scores" { const alloc = std.testing.allocator; var result = try parseRemoteSearchResult(alloc, @@ -20588,6 +21119,7 @@ fn parseRemoteGraphResults( canonical_path_results: ?[]const indexes_openapi.GraphPathResult = null, rows: ?[]const indexes_openapi.GraphResultRow = null, aggregates: ?std.json.ArrayHashMap(indexes_openapi.GraphAggregateValue) = null, + metric_status: ?std.json.ArrayHashMap(indexes_openapi.GraphMetricStatus) = null, truncated: bool = false, }; const view: ResultView = switch (result_value) { @@ -20598,6 +21130,7 @@ fn parseRemoteGraphResults( return error.InvalidRemoteResponse; break :blk .{ .canonical_nodes = result.nodes, + .metric_status = result.metric_status, .truncated = result.stats.truncated, }; }, @@ -20655,6 +21188,8 @@ fn parseRemoteGraphResults( for (aggregates) |*aggregate| aggregate.deinit(alloc); if (aggregates.len > 0) alloc.free(aggregates); } + const metric_status = try parseRemoteGraphMetricStatusMap(alloc, view.metric_status); + errdefer db_mod.types.freeGraphMetricStatuses(alloc, metric_status); const joined_hits = try concatGraphResultHits(alloc, parsed_nodes.hits, parsed_matches.hits); errdefer { @@ -20676,6 +21211,7 @@ fn parseRemoteGraphResults( .aggregates = aggregates, .hits = joined_hits, .total_hits = @intCast(@max(parsed_nodes.nodes.len, @max(paths.len, parsed_matches.matches.len))), + .metric_status = metric_status, .truncated = view.truncated, }; initialized += 1; @@ -20877,6 +21413,11 @@ fn parseRemoteGraphNodeWithKey( errdefer if (path_edges) |value| freeRemoteGraphNodePathEdges(alloc, value); const provenance = if (item.provenance) |value| try cloneRemoteGraphNodePath(alloc, value) else null; errdefer if (provenance) |value| freeRemoteGraphNodePath(alloc, value); + const metrics = try parseRemoteGraphMetricValues(alloc, item.metrics); + errdefer { + for (metrics) |*metric| metric.deinit(alloc); + if (metrics.len > 0) alloc.free(metrics); + } return .{ .key = owned_key, .table = owned_table, @@ -20886,9 +21427,44 @@ fn parseRemoteGraphNodeWithKey( .path_tables = if (owned_path) |value| value.tables else null, .path_edges = path_edges, .provenance = provenance, + .metrics = metrics, }; } +fn parseRemoteGraphMetricValues( + alloc: std.mem.Allocator, + maybe_metrics: ?std.json.ArrayHashMap(std.json.Value), +) ![]graph_query_mod.GraphMetricValue { + const values = maybe_metrics orelse return &.{}; + if (values.map.count() > graph_query_mod.graph_metric_projection_limit) + return error.InvalidRemoteResponse; + const metrics = try alloc.alloc(graph_query_mod.GraphMetricValue, values.map.count()); + var initialized: usize = 0; + errdefer { + for (metrics[0..initialized]) |*metric| metric.deinit(alloc); + if (metrics.len > 0) alloc.free(metrics); + } + var it = values.map.iterator(); + while (it.next()) |entry| { + if (!graph_query_mod.isValidIdentifier(entry.key_ptr.*)) + return error.InvalidRemoteResponse; + const score: ?f64 = switch (entry.value_ptr.*) { + .null => null, + .integer => |value| @floatFromInt(value), + .float => |value| value, + else => return error.InvalidRemoteResponse, + }; + if (score) |value| if (!std.math.isFinite(value)) + return error.InvalidRemoteResponse; + metrics[initialized] = .{ + .name = try alloc.dupe(u8, entry.key_ptr.*), + .score = score, + }; + initialized += 1; + } + return metrics; +} + fn remoteGraphDocumentHit( alloc: std.mem.Allocator, key: []const u8, @@ -31449,3 +32025,2399 @@ test "provisioned storage inspection uses table read admission" { try std.testing.expectEqual(@as(usize, 2), tracker.begins); try std.testing.expectEqual(@as(usize, 2), tracker.ends); } + +test "graph metric shard request carries internal status without mutating public request" { + const alloc = std.testing.allocator; + const start_keys = [_][]const u8{"doc:a"}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "related", + .query = .{ + .query_type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &start_keys }, + .metrics = &.{.{ .name = "pagerank", .freshness = .fresh }}, + }, + }}; + const public_req = db_mod.types.SearchRequest{ + .graph_queries = &graph_queries, + .graph_metric_rerank = .{ .index_name = "graph_idx", .metric_name = "pagerank" }, + }; + + var shard = try prepareGraphMetricFanInShardRequest(alloc, public_req); + defer shard.deinit(alloc); + try std.testing.expect(!public_req.graph_queries[0].query.include_metric_status); + try std.testing.expect(!public_req.profile); + try std.testing.expect(shard.req.graph_queries[0].query.include_metric_status); + try std.testing.expect(shard.req.profile); +} + +test "encode query request includes graph metric read rerank and traversal status" { + const alloc = std.testing.allocator; + const graph_operations = + \\{"related":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]},"metrics":["pagerank"],"order_by":[{"metric":"pagerank"}],"where_metric":[{"metric":"pagerank","op":"gte","value":0.25}],"metric_freshness":"fresh","include_metric_status":true}}} + ; + const start_keys = [_][]const u8{"doc:a"}; + const graph_queries = [_]db_mod.types.NamedGraphQuery{.{ + .name = "related", + .query = .{ + .query_type = .traverse, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &start_keys }, + .metrics = &.{.{ .name = "pagerank", .freshness = .fresh }}, + .order_by = &.{.{ .name = "pagerank", .freshness = .fresh }}, + .where_metric = &.{.{ .name = "pagerank", .op = .gte, .value = 0.25, .freshness = .fresh }}, + .include_metric_status = true, + }, + }}; + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "central", + .query = .{ .index_name = "graph_idx", .metric_name = "pagerank", .top_k = 25, .freshness = .fresh }, + }}; + const encoded = try encodeQueryRequest(alloc, .{ + .full_text = .{ .match_all = {} }, + .graph_queries = &graph_queries, + .graph_query_transport = .{ + .dialect = .canonical, + .operations_json = graph_operations, + .admitted_operations_ptr = @ptrCast(graph_queries[0..].ptr), + .admitted_operations_len = graph_queries.len, + }, + .graph_metric_queries = &graph_metric_queries, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .fresh, + .base_weight = 0.5, + .weight = 2.5, + .missing_score = -0.25, + }, + .limit = 25, + }); + defer alloc.free(encoded); + + var parsed = try parseJsonTestBody(std.json.Value, alloc, encoded); + defer parsed.deinit(); + const graph_metric = parsed.value.object.get("graph_metric").?.object; + try std.testing.expectEqualStrings("central", graph_metric.get("name").?.string); + try std.testing.expectEqualStrings("fresh", graph_metric.get("metric_freshness").?.string); + const rerank = parsed.value.object.get("graph_metric_rerank").?.object; + try std.testing.expectEqual(@as(f64, 2.5), rerank.get("weight").?.float); + const traversal = parsed.value.object.get("graph_queries").?.object.get("related").?.object.get("traverse").?.object; + try std.testing.expect(traversal.get("include_metric_status").?.bool); + try std.testing.expectEqualStrings("pagerank", traversal.get("metrics").?.array.items[0].string); + try std.testing.expectEqualStrings("gte", traversal.get("where_metric").?.array.items[0].object.get("op").?.string); +} + +test "remote query parser preserves graph metric fan-in provenance and durable status" { + const alloc = std.testing.allocator; + var parsed = try parseRemoteSearchResult(alloc, + \\{"responses":[{"hits":{"total":{"value":1,"relation":"exact"},"hits":[{"_id":"doc:a","_score":2.25,"_score_details":{"graph_metric_rerank":{"index_name":"graph_idx","metric_name":"pagerank","base_score":1,"base_weight":0.5,"metric_score":0.7,"metric_score_used":0.7,"metric_weight":2.5,"missing_score_used":false,"final_score":2.25,"published_generation":7}}}]},"graph_results":{"related":{"nodes":[{"key":"doc:a","depth":0,"metrics":{"pagerank":0.7,"degree":null}}],"metric_status":{"pagerank":{"state":"fresh","phase":"complete","edge_filter":{"mode":"types","types":["references"]},"metadata_version":2,"config_fingerprint":"ffffffffffffffff","maintenance_paused":false,"build_queued":false,"published_generation":7,"edge_generation":7,"target_edge_generation":7,"queued_generation":0,"building_generation":0,"build_job_id":12345,"build_started_at_ms":1780000000123,"build_iteration":2,"build_lease_expires_at_ms":0,"build_worker_id":"worker-a","build_cursor":"edge:42","build_completed_units":42,"build_total_units":100,"build_pages":[{"phase":"scan_edges_and_out_degree","iteration":2,"page_id":9,"state":"leased","range_kind":"reverse_edges","worker_id":"worker-a","lease_expires_at_ms":1780000001123,"attempt":1,"cursor":"edge:42","completed_units":42,"total_units":100}],"build_pages_truncated":false,"retry_count":0,"progress":1,"converged":true,"iterations_completed":12,"delta":0,"computed_at_ms":1780000000000,"last_event":{"sequence":3,"kind":"publish","at_ms":1780000000000,"target_edge_generation":7,"published_generation":7,"score_count":100}}},"kind":"nodes","stats":{"returned_items":1,"truncated":false}}},"graph_metric_results":{"central":{"index_name":"graph_idx","metric":"pagerank","scores":[{"node":"doc:a","score":0.7}],"status":{"state":"fresh","phase":"complete","build_queued":false,"published_generation":7,"edge_generation":7,"target_edge_generation":7,"progress":1,"converged":true,"iterations_completed":12,"delta":0,"computed_at_ms":1780000000000}}},"profile":{"graph_metrics":[{"query_name":"graph_metric_rerank","source":"graph_metric_rerank","index_name":"graph_idx","metric_name":"pagerank","freshness":"published","status":{"state":"fresh","phase":"complete","build_queued":false,"published_generation":7,"edge_generation":7,"target_edge_generation":7,"progress":1,"converged":true,"iterations_completed":12,"delta":0,"computed_at_ms":1780000000000}}]},"took":1,"status":200}]} + ); + defer parsed.deinit(); + + try std.testing.expectEqual(@as(usize, 1), parsed.hits.len); + try std.testing.expectEqual(@as(u64, 7), parsed.hits[0].score_details.?.published_generation); + try std.testing.expectEqual(@as(usize, 1), parsed.graph_metric_results.len); + try std.testing.expectEqualStrings("central", parsed.graph_metric_results[0].name); + try std.testing.expectEqual(@as(usize, 1), parsed.graph_results.len); + try std.testing.expectEqual(@as(usize, 2), parsed.graph_results[0].nodes[0].metrics.len); + try std.testing.expect(parsed.graph_results[0].nodes[0].metrics[1].score == null); + const status = parsed.graph_results[0].metric_status[0]; + try std.testing.expectEqual(@as(u32, 2), status.metadata_version); + try std.testing.expectEqual(std.math.maxInt(u64), status.config_fingerprint); + try std.testing.expectEqual(@as(usize, 1), status.build_pages.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPageState.leased, status.build_pages[0].state); + try std.testing.expectEqualStrings("edge:42", status.build_cursor); + try std.testing.expect(parsed.graph_metric_rerank_status != null); + try std.testing.expectEqual(@as(u64, 7), parsed.graph_metric_rerank_status.?.published_generation); +} + +test "remote query parser rejects invalid graph metric status and duplicate rerank profiles" { + const alloc = std.testing.allocator; + try std.testing.expectError(error.InvalidRemoteResponse, parseRemoteSearchResult(alloc, + \\{"responses":[{"hits":{"total":{"value":0,"relation":"exact"},"hits":[]},"graph_metric_results":{"central":{"index_name":"graph_idx","metric":"pagerank","scores":[],"status":{"state":"fresh","phase":"complete","build_queued":false,"published_generation":-1,"edge_generation":7,"target_edge_generation":7,"progress":1.0,"converged":true,"iterations_completed":1,"delta":0.0,"computed_at_ms":1}}},"took":0,"status":200}]} + )); + try std.testing.expectError(error.InvalidRemoteResponse, parseRemoteSearchResult(alloc, + \\{"responses":[{"hits":{"total":{"value":0,"relation":"exact"},"hits":[]},"profile":{"graph_metrics":[{"source":"graph_metric_rerank","metric_name":"pagerank","status":{"state":"fresh","phase":"complete","build_queued":false,"published_generation":7,"edge_generation":7,"target_edge_generation":7,"progress":1.0,"converged":true,"iterations_completed":1,"delta":0.0,"computed_at_ms":1}},{"source":"graph_metric_rerank","metric_name":"pagerank","status":{"state":"fresh","phase":"complete","build_queued":false,"published_generation":7,"edge_generation":7,"target_edge_generation":7,"progress":1.0,"converged":true,"iterations_completed":1,"delta":0.0,"computed_at_ms":1}}]},"took":0,"status":200}]} + )); +} + +test "remote query parser accepts nullable graph metrics in ordinary profiles" { + const alloc = std.testing.allocator; + var parsed = try parseRemoteSearchResult(alloc, + \\{"responses":[{"hits":{"total":{"value":0,"relation":"exact"},"hits":[]},"profile":{"shards":{"total":1,"successful":1,"failed":0},"graph_metrics":null},"took":0,"status":200}]} + ); + defer parsed.deinit(); + + try std.testing.expectEqual(@as(usize, 0), parsed.hits.len); + try std.testing.expect(parsed.graph_metric_rerank_status == null); +} + +// Ported hosted fan-in coverage from the combined graph-metrics branch. +const GraphMetricJsonTestSurface = union(enum) { + direct: []const u8, + traversal: []const u8, + rerank, +}; + +fn jsonQuotedTestAlloc(alloc: std.mem.Allocator, value: []const u8) ![]u8 { + var out: std.Io.Writer.Allocating = .init(alloc); + defer out.deinit(); + try std.json.Stringify.value(value, .{}, &out.writer); + return try alloc.dupe(u8, out.written()); +} + +fn jsonValueContainsString(value: std.json.Value, expected: []const u8) bool { + return switch (value) { + .string => |actual| std.mem.eql(u8, actual, expected), + .array => |array| for (array.items) |item| { + if (jsonValueContainsString(item, expected)) break true; + } else false, + .object => |object| blk: { + var entries = object.iterator(); + while (entries.next()) |entry| { + if (jsonValueContainsString(entry.value_ptr.*, expected)) break :blk true; + } + break :blk false; + }, + else => false, + }; +} + +fn expectJsonStringPresence(alloc: std.mem.Allocator, actual_json: []const u8, expected: []const u8, present: bool) !void { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, actual_json, .{}); + defer parsed.deinit(); + try std.testing.expectEqual(present, jsonValueContainsString(parsed.value, expected)); +} + +fn legacyGraphQueryTransportForTest(queries: []const db_mod.types.NamedGraphQuery) db_mod.types.GraphQueryTransport { + return .{ + .dialect = .legacy, + .operations_json = "{}", + .admitted_operations_ptr = @ptrCast(queries.ptr), + .admitted_operations_len = queries.len, + }; +} + +fn expectGraphMetricJsonStatus( + alloc: std.mem.Allocator, + actual_json: []const u8, + surface: GraphMetricJsonTestSurface, + metric_name: []const u8, + state: []const u8, + published_generation: ?u64, + building_generation: ?u64, +) !void { + const metric_json = try jsonQuotedTestAlloc(alloc, metric_name); + defer alloc.free(metric_json); + const state_json = try jsonQuotedTestAlloc(alloc, state); + defer alloc.free(state_json); + var status_out: std.Io.Writer.Allocating = .init(alloc); + defer status_out.deinit(); + try status_out.writer.writeAll("{\"state\":"); + try status_out.writer.writeAll(state_json); + if (published_generation) |published| { + try status_out.writer.print(",\"published_generation\":{d}", .{published}); + } + if (building_generation) |building| { + try status_out.writer.print(",\"building_generation\":{d}", .{building}); + } + try status_out.writer.writeByte('}'); + + var expected_out: std.Io.Writer.Allocating = .init(alloc); + defer expected_out.deinit(); + switch (surface) { + .direct => |result_name| { + const result_json = try jsonQuotedTestAlloc(alloc, result_name); + defer alloc.free(result_json); + try expected_out.writer.writeAll("{\"responses\":[{\"graph_metric_results\":{"); + try expected_out.writer.writeAll(result_json); + try expected_out.writer.writeAll(":{\"metric\":"); + try expected_out.writer.writeAll(metric_json); + try expected_out.writer.writeAll(",\"status\":"); + try expected_out.writer.writeAll(status_out.written()); + try expected_out.writer.writeAll("}}}]}"); + }, + .traversal => |result_name| { + const result_json = try jsonQuotedTestAlloc(alloc, result_name); + defer alloc.free(result_json); + try expected_out.writer.writeAll("{\"responses\":[{\"graph_results\":{"); + try expected_out.writer.writeAll(result_json); + try expected_out.writer.writeAll(":{\"metric_status\":{"); + try expected_out.writer.writeAll(metric_json); + try expected_out.writer.writeByte(':'); + try expected_out.writer.writeAll(status_out.written()); + try expected_out.writer.writeAll("}}}}]}"); + }, + .rerank => { + try expected_out.writer.writeAll("{\"responses\":[{\"profile\":{\"graph_metrics\":[{\"source\":\"graph_metric_rerank\",\"metric_name\":"); + try expected_out.writer.writeAll(metric_json); + try expected_out.writer.writeAll(",\"status\":"); + try expected_out.writer.writeAll(status_out.written()); + try expected_out.writer.writeAll("}]}}]}"); + }, + } + try ant_json.testing.expectSubsetJsonText(alloc, expected_out.written(), actual_json); +} + +test "hosted cross-range graph metric fan-in merges compatible published shard generations" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-merge", .{tmp.sub_path}); + defer alloc.free(path); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const left_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7101); + defer alloc.free(left_path); + const right_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7102); + defer alloc.free(right_path); + + const graph_indexes_json = + \\{"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + const graph_config_json = + \\{"edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}} + ; + var left_db = try db_mod.DB.open(alloc, left_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7101, .range_id = 7101 }, + }); + defer left_db.close(); + try left_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try left_db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"left-a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"left-b\"}" }, + }, + .sync_level = .write, + }); + try left_db.runUntilIdle(); + var left_status = try left_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer left_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, left_status.state); + + var right_db = try db_mod.DB.open(alloc, right_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7102, .range_id = 7102 }, + }); + defer right_db.close(); + try right_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try right_db.batch(.{ + .writes = &.{ + .{ .key = "doc:n", .value = "{\"title\":\"right-n\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:o\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:o", .value = "{\"title\":\"right-o\"}" }, + }, + .sync_level = .write, + }); + try right_db.runUntilIdle(); + var right_status = try right_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer right_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, right_status.state); + try std.testing.expectEqual(left_status.published_generation, right_status.published_generation); + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ + .group_id = 7101, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7101, + .namespace_range_id = 7101, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + .{ + .group_id = 7102, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7102, + .namespace_range_id = 7102, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7101, .table_id = 7, .range_id = 7101, .start_key = "", .end_key = "m" }, + .{ .group_id = 7102, .table_id = 7, .range_id = 7102, .start_key = "m", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .active; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 1; + } + + fn nodeStatus(_: *anyopaque, _: u64, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn nodeBaseUri(_: *anyopaque, _: std.mem.Allocator, _: u64) !?[]u8 { + return null; + } + }; + + const ExecutorState = struct { + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(_: *anyopaque, _: std.mem.Allocator, _: http_common.HttpRequest) !http_common.HttpResponse { + return error.UnexpectedHttpRequest; + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + const metric_request = db_mod.types.SearchRequest{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ .index_name = "graph_idx", .metric_name = "manual_degree", .top_k = 4 }, + }}, + .limit = 0, + }; + try std.testing.expectError(error.GraphMetricGlobalMaterializationRequired, hosted.source().query(alloc, "docs", metric_request, .read_index)); + var provisioned = ProvisionedTableReadSource.init(path, FakeCatalog.iface(), raft_mod.read_gate.alreadyReadSafeBarrier()); + try std.testing.expectError(error.GraphMetricGlobalMaterializationRequired, provisioned.source().query(alloc, "docs", metric_request, .read_index)); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + var response = (try hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 4, + .freshness = .published, + }, + }}, + .limit = 0, + }, .read_index)).?; + defer response.deinit(alloc); + + try expectGraphMetricJsonStatus(alloc, response.json, .{ .direct = "central" }, "manual_degree", "fresh", null, null); + try expectJsonStringPresence(alloc, response.json, "doc:b", true); + try expectJsonStringPresence(alloc, response.json, "doc:o", true); +} + +test "hosted cross-range graph metric fan-in merges active stale shard for published" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-active-stale", .{tmp.sub_path}); + defer alloc.free(path); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const left_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7341); + defer alloc.free(left_path); + const right_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7342); + defer alloc.free(right_path); + + const graph_indexes_json = + \\{"ft_v1":{"type":"full_text","store":true},"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + const graph_config_json = + \\{"edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}} + ; + + var left_db = try db_mod.DB.open(alloc, left_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7341, .range_id = 7341 }, + }); + var left_db_open = true; + defer if (left_db_open) left_db.close(); + try left_db.addIndex(.{ .name = "ft_v1", .kind = .full_text, .config_json = "{\"store\":true}" }); + try left_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try left_db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"left-a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"left-b\"}" }, + }, + .sync_level = .full_index, + }); + try left_db.runUntilIdle(); + var left_status = try left_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer left_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, left_status.state); + var left_pagerank_status = try left_db.refreshGraphMetric(alloc, "graph_idx", "pagerank"); + defer left_pagerank_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, left_pagerank_status.state); + try std.testing.expectEqual(left_status.published_generation, left_pagerank_status.published_generation); + var left_eigenvector_status = try left_db.refreshGraphMetric(alloc, "graph_idx", "eigenvector"); + defer left_eigenvector_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, left_eigenvector_status.state); + try std.testing.expectEqual(left_status.published_generation, left_eigenvector_status.published_generation); + + var right_db = try db_mod.DB.open(alloc, right_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7342, .range_id = 7342 }, + }); + var right_db_open = true; + defer if (right_db_open) right_db.close(); + try right_db.addIndex(.{ .name = "ft_v1", .kind = .full_text, .config_json = "{\"store\":true}" }); + try right_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try right_db.batch(.{ + .writes = &.{ + .{ .key = "doc:n", .value = "{\"title\":\"right-n\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:o\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:o", .value = "{\"title\":\"right-o\"}" }, + }, + .sync_level = .full_index, + }); + try right_db.runUntilIdle(); + var right_status = try right_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer right_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, right_status.state); + try std.testing.expectEqual(left_status.published_generation, right_status.published_generation); + var right_pagerank_status = try right_db.refreshGraphMetric(alloc, "graph_idx", "pagerank"); + defer right_pagerank_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, right_pagerank_status.state); + try std.testing.expectEqual(left_status.published_generation, right_pagerank_status.published_generation); + var right_eigenvector_status = try right_db.refreshGraphMetric(alloc, "graph_idx", "eigenvector"); + defer right_eigenvector_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, right_eigenvector_status.state); + try std.testing.expectEqual(left_status.published_generation, right_eigenvector_status.published_generation); + + try right_db.batch(.{ + .writes = &.{.{ .key = "doc:p", .value = "{\"title\":\"right-p\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:o\",\"weight\":1.0}]}}}" }}, + .sync_level = .full_index, + }); + try right_db.runUntilIdle(); + const active_target_generation = blk: { + const graph_entry = right_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + const active_metrics = [_][]const u8{ "manual_degree", "pagerank", "eigenvector" }; + for (active_metrics) |metric_name| { + var building = try graph_entry.index.ensureGraphMetricPlannedBuild(metric_name, target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + } + break :blk target_generation; + }; + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ + .group_id = 7341, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7341, + .namespace_range_id = 7341, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + .{ + .group_id = 7342, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7342, + .namespace_range_id = 7342, + .next_ordinal = 4, + .allocated_ordinals = 3, + .state_rows = 3, + .live_ordinals = 3, + .complete = true, + }, + }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7341, .table_id = 7, .range_id = 7341, .start_key = "", .end_key = "doc:m" }, + .{ .group_id = 7342, .table_id = 7, .range_id = 7342, .start_key = "doc:m", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .active; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 1; + } + + fn nodeStatus(_: *anyopaque, _: u64, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn nodeBaseUri(_: *anyopaque, _: std.mem.Allocator, _: u64) !?[]u8 { + return null; + } + }; + + const ExecutorState = struct { + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(_: *anyopaque, _: std.mem.Allocator, _: http_common.HttpRequest) !http_common.HttpResponse { + return error.UnexpectedHttpRequest; + } + }; + + left_db.close(); + left_db_open = false; + right_db.close(); + right_db_open = false; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + const active_metrics = [_][]const u8{ "manual_degree", "pagerank", "eigenvector" }; + for (active_metrics) |metric_name| { + var response = (try hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 8, + .freshness = .published, + }, + }}, + .limit = 0, + }, .read_index)).?; + defer response.deinit(alloc); + + try expectGraphMetricJsonStatus(alloc, response.json, .{ .direct = "central" }, metric_name, "building", right_status.published_generation, active_target_generation); + try expectJsonStringPresence(alloc, response.json, "doc:b", true); + try expectJsonStringPresence(alloc, response.json, "doc:o", true); + try expectJsonStringPresence(alloc, response.json, "doc:p", false); + + var rerank_response = (try hosted.source().query(alloc, "docs", .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .freshness = .published, + .weight = 1.0, + }, + .limit = 5, + .include_stored = false, + .profile = true, + }, .read_index)).?; + defer rerank_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, rerank_response.json, .rerank, metric_name, "building", right_status.published_generation, active_target_generation); + try expectJsonStringPresence(alloc, rerank_response.json, "doc:b", true); + try expectJsonStringPresence(alloc, rerank_response.json, "doc:o", true); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = metric_name, + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{ "doc:a", "doc:n" } }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_results = 8 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + const published_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "neighbors", .query = published_graph_query }}; + var traversal_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &published_graph_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&published_graph_queries), + }, .read_index)).?; + defer traversal_response.deinit(alloc); + + try expectGraphMetricJsonStatus(alloc, traversal_response.json, .{ .traversal = "neighbors" }, metric_name, "building", right_status.published_generation, active_target_generation); + try expectJsonStringPresence(alloc, traversal_response.json, "doc:b", true); + try expectJsonStringPresence(alloc, traversal_response.json, "doc:o", true); + try expectJsonStringPresence(alloc, traversal_response.json, "doc:p", false); + + const published_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = metric_name, + .freshness = .published, + }}; + var order_query = published_graph_query; + order_query.order_by = &published_metric_orders; + const order_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "ordered", .query = order_query }}; + var order_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &order_graph_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&order_graph_queries), + }, .read_index)).?; + defer order_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, order_response.json, .{ .traversal = "ordered" }, metric_name, "building", right_status.published_generation, active_target_generation); + + const published_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = metric_name, + .op = .gte, + .value = 0.0, + .freshness = .published, + }}; + var filter_query = published_graph_query; + filter_query.where_metric = &published_metric_filters; + const filter_graph_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "filtered", .query = filter_query }}; + var filter_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &filter_graph_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&filter_graph_queries), + }, .read_index)).?; + defer filter_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, filter_response.json, .{ .traversal = "filtered" }, metric_name, "building", right_status.published_generation, active_target_generation); + try expectJsonStringPresence(alloc, filter_response.json, "doc:b", true); + try expectJsonStringPresence(alloc, filter_response.json, "doc:o", true); + + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 8, + .freshness = .fresh, + }, + }}, + .limit = 0, + }, .read_index)); + + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 5, + .include_stored = false, + }, .read_index)); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = metric_name, + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_neighbors", .query = fresh_projection_query }}, + }, .read_index)); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = metric_name, + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_ordered", .query = fresh_order_query }}, + }, .read_index)); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = metric_name, + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_filtered", .query = fresh_filter_query }}, + }, .read_index)); + } +} + +test "hosted cross-range graph metric fan-in merges nonuniform promotion shard layout" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-promotion-merge", .{tmp.sub_path}); + defer alloc.free(path); + const shard_count = 8; + const group_ids = [_]u64{ 7301, 7302, 7303, 7304, 7305, 7306, 7307, 7308 }; + const prefixes = [_][]const u8{ "a", "b", "c", "d", "e", "f", "g", "h" }; + const source_counts = [_]usize{ 1, 2, 3, 1, 2, 3, 1, 2 }; + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const graph_indexes_json = + \\{"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + const graph_config_json = + \\{"edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}},"pagerank":{"enabled":true,"kind":"pagerank","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"eigenvector":{"enabled":true,"kind":"eigenvector","refresh":"manual","max_iterations":2,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}} + ; + const metric_names = [_][]const u8{ "manual_degree", "pagerank", "eigenvector" }; + + var db_paths: [shard_count][]u8 = undefined; + var db_path_count: usize = 0; + defer { + for (db_paths[0..db_path_count]) |db_path| alloc.free(db_path); + } + var dbs: [shard_count]db_mod.DB = undefined; + var db_count: usize = 0; + defer { + for (dbs[0..db_count]) |*db| db.close(); + } + + var published_generation: u64 = 0; + for (group_ids, prefixes, source_counts, 0..) |group_id, prefix, source_count, shard_index| { + db_paths[shard_index] = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, group_id); + db_path_count += 1; + dbs[shard_index] = try db_mod.DB.open(alloc, db_paths[shard_index], .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = group_id, .range_id = group_id }, + }); + db_count += 1; + try dbs[shard_index].addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + + var writes: [4]db_mod.types.BatchWrite = undefined; + var write_count: usize = 0; + var owned: [8][]u8 = undefined; + var owned_count: usize = 0; + defer { + for (owned[0..owned_count]) |item| alloc.free(item); + } + + const sink_key = try std.fmt.allocPrint(alloc, "doc:{s}:target", .{prefix}); + owned[owned_count] = sink_key; + owned_count += 1; + const sink_value = try std.fmt.allocPrint(alloc, "{{\"title\":\"target {s}\"}}", .{prefix}); + owned[owned_count] = sink_value; + owned_count += 1; + writes[write_count] = .{ .key = sink_key, .value = sink_value }; + write_count += 1; + + for (0..source_count) |source_index| { + const source_key = try std.fmt.allocPrint(alloc, "doc:{s}:source:{d}", .{ prefix, source_index }); + owned[owned_count] = source_key; + owned_count += 1; + const source_value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {s}-{d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"{s}\",\"weight\":1.0}}]}}}}}}", + .{ prefix, source_index, sink_key }, + ); + owned[owned_count] = source_value; + owned_count += 1; + writes[write_count] = .{ .key = source_key, .value = source_value }; + write_count += 1; + } + + try dbs[shard_index].batch(.{ + .writes = writes[0..write_count], + .sync_level = .write, + }); + try dbs[shard_index].runUntilIdle(); + for (metric_names) |metric_name| { + var status = try dbs[shard_index].refreshGraphMetric(alloc, "graph_idx", metric_name); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + if (published_generation == 0) { + published_generation = status.published_generation; + } else { + try std.testing.expectEqual(published_generation, status.published_generation); + } + } + } + + const active_shard_indices = [_]usize{ 1, 3, 5, 7 }; + for (active_shard_indices) |shard_index| { + const prefix = prefixes[shard_index]; + const group_id = group_ids[shard_index]; + + var writes: [4]db_mod.types.BatchWrite = undefined; + var owned: [8][]u8 = undefined; + var owned_count: usize = 0; + defer { + for (owned[0..owned_count]) |item| alloc.free(item); + } + + const active_target_key = try std.fmt.allocPrint(alloc, "doc:{s}:active-target", .{prefix}); + owned[owned_count] = active_target_key; + owned_count += 1; + const active_target_value = try std.fmt.allocPrint(alloc, "{{\"title\":\"active target {s}\"}}", .{prefix}); + owned[owned_count] = active_target_value; + owned_count += 1; + writes[0] = .{ .key = active_target_key, .value = active_target_value }; + + for (0..3) |source_index| { + const source_key = try std.fmt.allocPrint(alloc, "doc:{s}:active-source:{d}", .{ prefix, source_index }); + owned[owned_count] = source_key; + owned_count += 1; + const source_value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"active source {s}-{d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"{s}\",\"weight\":1.0}}]}}}}}}", + .{ prefix, source_index, active_target_key }, + ); + owned[owned_count] = source_value; + owned_count += 1; + writes[source_index + 1] = .{ .key = source_key, .value = source_value }; + } + + try dbs[shard_index].batch(.{ + .writes = writes[0..], + .sync_level = .full_index, + }); + try dbs[shard_index].runUntilIdle(); + + const graph_entry = dbs[shard_index].core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + try std.testing.expect(target_generation > published_generation); + for (metric_names) |metric_name| { + var building = try graph_entry.index.ensureGraphMetricPlannedBuild(metric_name, target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + } + + _ = group_id; + } + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ .group_id = 7301, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7301, .namespace_range_id = 7301, .next_ordinal = 3, .allocated_ordinals = 2, .state_rows = 2, .live_ordinals = 2, .complete = true } }, + .{ .group_id = 7302, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7302, .namespace_range_id = 7302, .next_ordinal = 8, .allocated_ordinals = 7, .state_rows = 7, .live_ordinals = 7, .complete = true } }, + .{ .group_id = 7303, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7303, .namespace_range_id = 7303, .next_ordinal = 5, .allocated_ordinals = 4, .state_rows = 4, .live_ordinals = 4, .complete = true } }, + .{ .group_id = 7304, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7304, .namespace_range_id = 7304, .next_ordinal = 7, .allocated_ordinals = 6, .state_rows = 6, .live_ordinals = 6, .complete = true } }, + .{ .group_id = 7305, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7305, .namespace_range_id = 7305, .next_ordinal = 4, .allocated_ordinals = 3, .state_rows = 3, .live_ordinals = 3, .complete = true } }, + .{ .group_id = 7306, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7306, .namespace_range_id = 7306, .next_ordinal = 9, .allocated_ordinals = 8, .state_rows = 8, .live_ordinals = 8, .complete = true } }, + .{ .group_id = 7307, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7307, .namespace_range_id = 7307, .next_ordinal = 3, .allocated_ordinals = 2, .state_rows = 2, .live_ordinals = 2, .complete = true } }, + .{ .group_id = 7308, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7308, .namespace_range_id = 7308, .next_ordinal = 8, .allocated_ordinals = 7, .state_rows = 7, .live_ordinals = 7, .complete = true } }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7301, .table_id = 7, .range_id = 7301, .start_key = "", .end_key = "doc:b:" }, + .{ .group_id = 7302, .table_id = 7, .range_id = 7302, .start_key = "doc:b:", .end_key = "doc:c:" }, + .{ .group_id = 7303, .table_id = 7, .range_id = 7303, .start_key = "doc:c:", .end_key = "doc:d:" }, + .{ .group_id = 7304, .table_id = 7, .range_id = 7304, .start_key = "doc:d:", .end_key = "doc:e:" }, + .{ .group_id = 7305, .table_id = 7, .range_id = 7305, .start_key = "doc:e:", .end_key = "doc:f:" }, + .{ .group_id = 7306, .table_id = 7, .range_id = 7306, .start_key = "doc:f:", .end_key = "doc:g:" }, + .{ .group_id = 7307, .table_id = 7, .range_id = 7307, .start_key = "doc:g:", .end_key = "doc:h:" }, + .{ .group_id = 7308, .table_id = 7, .range_id = 7308, .start_key = "doc:h:", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .active; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 1; + } + + fn nodeStatus(_: *anyopaque, _: u64, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn nodeBaseUri(_: *anyopaque, _: std.mem.Allocator, _: u64) !?[]u8 { + return null; + } + }; + + const ExecutorState = struct { + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(_: *anyopaque, _: std.mem.Allocator, _: http_common.HttpRequest) !http_common.HttpResponse { + return error.UnexpectedHttpRequest; + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + for (metric_names) |metric_name| { + var response = (try hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 32, + .freshness = .published, + }, + }}, + .limit = 0, + }, .read_index)).?; + defer response.deinit(alloc); + + try expectGraphMetricJsonStatus(alloc, response.json, .{ .direct = "central" }, metric_name, "building", published_generation, null); + for (prefixes) |prefix| { + const needle = try std.fmt.allocPrint(alloc, "doc:{s}:target", .{prefix}); + defer alloc.free(needle); + try expectJsonStringPresence(alloc, response.json, needle, true); + } + for (active_shard_indices) |shard_index| { + const active_needle = try std.fmt.allocPrint(alloc, "doc:{s}:active-target", .{prefixes[shard_index]}); + defer alloc.free(active_needle); + try expectJsonStringPresence(alloc, response.json, active_needle, false); + } + + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 32, + .freshness = .fresh, + }, + }}, + .limit = 0, + }, .read_index)); + } +} + +test "hosted cross-range graph metric fan-in merges compatible hits pair" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-hits-pair", .{tmp.sub_path}); + defer alloc.free(path); + const shard_count = 8; + const group_ids = [_]u64{ 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318 }; + const prefixes = [_][]const u8{ "j", "k", "l", "m", "n", "o", "p", "q" }; + const hub_counts = [_]usize{ 1, 2, 3, 2, 1, 3, 2, 1 }; + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const graph_indexes_json = + \\{"ft_v1":{"type":"full_text","store":true},"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"hits_authority":{"enabled":true,"kind":"hits_authority","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"hits_hub":{"enabled":true,"kind":"hits_hub","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + const graph_config_json = + \\{"edge_types":[{"name":"cites"}],"metrics":{"hits_authority":{"enabled":true,"kind":"hits_authority","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"hits_hub":{"enabled":true,"kind":"hits_hub","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}} + ; + + var db_paths: [shard_count][]u8 = undefined; + var db_path_count: usize = 0; + defer { + for (db_paths[0..db_path_count]) |db_path| alloc.free(db_path); + } + var dbs: [shard_count]db_mod.DB = undefined; + var db_count: usize = 0; + defer { + for (dbs[0..db_count]) |*db| db.close(); + } + + var published_generation: u64 = 0; + for (group_ids, prefixes, hub_counts, 0..) |group_id, prefix, hub_count, shard_index| { + db_paths[shard_index] = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, group_id); + db_path_count += 1; + dbs[shard_index] = try db_mod.DB.open(alloc, db_paths[shard_index], .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = group_id, .range_id = group_id }, + }); + db_count += 1; + try dbs[shard_index].addIndex(.{ .name = "ft_v1", .kind = .full_text, .config_json = "{\"store\":true}" }); + try dbs[shard_index].addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + + var writes: [4]db_mod.types.BatchWrite = undefined; + var write_count: usize = 0; + var owned: [8][]u8 = undefined; + var owned_count: usize = 0; + defer { + for (owned[0..owned_count]) |item| alloc.free(item); + } + + const authority_key = try std.fmt.allocPrint(alloc, "doc:{s}:authority", .{prefix}); + owned[owned_count] = authority_key; + owned_count += 1; + const authority_value = try std.fmt.allocPrint(alloc, "{{\"title\":\"authority {s}\"}}", .{prefix}); + owned[owned_count] = authority_value; + owned_count += 1; + writes[write_count] = .{ .key = authority_key, .value = authority_value }; + write_count += 1; + + for (0..hub_count) |hub_index| { + const hub_key = try std.fmt.allocPrint(alloc, "doc:{s}:hub:{d}", .{ prefix, hub_index }); + owned[owned_count] = hub_key; + owned_count += 1; + const hub_value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"hub {s}-{d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"{s}\",\"weight\":1.0}}]}}}}}}", + .{ prefix, hub_index, authority_key }, + ); + owned[owned_count] = hub_value; + owned_count += 1; + writes[write_count] = .{ .key = hub_key, .value = hub_value }; + write_count += 1; + } + + try dbs[shard_index].batch(.{ + .writes = writes[0..write_count], + .sync_level = .full_index, + }); + try dbs[shard_index].runUntilIdle(); + var authority_status = try dbs[shard_index].refreshGraphMetric(alloc, "graph_idx", "hits_authority"); + defer authority_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, authority_status.state); + var hub_status = try (dbs[shard_index].core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expectEqual(authority_status.published_generation, hub_status.published_generation); + if (published_generation == 0) { + published_generation = authority_status.published_generation; + } else { + try std.testing.expectEqual(published_generation, authority_status.published_generation); + } + } + + var active_target_generation: u64 = 0; + const active_shard_indices = [_]usize{ 1, 3, 5, 7 }; + for (active_shard_indices) |shard_index| { + const prefix = prefixes[shard_index]; + + var writes: [4]db_mod.types.BatchWrite = undefined; + var owned: [8][]u8 = undefined; + var owned_count: usize = 0; + defer { + for (owned[0..owned_count]) |item| alloc.free(item); + } + + const active_authority_key = try std.fmt.allocPrint(alloc, "doc:{s}:active-authority", .{prefix}); + owned[owned_count] = active_authority_key; + owned_count += 1; + const active_authority_value = try std.fmt.allocPrint(alloc, "{{\"title\":\"active authority {s}\"}}", .{prefix}); + owned[owned_count] = active_authority_value; + owned_count += 1; + writes[0] = .{ .key = active_authority_key, .value = active_authority_value }; + + for (0..3) |hub_index| { + const active_hub_key = try std.fmt.allocPrint(alloc, "doc:{s}:active-hub:{d}", .{ prefix, hub_index }); + owned[owned_count] = active_hub_key; + owned_count += 1; + const active_hub_value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"active hub {s}-{d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"{s}\",\"weight\":1.0}}]}}}}}}", + .{ prefix, hub_index, active_authority_key }, + ); + owned[owned_count] = active_hub_value; + owned_count += 1; + writes[hub_index + 1] = .{ .key = active_hub_key, .value = active_hub_value }; + } + + try dbs[shard_index].batch(.{ + .writes = writes[0..], + .sync_level = .full_index, + }); + try dbs[shard_index].runUntilIdle(); + + const graph_entry = dbs[shard_index].core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + try std.testing.expect(target_generation > published_generation); + if (active_target_generation == 0) { + active_target_generation = target_generation; + } else { + try std.testing.expectEqual(active_target_generation, target_generation); + } + + var building = try graph_entry.index.ensureGraphMetricPlannedBuild("hits_authority", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + } + + for (dbs[0..db_count]) |*db| db.close(); + db_count = 0; + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ .group_id = 7311, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7311, .namespace_range_id = 7311, .next_ordinal = 3, .allocated_ordinals = 2, .state_rows = 2, .live_ordinals = 2, .complete = true } }, + .{ .group_id = 7312, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7312, .namespace_range_id = 7312, .next_ordinal = 8, .allocated_ordinals = 7, .state_rows = 7, .live_ordinals = 7, .complete = true } }, + .{ .group_id = 7313, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7313, .namespace_range_id = 7313, .next_ordinal = 5, .allocated_ordinals = 4, .state_rows = 4, .live_ordinals = 4, .complete = true } }, + .{ .group_id = 7314, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7314, .namespace_range_id = 7314, .next_ordinal = 8, .allocated_ordinals = 7, .state_rows = 7, .live_ordinals = 7, .complete = true } }, + .{ .group_id = 7315, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7315, .namespace_range_id = 7315, .next_ordinal = 3, .allocated_ordinals = 2, .state_rows = 2, .live_ordinals = 2, .complete = true } }, + .{ .group_id = 7316, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7316, .namespace_range_id = 7316, .next_ordinal = 9, .allocated_ordinals = 8, .state_rows = 8, .live_ordinals = 8, .complete = true } }, + .{ .group_id = 7317, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7317, .namespace_range_id = 7317, .next_ordinal = 4, .allocated_ordinals = 3, .state_rows = 3, .live_ordinals = 3, .complete = true } }, + .{ .group_id = 7318, .doc_identity = .{ .namespace_table_id = 7, .namespace_shard_id = 7318, .namespace_range_id = 7318, .next_ordinal = 7, .allocated_ordinals = 6, .state_rows = 6, .live_ordinals = 6, .complete = true } }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7311, .table_id = 7, .range_id = 7311, .start_key = "", .end_key = "doc:k:" }, + .{ .group_id = 7312, .table_id = 7, .range_id = 7312, .start_key = "doc:k:", .end_key = "doc:l:" }, + .{ .group_id = 7313, .table_id = 7, .range_id = 7313, .start_key = "doc:l:", .end_key = "doc:m:" }, + .{ .group_id = 7314, .table_id = 7, .range_id = 7314, .start_key = "doc:m:", .end_key = "doc:n:" }, + .{ .group_id = 7315, .table_id = 7, .range_id = 7315, .start_key = "doc:n:", .end_key = "doc:o:" }, + .{ .group_id = 7316, .table_id = 7, .range_id = 7316, .start_key = "doc:o:", .end_key = "doc:p:" }, + .{ .group_id = 7317, .table_id = 7, .range_id = 7317, .start_key = "doc:p:", .end_key = "doc:q:" }, + .{ .group_id = 7318, .table_id = 7, .range_id = 7318, .start_key = "doc:q:", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .active; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 1; + } + + fn nodeStatus(_: *anyopaque, _: u64, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn nodeBaseUri(_: *anyopaque, _: std.mem.Allocator, _: u64) !?[]u8 { + return null; + } + }; + + const ExecutorState = struct { + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(_: *anyopaque, _: std.mem.Allocator, _: http_common.HttpRequest) !http_common.HttpResponse { + return error.UnexpectedHttpRequest; + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + var response = (try hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 16, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 16, + .freshness = .published, + }, + }, + }, + .limit = 0, + }, .read_index)).?; + defer response.deinit(alloc); + + try expectGraphMetricJsonStatus(alloc, response.json, .{ .direct = "authority" }, "hits_authority", "building", published_generation, active_target_generation); + // HITS authority and hub are one atomic lifecycle pair. Both surfaces must + // report the shared in-flight generation even when only the authority + // metric was used to start the build. + try expectGraphMetricJsonStatus(alloc, response.json, .{ .direct = "hub" }, "hits_hub", "building", published_generation, active_target_generation); + for (prefixes) |prefix| { + const authority_needle = try std.fmt.allocPrint(alloc, "doc:{s}:authority", .{prefix}); + defer alloc.free(authority_needle); + try expectJsonStringPresence(alloc, response.json, authority_needle, true); + const hub_needle = try std.fmt.allocPrint(alloc, "doc:{s}:hub:0", .{prefix}); + defer alloc.free(hub_needle); + try expectJsonStringPresence(alloc, response.json, hub_needle, true); + } + for (active_shard_indices) |shard_index| { + const active_authority_needle = try std.fmt.allocPrint(alloc, "doc:{s}:active-authority", .{prefixes[shard_index]}); + defer alloc.free(active_authority_needle); + try expectJsonStringPresence(alloc, response.json, active_authority_needle, false); + const active_hub_needle = try std.fmt.allocPrint(alloc, "doc:{s}:active-hub:0", .{prefixes[shard_index]}); + defer alloc.free(active_hub_needle); + try expectJsonStringPresence(alloc, response.json, active_hub_needle, false); + } + + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 16, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 16, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }, .read_index)); + + var rerank_response = (try hosted.source().query(alloc, "docs", .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .published, + .weight = 1.0, + }, + .limit = 32, + .include_stored = false, + .profile = true, + }, .read_index)).?; + defer rerank_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, rerank_response.json, .rerank, "hits_authority", "building", published_generation, active_target_generation); + + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 32, + .include_stored = false, + }, .read_index)); + + const hits_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .published }, + .{ .name = "hits_hub", .freshness = .published }, + }; + const traversal_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{ "doc:j:hub:0", "doc:k:hub:0", "doc:l:hub:0", "doc:m:hub:0", "doc:n:hub:0", "doc:o:hub:0", "doc:p:hub:0", "doc:q:hub:0" } }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_results = 16 }, + .metrics = &hits_metric_reads, + .include_metric_status = true, + }; + const traversal_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "hits_neighbors", .query = traversal_query }}; + var traversal_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &traversal_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&traversal_queries), + }, .read_index)).?; + defer traversal_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, traversal_response.json, .{ .traversal = "hits_neighbors" }, "hits_authority", "building", published_generation, active_target_generation); + try expectGraphMetricJsonStatus(alloc, traversal_response.json, .{ .traversal = "hits_neighbors" }, "hits_hub", "building", published_generation, active_target_generation); + for (prefixes) |prefix| { + const authority_needle = try std.fmt.allocPrint(alloc, "doc:{s}:authority", .{prefix}); + defer alloc.free(authority_needle); + try expectJsonStringPresence(alloc, traversal_response.json, authority_needle, true); + } + for (active_shard_indices) |shard_index| { + const active_authority_needle = try std.fmt.allocPrint(alloc, "doc:{s}:active-authority", .{prefixes[shard_index]}); + defer alloc.free(active_authority_needle); + try expectJsonStringPresence(alloc, traversal_response.json, active_authority_needle, false); + } + + const hits_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .published, + }}; + var ordered_traversal_query = traversal_query; + ordered_traversal_query.order_by = &hits_metric_orders; + const ordered_traversal_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "ordered_hits_neighbors", .query = ordered_traversal_query }}; + var ordered_traversal_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &ordered_traversal_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&ordered_traversal_queries), + }, .read_index)).?; + defer ordered_traversal_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, ordered_traversal_response.json, .{ .traversal = "ordered_hits_neighbors" }, "hits_authority", "building", published_generation, active_target_generation); + try expectGraphMetricJsonStatus(alloc, ordered_traversal_response.json, .{ .traversal = "ordered_hits_neighbors" }, "hits_hub", "building", published_generation, active_target_generation); + + const hits_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.0, + .freshness = .published, + }}; + var filtered_traversal_query = traversal_query; + filtered_traversal_query.where_metric = &hits_metric_filters; + const filtered_traversal_queries = [_]db_mod.types.NamedGraphQuery{.{ .name = "filtered_hits_neighbors", .query = filtered_traversal_query }}; + var filtered_traversal_response = (try hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &filtered_traversal_queries, + .graph_query_transport = legacyGraphQueryTransportForTest(&filtered_traversal_queries), + }, .read_index)).?; + defer filtered_traversal_response.deinit(alloc); + try expectGraphMetricJsonStatus(alloc, filtered_traversal_response.json, .{ .traversal = "filtered_hits_neighbors" }, "hits_authority", "building", published_generation, active_target_generation); + try expectGraphMetricJsonStatus(alloc, filtered_traversal_response.json, .{ .traversal = "filtered_hits_neighbors" }, "hits_hub", "building", published_generation, active_target_generation); + + const fresh_hits_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .fresh }, + .{ .name = "hits_hub", .freshness = .fresh }, + }; + var fresh_traversal_query = traversal_query; + fresh_traversal_query.metrics = &fresh_hits_metric_reads; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_hits_neighbors", .query = fresh_traversal_query }}, + }, .read_index)); + + const fresh_hits_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_ordered_traversal_query = traversal_query; + fresh_ordered_traversal_query.order_by = &fresh_hits_metric_orders; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_ordered_hits_neighbors", .query = fresh_ordered_traversal_query }}, + }, .read_index)); + + const fresh_hits_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filtered_traversal_query = traversal_query; + fresh_filtered_traversal_query.where_metric = &fresh_hits_metric_filters; + try std.testing.expectError(error.MetricStale, hosted.source().query(alloc, "docs", .{ + .query = .{ .match_all = {} }, + .limit = 0, + .graph_queries = &.{.{ .name = "fresh_filtered_hits_neighbors", .query = fresh_filtered_traversal_query }}, + }, .read_index)); +} + +test "hosted cross-range graph metric fan-in rejects incompatible remote hits pair" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-hits-pair-reject", .{tmp.sub_path}); + defer alloc.free(path); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const graph_indexes_json = + \\{"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"hits_authority":{"enabled":true,"kind":"hits_authority","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"hits_hub":{"enabled":true,"kind":"hits_hub","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ + .group_id = 7321, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7321, + .namespace_range_id = 7321, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + .{ + .group_id = 7322, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7322, + .namespace_range_id = 7322, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7321, .table_id = 7, .range_id = 7321, .start_key = "", .end_key = "doc:r:" }, + .{ .group_id = 7322, .table_id = 7, .range_id = 7322, .start_key = "doc:r:", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 2; + } + + fn nodeStatus(_: *anyopaque, node_id: u64, _: u64) raft_mod.HostedReplicaStatus { + return if (node_id == 2) .active else .absent; + } + + fn nodeBaseUri(_: *anyopaque, alloc_inner: std.mem.Allocator, node_id: u64) !?[]u8 { + if (node_id != 2) return null; + return try alloc_inner.dupe(u8, "http://remote.test"); + } + }; + + const ExecutorState = struct { + const Scenario = enum { + generation_mismatch, + metadata_mismatch, + edge_filter_mismatch, + }; + + scenario: Scenario = .generation_mismatch, + query_calls: std.atomic.Value(usize) = .init(0), + + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(ptr: *anyopaque, alloc_inner: std.mem.Allocator, req: http_common.HttpRequest) !http_common.HttpResponse { + const self: *@This() = @ptrCast(@alignCast(ptr)); + try std.testing.expectEqual(http_common.Method.POST, req.method); + _ = self.query_calls.fetchAdd(1, .monotonic); + if (std.mem.endsWith(u8, req.uri, "/internal/v1/groups/7321/tables/docs/query")) { + return .{ + .status = 200, + .headers = try ownedIdentityReadGenerationHeaderForTest(alloc_inner, "1"), + .body = try remoteHitsPairBody(alloc_inner, "q", 11, 11, 1, "cites"), + }; + } + if (std.mem.endsWith(u8, req.uri, "/internal/v1/groups/7322/tables/docs/query")) { + const authority_generation: u64 = 11; + const hub_generation: u64 = if (self.scenario == .generation_mismatch) 12 else 11; + const metadata_version: u32 = if (self.scenario == .metadata_mismatch) 2 else 1; + const edge_type: []const u8 = if (self.scenario == .edge_filter_mismatch) "mentions" else "cites"; + return .{ + .status = 200, + .headers = try ownedIdentityReadGenerationHeaderForTest(alloc_inner, "1"), + .body = try remoteHitsPairBody(alloc_inner, "r", authority_generation, hub_generation, metadata_version, edge_type), + }; + } + return error.UnexpectedHttpRequest; + } + + fn remoteHitsPairBody( + alloc_inner: std.mem.Allocator, + prefix: []const u8, + authority_generation: u64, + hub_generation: u64, + metadata_version: u32, + edge_type: []const u8, + ) ![]u8 { + return try std.fmt.allocPrint( + alloc_inner, + "{{\"responses\":[{{\"hits\":{{\"total\":{{\"value\":0,\"relation\":\"exact\"}},\"hits\":[]}},\"graph_metric_results\":{{\"authority\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_authority\",\"scores\":[{{\"node\":\"doc:{s}:authority\",\"score\":1.0}}],\"status\":{{\"state\":\"fresh\",\"phase\":\"complete\",\"maintenance_paused\":false,\"build_queued\":false,\"published_generation\":{d},\"edge_generation\":{d},\"target_edge_generation\":{d},\"queued_generation\":0,\"building_generation\":0,\"metadata_version\":{d},\"edge_filter\":{{\"mode\":\"types\",\"types\":[\"{s}\"]}},\"progress\":1.0,\"converged\":true,\"iterations_completed\":1,\"delta\":0.0,\"computed_at_ms\":1780000000000}}}},\"hub\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_hub\",\"scores\":[{{\"node\":\"doc:{s}:hub\",\"score\":1.0}}],\"status\":{{\"state\":\"fresh\",\"phase\":\"complete\",\"maintenance_paused\":false,\"build_queued\":false,\"published_generation\":{d},\"edge_generation\":{d},\"target_edge_generation\":{d},\"queued_generation\":0,\"building_generation\":0,\"metadata_version\":{d},\"edge_filter\":{{\"mode\":\"types\",\"types\":[\"{s}\"]}},\"progress\":1.0,\"converged\":true,\"iterations_completed\":1,\"delta\":0.0,\"computed_at_ms\":1780000000000}}}}}},\"took\":0,\"status\":200,\"table\":\"docs\"}}]}}", + .{ + prefix, + authority_generation, + authority_generation, + authority_generation, + metadata_version, + edge_type, + prefix, + hub_generation, + hub_generation, + hub_generation, + metadata_version, + edge_type, + }, + ); + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + try std.testing.expectError(error.UnsupportedQueryRequest, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 8, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 8, + .freshness = .published, + }, + }, + }, + .limit = 0, + }, .read_index)); + try std.testing.expectEqual(@as(usize, 2), executor_state.query_calls.load(.monotonic)); + + executor_state.scenario = .metadata_mismatch; + executor_state.query_calls.store(0, .monotonic); + try std.testing.expectError(error.UnsupportedQueryRequest, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 8, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 8, + .freshness = .published, + }, + }, + }, + .limit = 0, + }, .read_index)); + try std.testing.expectEqual(@as(usize, 2), executor_state.query_calls.load(.monotonic)); + + executor_state.scenario = .edge_filter_mismatch; + executor_state.query_calls.store(0, .monotonic); + try std.testing.expectError(error.UnsupportedQueryRequest, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 8, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 8, + .freshness = .published, + }, + }, + }, + .limit = 0, + }, .read_index)); + try std.testing.expectEqual(@as(usize, 2), executor_state.query_calls.load(.monotonic)); +} + +test "hosted cross-range graph metric fan-in rejects missing remote hits status" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-hits-missing-status", .{tmp.sub_path}); + defer alloc.free(path); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const graph_indexes_json = + \\{"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"hits_authority":{"enabled":true,"kind":"hits_authority","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}},"hits_hub":{"enabled":true,"kind":"hits_hub","refresh":"manual","max_iterations":1,"tolerance":0.000001,"edge_filter":{"types":["cites"]}}}}} + ; + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ + .group_id = 7331, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7331, + .namespace_range_id = 7331, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + .{ + .group_id = 7332, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7332, + .namespace_range_id = 7332, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7331, .table_id = 7, .range_id = 7331, .start_key = "", .end_key = "doc:t:" }, + .{ .group_id = 7332, .table_id = 7, .range_id = 7332, .start_key = "doc:t:", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 2; + } + + fn nodeStatus(_: *anyopaque, node_id: u64, _: u64) raft_mod.HostedReplicaStatus { + return if (node_id == 2) .active else .absent; + } + + fn nodeBaseUri(_: *anyopaque, alloc_inner: std.mem.Allocator, node_id: u64) !?[]u8 { + if (node_id != 2) return null; + return try alloc_inner.dupe(u8, "http://remote.test"); + } + }; + + const ExecutorState = struct { + query_calls: std.atomic.Value(usize) = .init(0), + + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(ptr: *anyopaque, alloc_inner: std.mem.Allocator, req: http_common.HttpRequest) !http_common.HttpResponse { + const self: *@This() = @ptrCast(@alignCast(ptr)); + try std.testing.expectEqual(http_common.Method.POST, req.method); + _ = self.query_calls.fetchAdd(1, .monotonic); + if (std.mem.endsWith(u8, req.uri, "/internal/v1/groups/7331/tables/docs/query")) { + return .{ + .status = 200, + .headers = try ownedIdentityReadGenerationHeaderForTest(alloc_inner, "1"), + .body = try remoteHitsPairBody(alloc_inner, "s", 21), + }; + } + if (std.mem.endsWith(u8, req.uri, "/internal/v1/groups/7332/tables/docs/query")) { + return .{ + .status = 200, + .headers = try ownedIdentityReadGenerationHeaderForTest(alloc_inner, "1"), + .body = try remoteHitsMissingHubStatusBody(alloc_inner, "t", 21), + }; + } + return error.UnexpectedHttpRequest; + } + + fn remoteHitsPairBody( + alloc_inner: std.mem.Allocator, + prefix: []const u8, + generation: u64, + ) ![]u8 { + return try std.fmt.allocPrint( + alloc_inner, + "{{\"responses\":[{{\"hits\":{{\"total\":{{\"value\":0,\"relation\":\"exact\"}},\"hits\":[]}},\"graph_metric_results\":{{\"authority\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_authority\",\"scores\":[{{\"node\":\"doc:{s}:authority\",\"score\":1.0}}],\"status\":{{\"state\":\"fresh\",\"phase\":\"complete\",\"maintenance_paused\":false,\"build_queued\":false,\"published_generation\":{d},\"edge_generation\":{d},\"target_edge_generation\":{d},\"queued_generation\":0,\"building_generation\":0,\"progress\":1.0,\"converged\":true,\"iterations_completed\":1,\"delta\":0.0,\"computed_at_ms\":1780000000000}}}},\"hub\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_hub\",\"scores\":[{{\"node\":\"doc:{s}:hub\",\"score\":1.0}}],\"status\":{{\"state\":\"fresh\",\"phase\":\"complete\",\"maintenance_paused\":false,\"build_queued\":false,\"published_generation\":{d},\"edge_generation\":{d},\"target_edge_generation\":{d},\"queued_generation\":0,\"building_generation\":0,\"progress\":1.0,\"converged\":true,\"iterations_completed\":1,\"delta\":0.0,\"computed_at_ms\":1780000000000}}}}}},\"took\":0,\"status\":200,\"table\":\"docs\"}}]}}", + .{ prefix, generation, generation, generation, prefix, generation, generation, generation }, + ); + } + + fn remoteHitsMissingHubStatusBody( + alloc_inner: std.mem.Allocator, + prefix: []const u8, + generation: u64, + ) ![]u8 { + return try std.fmt.allocPrint( + alloc_inner, + "{{\"responses\":[{{\"hits\":{{\"total\":{{\"value\":0,\"relation\":\"exact\"}},\"hits\":[]}},\"graph_metric_results\":{{\"authority\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_authority\",\"scores\":[{{\"node\":\"doc:{s}:authority\",\"score\":1.0}}],\"status\":{{\"state\":\"fresh\",\"phase\":\"complete\",\"maintenance_paused\":false,\"build_queued\":false,\"published_generation\":{d},\"edge_generation\":{d},\"target_edge_generation\":{d},\"queued_generation\":0,\"building_generation\":0,\"progress\":1.0,\"converged\":true,\"iterations_completed\":1,\"delta\":0.0,\"computed_at_ms\":1780000000000}}}},\"hub\":{{\"index_name\":\"graph_idx\",\"metric\":\"hits_hub\",\"scores\":[{{\"node\":\"doc:{s}:hub\",\"score\":1.0}}]}}}},\"took\":0,\"status\":200,\"table\":\"docs\"}}]}}", + .{ prefix, generation, generation, generation, prefix }, + ); + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + try std.testing.expectError(error.InvalidRemoteResponse, hosted.source().query(alloc, "docs", .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 8, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 8, + .freshness = .published, + }, + }, + }, + .limit = 0, + }, .read_index)); + try std.testing.expectEqual(@as(usize, 2), executor_state.query_calls.load(.monotonic)); +} + +test "hosted cross-range graph metric fan-in rejects unpublished or incompatible shard generations" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/hosted-cross-range-graph-metric-reject", .{tmp.sub_path}); + defer alloc.free(path); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + defer std.Io.Dir.cwd().deleteTree(io_impl.io(), path) catch {}; + + const left_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7201); + defer alloc.free(left_path); + const right_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7202); + defer alloc.free(right_path); + + const graph_indexes_json = + \\{"graph_idx":{"type":"graph","edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}}}}} + ; + const graph_config_json = + \\{"edge_types":[{"name":"cites"}],"metrics":{"manual_degree":{"enabled":true,"kind":"degree","refresh":"manual","edge_filter":{"types":["cites"]}}}} + ; + + var left_db = try db_mod.DB.open(alloc, left_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7201, .range_id = 7201 }, + }); + defer left_db.close(); + try left_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try left_db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"left-a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"left-b\"}" }, + }, + .sync_level = .write, + }); + try left_db.runUntilIdle(); + var left_status = try left_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer left_status.deinit(alloc); + + var right_db = try db_mod.DB.open(alloc, right_path, .{ + .start_index_workers = false, + .identity_namespace = .{ .table_id = 7, .shard_id = 7202, .range_id = 7202 }, + }); + defer right_db.close(); + try right_db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = graph_config_json }); + try right_db.batch(.{ + .writes = &.{ + .{ .key = "doc:n", .value = "{\"title\":\"right-n\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:o\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:o", .value = "{\"title\":\"right-o\"}" }, + }, + .sync_level = .write, + }); + try right_db.runUntilIdle(); + + const FakeCatalog = struct { + const statuses = [_]metadata_reconciler.MergedGroupStatus{ + .{ + .group_id = 7201, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7201, + .namespace_range_id = 7201, + .next_ordinal = 3, + .allocated_ordinals = 2, + .state_rows = 2, + .live_ordinals = 2, + .complete = true, + }, + }, + .{ + .group_id = 7202, + .doc_identity = .{ + .namespace_table_id = 7, + .namespace_shard_id = 7202, + .namespace_range_id = 7202, + .next_ordinal = 4, + .allocated_ordinals = 3, + .state_rows = 3, + .live_ordinals = 3, + .complete = true, + }, + }, + }; + + fn iface() table_catalog.CatalogSource { + const Routing = table_catalog.TestAdminRoutingAdapter(adminSnapshot, freeAdminSnapshot); + return .{ + .ptr = undefined, + .vtable = &.{ + .admin_snapshot = adminSnapshot, + .free_admin_snapshot = freeAdminSnapshot, + .routing_snapshot = Routing.routingSnapshot, + .linearizable_routing_snapshot = Routing.linearizableSnapshot, + .free_routing_snapshot = Routing.freeRoutingSnapshot, + }, + }; + } + + fn adminSnapshot(_: *anyopaque) !metadata_api.AdminSnapshot { + return .{ + .status = .{ .metadata_group_id = 1, .metrics = .{} }, + .tables = @constCast((&[_]metadata_table_manager.TableRecord{.{ + .table_id = 7, + .name = "docs", + .placement_role = "data", + .indexes_json = graph_indexes_json, + }})[0..]), + .ranges = @constCast((&[_]metadata_table_manager.RangeRecord{ + .{ .group_id = 7201, .table_id = 7, .range_id = 7201, .start_key = "", .end_key = "m" }, + .{ .group_id = 7202, .table_id = 7, .range_id = 7202, .start_key = "m", .end_key = null }, + })[0..]), + .stores = @constCast((&[_]metadata_table_manager.StoreRecord{})[0..]), + .placement_intents = @constCast((&[_]raft_reconciler.PlacementIntent{})[0..]), + .split_transitions = @constCast((&[_]metadata_transition_state.SplitTransitionRecord{})[0..]), + .merge_transitions = @constCast((&[_]metadata_transition_state.MergeTransitionRecord{})[0..]), + .merged_group_statuses = @constCast(statuses[0..]), + }; + } + + fn freeAdminSnapshot(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + }; + + const FakeRouter = struct { + fn iface() table_router.HostedGroupRouter { + return .{ + .ptr = undefined, + .vtable = &.{ + .local_node_id = localNodeId, + .local_status = localStatus, + .group_leader_node_id = groupLeaderNodeId, + .node_status = nodeStatus, + .node_base_uri = nodeBaseUri, + }, + }; + } + + fn localNodeId(_: *anyopaque) u64 { + return 1; + } + + fn localStatus(_: *anyopaque, _: u64) raft_mod.HostedReplicaStatus { + return .active; + } + + fn groupLeaderNodeId(_: *anyopaque, _: u64) ?u64 { + return 1; + } + + fn nodeStatus(_: *anyopaque, _: u64, _: u64) raft_mod.HostedReplicaStatus { + return .absent; + } + + fn nodeBaseUri(_: *anyopaque, _: std.mem.Allocator, _: u64) !?[]u8 { + return null; + } + }; + + const ExecutorState = struct { + fn iface(self: *@This()) http_common.RequestExecutor { + return .{ .ptr = self, .vtable = &.{ .execute = execute } }; + } + + fn execute(_: *anyopaque, _: std.mem.Allocator, _: http_common.HttpRequest) !http_common.HttpResponse { + return error.UnexpectedHttpRequest; + } + }; + + var executor_state = ExecutorState{}; + var hosted = HostedProvisionedTableReadSource.init( + path, + FakeCatalog.iface(), + raft_mod.read_gate.alreadyReadSafeBarrier(), + FakeRouter.iface(), + executor_state.iface(), + ); + _ = hosted.withIo(&io_impl); + hosted.testing_allow_non_global_graph_metric_fanout = true; + + const metric_req = db_mod.types.SearchRequest{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 4, + .freshness = .published, + }, + }}, + .limit = 0, + }; + + try std.testing.expectError(error.MetricNotReady, hosted.source().query(alloc, "docs", metric_req, .read_index)); + + var right_first = try right_db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer right_first.deinit(alloc); + try std.testing.expectEqual(left_status.published_generation, right_first.published_generation); + try right_db.batch(.{ + .writes = &.{.{ .key = "doc:p", .value = "{\"title\":\"right-p\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:o\",\"weight\":1.0}]}}}" }}, + .sync_level = .write, + }); + try right_db.runUntilIdle(); + var right_second = try right_db.rebuildGraphMetric(alloc, "graph_idx", "manual_degree"); + defer right_second.deinit(alloc); + try std.testing.expect(right_second.published_generation > left_status.published_generation); + + try std.testing.expectError(error.UnsupportedQueryRequest, hosted.source().query(alloc, "docs", metric_req, .read_index)); +} + +test "graph metric queries use general table read preparation and search path" { + const graph_metric_queries = [_]db_mod.types.NamedGraphMetricQuery{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 10, + }, + }}; + + const dense_only = db_mod.types.SearchRequest{ + .profile = true, + .index_name = "dense_idx", + .dense = .{ .vector = &.{ 1.0, 0.0 }, .k = 5 }, + }; + try std.testing.expect(profiledDenseQuery(dense_only) != null); + try std.testing.expectEqual(ReadPreparation.Kind.dense_query, readPreparationKindForQuery(dense_only)); + + var graph_metric_req = dense_only; + graph_metric_req.graph_metric_queries = &graph_metric_queries; + try std.testing.expect(profiledDenseQuery(graph_metric_req) == null); + try std.testing.expectEqual(ReadPreparation.Kind.general, readPreparationKindForQuery(graph_metric_req)); + + var graph_metric_rerank_req = dense_only; + graph_metric_rerank_req.graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + }; + try std.testing.expect(profiledDenseQuery(graph_metric_rerank_req) == null); + try std.testing.expectEqual(ReadPreparation.Kind.general, readPreparationKindForQuery(graph_metric_rerank_req)); + + var pruner_req = dense_only; + pruner_req.pruner = .{ .min_score_ratio = 0.5 }; + try std.testing.expect(profiledDenseQuery(pruner_req) == null); +} diff --git a/zig/pkg/antfly/src/api/table_reads/graph.zig b/zig/pkg/antfly/src/api/table_reads/graph.zig new file mode 100644 index 0000000000..d0e66dc34f --- /dev/null +++ b/zig/pkg/antfly/src/api/table_reads/graph.zig @@ -0,0 +1,236 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); + +const graph_query_mod = @import("../../graph/query.zig"); +const metadata_mod = @import("../../metadata/domain.zig"); +const raft_mod = @import("../../raft/mod.zig"); +const db_mod = @import("../../storage/db/mod.zig"); +const distributed_graph = @import("../distributed_graph.zig"); +const table_catalog = @import("../table_catalog.zig"); + +pub const GraphMetricFanInShardRequest = struct { + req: db_mod.types.SearchRequest, + graph_queries: []db_mod.types.NamedGraphQuery = &.{}, + + pub fn deinit(self: *@This(), alloc: std.mem.Allocator) void { + if (self.graph_queries.len > 0) alloc.free(self.graph_queries); + self.* = undefined; + } +}; + +pub fn graphSearchQueryNeedsInternalMetricStatus(query: graph_query_mod.GraphQuery) bool { + return query.metrics.len > 0 or query.order_by.len > 0 or query.where_metric.len > 0; +} + +pub fn rejectNonGlobalGraphMetricFanout(group_count: usize, req: db_mod.types.SearchRequest) !void { + if (group_count <= 1) return; + if (req.graph_metric_queries.len > 0 or req.graph_metric_rerank != null) { + return error.GraphMetricGlobalMaterializationRequired; + } + for (req.graph_queries) |named| { + if (graphSearchQueryNeedsInternalMetricStatus(named.query)) { + // Shard-local PageRank/eigenvector/HITS scores are normalized over + // different graphs and their numeric generations are not a global + // snapshot identity. Merging them would return plausible but + // mathematically invalid results. Fail closed until a coordinator + // supplies one globally materialized metric snapshot. + return error.GraphMetricGlobalMaterializationRequired; + } + } +} + +pub fn prepareGraphMetricFanInShardRequest( + alloc: std.mem.Allocator, + req: db_mod.types.SearchRequest, +) !GraphMetricFanInShardRequest { + var needs_graph_query_status = false; + for (req.graph_queries) |query| { + if (!query.query.include_metric_status and graphSearchQueryNeedsInternalMetricStatus(query.query)) { + needs_graph_query_status = true; + break; + } + } + const needs_rerank_status = req.graph_metric_rerank != null and !req.profile; + if (!needs_graph_query_status and !needs_rerank_status) return .{ .req = req }; + + const graph_queries = if (needs_graph_query_status) + try alloc.alloc(db_mod.types.NamedGraphQuery, req.graph_queries.len) + else + @constCast((&[_]db_mod.types.NamedGraphQuery{})[0..]); + if (needs_graph_query_status) { + @memcpy(graph_queries, req.graph_queries); + for (graph_queries) |*query| { + if (graphSearchQueryNeedsInternalMetricStatus(query.query)) query.query.include_metric_status = true; + } + } + + var out = req; + if (needs_graph_query_status) out.graph_queries = graph_queries; + // The public response keeps metric maintenance state in the optional + // profile. Shards must return it so the coordinator can validate a rerank + // generation before merging, while the caller's public request remains + // unchanged. + if (needs_rerank_status) out.profile = true; + return .{ .req = out, .graph_queries = graph_queries }; +} + +pub fn graphHydrateRequestHasResolvedDocFilter(req: distributed_graph.GraphHydrateRequest) bool { + return req.resolved_doc_filter != null; +} + +pub fn requiresDistributedGraphCoordinator( + group_count: usize, + req: db_mod.types.SearchRequest, +) bool { + return distributed_graph.supportsCrossRange(req) and + (group_count > 1 or req.graph_table_read_authorizer != null); +} + +pub fn validateGraphHydrateResolvedDocFilterForDb(req: distributed_graph.GraphHydrateRequest, db: *db_mod.DB) !void { + if (!graphHydrateRequestHasResolvedDocFilter(req)) return; + const ctx = req.resolved_doc_filter_wire_context orelse return error.UnsupportedQueryRequest; + if (!ctx.namespace.eql(db.core.identity_namespace)) return error.DocIdentityNamespaceMismatch; + const generation = try db.currentIdentityReadGenerationForRequest(req.identity_read_generation); + if (generation != ctx.identity_read_generation) return error.IdentityReadGenerationChanged; +} + +pub fn graphHydrateSearchRequest(req: distributed_graph.GraphHydrateRequest) db_mod.types.SearchRequest { + return .{ + .query = .{ .match_all = {} }, + .filter_query_json = req.filter_query_json, + .exclusion_query_json = req.exclusion_query_json, + .include_stored = req.include_stored, + .fields = req.fields, + .include_all_fields = req.include_all_fields, + .resolved_doc_filter = req.resolved_doc_filter, + .resolved_doc_filter_wire_context = req.resolved_doc_filter_wire_context, + .identity_read_generation = req.identity_read_generation, + .execution_deadline_ns = distributed_graph.executionDeadlineFromTimeoutMs(req.timeout_ms), + .cancellation = req.cancellation, + }; +} + +fn prepareGraphSearchConsistency( + reads: raft_mod.FeatureDBReads, + req: db_mod.types.SearchRequest, + consistency: raft_mod.ReadConsistency, + fallback_to_stale_on_not_leader: bool, +) !void { + reads.reads.prepareSearchWithConsistency(reads.group_id, req, consistency) catch |err| switch (err) { + error.NotLeader => { + if (!fallback_to_stale_on_not_leader or consistency == .stale) return err; + try reads.reads.prepareSearchWithConsistency(reads.group_id, req, .stale); + }, + else => return err, + }; +} + +pub fn graphHydrateOnPreparedDb( + alloc: std.mem.Allocator, + db: *db_mod.DB, + req: distributed_graph.GraphHydrateRequest, + search_req: db_mod.types.SearchRequest, +) !distributed_graph.GraphHydrateResponse { + if (req.incoming_index_name.len > 0 and !req.incoming_index_identity.valid()) return error.InvalidArgument; + const hits = if (req.include_hits) + try db.graphHydrateKeysForInternalRead(alloc, search_req, req.keys) + else + @constCast((&[_]db_mod.types.SearchHit{})[0..]); + errdefer { + for (hits) |*hit| hit.deinit(alloc); + if (hits.len > 0) alloc.free(hits); + } + return .{ + .hits = hits, + .has_incoming = if (req.incoming_index_name.len > 0) + try db.graphHasIncomingEdgesForInternalRead(alloc, req.incoming_index_name, req.keys, .{ + .generation = req.incoming_index_identity.incarnation, + .config_fingerprint = req.incoming_index_identity.config_hash, + }, req.identity_read_generation) + else + @constCast((&[_]bool{})[0..]), + .incoming_index_identity = req.incoming_index_identity, + }; +} + +pub fn graphHydrateOnOpenDb( + alloc: std.mem.Allocator, + reads: raft_mod.FeatureDBReads, + db: *db_mod.DB, + req: distributed_graph.GraphHydrateRequest, + consistency: raft_mod.ReadConsistency, + fallback_to_stale_on_not_leader: bool, +) !distributed_graph.GraphHydrateResponse { + const search_req = graphHydrateSearchRequest(req); + try prepareGraphSearchConsistency(reads, search_req, consistency, fallback_to_stale_on_not_leader); + return try graphHydrateOnPreparedDb(alloc, db, req, search_req); +} + +pub const OpenProvisionedQueryDbFn = fn ( + alloc: std.mem.Allocator, + path: []const u8, + catalog: table_catalog.CatalogSource, + table_name: []const u8, + group_id: u64, + lsm_root_generation: u64, + backend_runtime: ?*db_mod.background_runtime.BackendRuntime, +) anyerror!db_mod.DB; + +pub fn graphGetEdgesLocal( + alloc: std.mem.Allocator, + replica_root_dir: []const u8, + catalog: table_catalog.CatalogSource, + requester: raft_mod.ReadableLeaseRequester, + group_id: u64, + lsm_root_generation: u64, + backend_runtime: ?*db_mod.background_runtime.BackendRuntime, + table_name: []const u8, + req: distributed_graph.GraphEdgesRequest, + consistency: raft_mod.ReadConsistency, + comptime open_db: OpenProvisionedQueryDbFn, +) anyerror!distributed_graph.GraphEdgesResponse { + try table_catalog.validateTopologyEpoch(alloc, catalog, table_name, req.topology_epoch); + try distributed_graph.validateGraphEdgesTensorAccessPath(alloc, req); + + const path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, replica_root_dir, group_id); + defer alloc.free(path); + var db = try open_db(alloc, path, catalog, table_name, group_id, lsm_root_generation, backend_runtime); + defer db.close(); + _ = try db.currentIdentityReadGenerationForRequest(req.identity_read_generation); + + const reads = raft_mod.FeatureDBReads.init(group_id, requester); + try reads.reads.prepareLookupWithConsistency(group_id, req.key, .{}, consistency); + + const graph_entry = db.core.graphIndex(req.index_name) orelse return error.IndexNotFound; + return .{ .edges = try graph_entry.index.getEdgesByTypes(alloc, req.key, req.edge_types, req.direction) }; +} + +test "multi-shard reads fail closed for shard-local graph metric scores" { + const metric_req = db_mod.types.SearchRequest{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ .index_name = "graph_idx", .metric_name = "pagerank" }, + }}, + }; + try rejectNonGlobalGraphMetricFanout(1, metric_req); + try std.testing.expectError(error.GraphMetricGlobalMaterializationRequired, rejectNonGlobalGraphMetricFanout(2, metric_req)); + + const traversal_only = db_mod.types.SearchRequest{ .graph_queries = &.{.{ + .name = "neighbors", + .query = .{ .query_type = .neighbors, .index_name = "graph_idx", .start_nodes = .{ .keys = &.{"doc-a"} } }, + }} }; + try rejectNonGlobalGraphMetricFanout(2, traversal_only); +} diff --git a/zig/pkg/antfly/src/api/table_write_source.zig b/zig/pkg/antfly/src/api/table_write_source.zig index aff21667c0..15e35d47ec 100644 --- a/zig/pkg/antfly/src/api/table_write_source.zig +++ b/zig/pkg/antfly/src/api/table_write_source.zig @@ -75,6 +75,30 @@ pub const TableWriteSource = struct { table_name: []const u8, index_name: []const u8, ) anyerror!?void = null, + graph_metric_action: ?*const fn ( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) anyerror!?db_mod.types.GraphMetricStatus = null, + graph_metric_action_with_cancellation: ?*const fn ( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + cancellation: db_mod.types.CancellationToken, + ) anyerror!?db_mod.types.GraphMetricStatus = null, + graph_metric_maintenance_group_local: ?*const fn ( + ptr: *anyopaque, + alloc: std.mem.Allocator, + group_id: u64, + table_name: []const u8, + body: []const u8, + ) anyerror!?[]u8 = null, drop_table: ?*const fn ( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -544,6 +568,44 @@ pub const TableWriteSource = struct { return try BoundaryAbi.call("drop_index", self.boundary_dispatch, fn_ptr, .{ self.ptr, alloc, table_name, index_name }); } + pub fn graphMetricAction( + self: TableWriteSource, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) !?db_mod.types.GraphMetricStatus { + const fn_ptr = self.vtable.graph_metric_action orelse return null; + return try BoundaryAbi.call("graph_metric_action", self.boundary_dispatch, fn_ptr, .{ self.ptr, alloc, table_name, index_name, metric_name, action }); + } + + pub fn graphMetricActionWithCancellation( + self: TableWriteSource, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + cancellation: db_mod.types.CancellationToken, + ) !?db_mod.types.GraphMetricStatus { + if (cancellation.isCancelled()) return error.Canceled; + const fn_ptr = self.vtable.graph_metric_action_with_cancellation orelse + return try self.graphMetricAction(alloc, table_name, index_name, metric_name, action); + return try BoundaryAbi.call("graph_metric_action_with_cancellation", self.boundary_dispatch, fn_ptr, .{ self.ptr, alloc, table_name, index_name, metric_name, action, cancellation }); + } + + pub fn graphMetricMaintenanceGroupLocal( + self: TableWriteSource, + alloc: std.mem.Allocator, + group_id: u64, + table_name: []const u8, + body: []const u8, + ) !?[]u8 { + const fn_ptr = self.vtable.graph_metric_maintenance_group_local orelse return null; + return try BoundaryAbi.call("graph_metric_maintenance_group_local", self.boundary_dispatch, fn_ptr, .{ self.ptr, alloc, group_id, table_name, body }); + } + pub fn dropTable( self: TableWriteSource, alloc: std.mem.Allocator, diff --git a/zig/pkg/antfly/src/api/table_writes.zig b/zig/pkg/antfly/src/api/table_writes.zig index 9e1327afc4..1224ec4d5c 100644 --- a/zig/pkg/antfly/src/api/table_writes.zig +++ b/zig/pkg/antfly/src/api/table_writes.zig @@ -733,6 +733,91 @@ fn pinWriteCacheLsmOwnerEntriesBestEffort( return .{ .cache = cache, .storage = storage, .count = entry_count }; } +fn applyGraphMetricActionToDb( + alloc: std.mem.Allocator, + db: *db_mod.DB, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, +) !db_mod.types.GraphMetricStatus { + if (std.mem.eql(u8, action, "refresh")) return try db.scheduleGraphMetricBuild(alloc, index_name, metric_name, false); + if (std.mem.eql(u8, action, "rebuild")) return try db.scheduleGraphMetricBuild(alloc, index_name, metric_name, true); + if (std.mem.eql(u8, action, "delete")) return try db.deleteGraphMetricMaterialization(alloc, index_name, metric_name); + if (std.mem.eql(u8, action, "pause")) return try db.pauseGraphMetricMaintenance(alloc, index_name, metric_name); + if (std.mem.eql(u8, action, "resume")) return try db.resumeGraphMetricMaintenance(alloc, index_name, metric_name); + return error.InvalidGraphMetricAction; +} + +const GraphMetricGroupActionRequest = struct { + operation: ?[]const u8 = null, + index_name: []const u8 = "", + metric_name: []const u8 = "", + action: []const u8 = "", +}; + +const graph_metric_group_action_operation = "metric_action_v1"; + +fn graphMetricGroupActionBodyAlloc( + alloc: std.mem.Allocator, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, +) ![]u8 { + return try std.json.Stringify.valueAlloc(alloc, GraphMetricGroupActionRequest{ + .operation = graph_metric_group_action_operation, + .index_name = index_name, + .metric_name = metric_name, + .action = action, + }, .{ .emit_null_optional_fields = false }); +} + +fn runGraphMetricMaintenanceOrActionJsonAlloc( + alloc: std.mem.Allocator, + db: *db_mod.DB, + body: []const u8, +) ![]u8 { + var parsed = std.json.parseFromSlice(GraphMetricGroupActionRequest, alloc, body, .{ .ignore_unknown_fields = true }) catch + return try db.runGraphMetricServiceMaintenanceJsonAlloc(alloc, body); + defer parsed.deinit(); + const operation = parsed.value.operation orelse return try db.runGraphMetricServiceMaintenanceJsonAlloc(alloc, body); + if (!std.mem.eql(u8, operation, graph_metric_group_action_operation)) { + return try db.runGraphMetricServiceMaintenanceJsonAlloc(alloc, body); + } + if (parsed.value.index_name.len == 0 or parsed.value.metric_name.len == 0 or parsed.value.action.len == 0) { + return error.InvalidGraphMetricAction; + } + var status = try applyGraphMetricActionToDb( + alloc, + db, + parsed.value.index_name, + parsed.value.metric_name, + parsed.value.action, + ); + defer status.deinit(alloc); + return try std.json.Stringify.valueAlloc(alloc, status, .{ .emit_null_optional_fields = false }); +} + +fn parseGraphMetricGroupActionStatusAlloc( + alloc: std.mem.Allocator, + body: []const u8, +) !db_mod.types.GraphMetricStatus { + var parsed = try std.json.parseFromSlice(db_mod.types.GraphMetricStatus, alloc, body, .{ .ignore_unknown_fields = true }); + defer parsed.deinit(); + return try query_api.cloneGraphMetricStatus(alloc, parsed.value); +} + +test "graph metric group action envelope is typed and versioned" { + const alloc = std.testing.allocator; + const body = try graphMetricGroupActionBodyAlloc(alloc, "graph_idx", "pagerank", "refresh"); + defer alloc.free(body); + var parsed = try std.json.parseFromSlice(GraphMetricGroupActionRequest, alloc, body, .{}); + defer parsed.deinit(); + try std.testing.expectEqualStrings(graph_metric_group_action_operation, parsed.value.operation.?); + try std.testing.expectEqualStrings("graph_idx", parsed.value.index_name); + try std.testing.expectEqualStrings("pagerank", parsed.value.metric_name); + try std.testing.expectEqualStrings("refresh", parsed.value.action); +} + fn publishRuntimeStatusGroupForTest( cache: *runtime_status.TableRuntimeSnapshotCache, table_name: []const u8, @@ -789,6 +874,7 @@ const startup_catch_up_no_progress_threshold: u8 = 3; const startup_catch_up_quarantine_base_ms: u64 = 30 * std.time.ms_per_s; const startup_catch_up_quarantine_max_ms: u64 = 10 * std.time.s_per_min * std.time.ms_per_s; const artifact_repair_max_groups_per_request: usize = 64; +const graph_metric_action_fanout_max: usize = 16; const restore_trash_dir_name = ".antfly-restore-trash"; // Explicit cache bulk sessions are reserved for rebuild/import paths. Normal // API uploads no longer start these windows automatically; DB/storage owns @@ -6353,6 +6439,9 @@ pub const BoundTableWriteSource = struct { .put_artifact_enrichment = putArtifactEnrichment, .delete_artifact_enrichment = deleteArtifactEnrichment, .drop_index = dropIndex, + .graph_metric_action = graphMetricAction, + .graph_metric_action_with_cancellation = graphMetricActionWithCancellation, + .graph_metric_maintenance_group_local = graphMetricMaintenanceGroupLocal, .backup_table = backupTable, .restore_table = restoreTable, .commit_transaction = commitTransaction, @@ -7213,6 +7302,44 @@ pub const BoundTableWriteSource = struct { _ = try (try self.activeDb()).deleteIndex(index_name); } + fn graphMetricAction( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) !?db_mod.types.GraphMetricStatus { + return try graphMetricActionWithCancellation(ptr, alloc, table_name, index_name, metric_name, action, .none); + } + + fn graphMetricActionWithCancellation( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + cancellation: db_mod.types.CancellationToken, + ) !?db_mod.types.GraphMetricStatus { + const self: *BoundTableWriteSource = @ptrCast(@alignCast(ptr)); + if (!std.mem.eql(u8, self.table_name, table_name)) return null; + if (cancellation.isCancelled()) return error.Canceled; + return try applyGraphMetricActionToDb(alloc, try self.activeDb(), index_name, metric_name, action); + } + + fn graphMetricMaintenanceGroupLocal( + ptr: *anyopaque, + alloc: std.mem.Allocator, + _: u64, + table_name: []const u8, + body: []const u8, + ) !?[]u8 { + const self: *BoundTableWriteSource = @ptrCast(@alignCast(ptr)); + if (!std.mem.eql(u8, self.table_name, table_name)) return null; + return try runGraphMetricMaintenanceOrActionJsonAlloc(alloc, try self.activeDb(), body); + } + fn batchGroupLocal( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -19876,7 +20003,10 @@ pub const ProvisionedTableWriteSource = struct { .put_artifact_enrichment = putArtifactEnrichment, .delete_artifact_enrichment = deleteArtifactEnrichment, .drop_index = dropIndex, + .graph_metric_action = graphMetricAction, + .graph_metric_action_with_cancellation = graphMetricActionWithCancellation, .drop_table = dropTable, + .graph_metric_maintenance_group_local = graphMetricMaintenanceGroupLocal, .commit_transaction = commitTransaction, .commit_transaction_with_cancellation = commitTransactionWithCancellation, .commit_batch = commitBatch, @@ -19927,6 +20057,88 @@ pub const ProvisionedTableWriteSource = struct { }; } + fn graphMetricMaintenanceGroupLocal( + ptr: *anyopaque, + alloc: std.mem.Allocator, + group_id: u64, + table_name: []const u8, + body: []const u8, + ) !?[]u8 { + const self: *ProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + const path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, self.replica_root_dir, group_id); + defer alloc.free(path); + self.beginGroupOperation(table_name, group_id); + defer self.endGroupOperation(table_name, group_id); + + if (self.write_cache) |cache| { + var cached = try self.getOrOpenCachedDbMode(alloc, cache, path, group_id, table_name, .default, null, null); + defer cached.deinit(alloc); + return try runGraphMetricMaintenanceOrActionJsonAlloc(alloc, cached.db, body); + } + + var db = openManagedDbForTableGroupWithRuntimeAndHAWriteGate(alloc, path, self.catalog, table_name, group_id, self.backend_runtime, self.ha_write_gate, self.ha_async_mirror) catch |err| switch (err) { + error.FileNotFound => return error.UnknownGroup, + else => return err, + }; + defer db.close(); + return try runGraphMetricMaintenanceOrActionJsonAlloc(alloc, &db, body); + } + + fn graphMetricAction( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) !?db_mod.types.GraphMetricStatus { + return try graphMetricActionWithCancellation(ptr, alloc, table_name, index_name, metric_name, action, .none); + } + + fn graphMetricActionWithCancellation( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + cancellation: db_mod.types.CancellationToken, + ) !?db_mod.types.GraphMetricStatus { + const self: *ProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + if (self.localWriteOwnerSource()) |owner| return try owner.graphMetricActionWithCancellation(alloc, table_name, index_name, metric_name, action, cancellation); + try enforceHAWriteGateOptional(self.ha_write_gate); + const group_ids = try resolveCatalogGroupsEventually(alloc, self.catalog, table_name, "", "", 5 * std.time.ns_per_s, 10); + defer alloc.free(group_ids); + if (group_ids.len == 0) return null; + + var aggregate: ?db_mod.types.GraphMetricStatus = null; + errdefer if (aggregate) |*status| status.deinit(alloc); + for (group_ids) |group_id| { + if (cancellation.isCancelled()) return error.Canceled; + const path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, self.replica_root_dir, group_id); + defer alloc.free(path); + self.beginGroupOperation(table_name, group_id); + defer self.endGroupOperation(table_name, group_id); + var shard_status = if (self.write_cache) |cache| blk: { + var cached = try self.getOrOpenCachedDbMode(alloc, cache, path, group_id, table_name, .default, null, null); + defer cached.deinit(alloc); + break :blk try applyGraphMetricActionToDb(alloc, cached.db, index_name, metric_name, action); + } else blk: { + var db = try openManagedDbForTableGroupWithRuntimeAndHAWriteGate(alloc, path, self.catalog, table_name, group_id, self.backend_runtime, self.ha_write_gate, self.ha_async_mirror); + defer db.close(); + break :blk try applyGraphMetricActionToDb(alloc, &db, index_name, metric_name, action); + }; + if (aggregate) |*status| { + query_api.mergeCompatibleGraphMetricStatusInto(alloc, status, shard_status) catch |err| { + shard_status.deinit(alloc); + return err; + }; + shard_status.deinit(alloc); + } else aggregate = shard_status; + } + return aggregate; + } + fn putArtifactEnrichment( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -24461,6 +24673,9 @@ pub const HostedProvisionedTableWriteSource = struct { .put_artifact_enrichment = putArtifactEnrichment, .delete_artifact_enrichment = deleteArtifactEnrichment, .drop_index = dropIndex, + .graph_metric_action = graphMetricAction, + .graph_metric_action_with_cancellation = graphMetricActionWithCancellation, + .graph_metric_maintenance_group_local = graphMetricMaintenanceGroupLocal, .accept_committed_index_mutation = acceptCommittedIndexMutation, .commit_transaction = commitTransaction, .commit_transaction_with_cancellation = commitTransactionWithCancellation, @@ -24502,6 +24717,224 @@ pub const HostedProvisionedTableWriteSource = struct { }; } + fn graphMetricMaintenanceGroupLocal( + ptr: *anyopaque, + alloc: std.mem.Allocator, + group_id: u64, + table_name: []const u8, + body: []const u8, + ) !?[]u8 { + const self: *HostedProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + const path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, self.replica_root_dir, group_id); + defer alloc.free(path); + const hosted_cache = try hostedManagedDbCacheForRoot(self.replica_root_dir); + var cached = try self.getOrOpenCachedDbMode(hosted_cache, path, group_id, table_name, .default_async); + defer cached.deinit(hosted_cache.write_cache.alloc); + return try runGraphMetricMaintenanceOrActionJsonAlloc(alloc, cached.db, body); + } + + fn graphMetricActionForRoute( + self: *HostedProvisionedTableWriteSource, + alloc: std.mem.Allocator, + route: table_router.GroupRoute, + group_id: u64, + table_name: []const u8, + body: []const u8, + cancellation: db_mod.types.CancellationToken, + ) !db_mod.types.GraphMetricStatus { + if (cancellation.isCancelled()) return error.Canceled; + return switch (route) { + .local => blk: { + const response_body = (try graphMetricMaintenanceGroupLocal(self, alloc, group_id, table_name, body)) orelse + return error.UnknownGroup; + defer alloc.free(response_body); + break :blk try parseGraphMetricGroupActionStatusAlloc(alloc, response_body); + }, + .remote => |remote| blk: { + var client = http_client.ApiHttpClient.init(alloc, self.executor); + var request_cancellation = http_common.RequestCancellation.fromToken(cancellation); + var response = try client.fetchGroupGraphMetricMaintenanceWithCancellation( + remote.base_uri, + group_id, + table_name, + body, + if (cancellation.ptr != null) &request_cancellation else null, + ); + defer response.deinit(alloc); + break :blk try parseGraphMetricGroupActionStatusAlloc(alloc, response.body); + }, + }; + } + + fn graphMetricAction( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + ) !?db_mod.types.GraphMetricStatus { + return try graphMetricActionWithCancellation(ptr, alloc, table_name, index_name, metric_name, action, .none); + } + + fn graphMetricActionWithCancellation( + ptr: *anyopaque, + alloc: std.mem.Allocator, + table_name: []const u8, + index_name: []const u8, + metric_name: []const u8, + action: []const u8, + cancellation: db_mod.types.CancellationToken, + ) !?db_mod.types.GraphMetricStatus { + const self: *HostedProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + const group_ids = try resolveCatalogGroupsEventually(alloc, self.catalog, table_name, "", "", 5 * std.time.ns_per_s, 10); + defer alloc.free(group_ids); + if (group_ids.len == 0) return null; + const body = try graphMetricGroupActionBodyAlloc(alloc, index_name, metric_name, action); + defer alloc.free(body); + + // Resolve the complete route set before mutating any shard. This avoids + // an avoidable partial action when topology is already incomplete. + const routes = try alloc.alloc(table_router.GroupRoute, group_ids.len); + var routes_initialized: usize = 0; + defer { + for (routes[0..routes_initialized]) |*route| route.deinit(alloc); + alloc.free(routes); + } + for (group_ids, 0..) |group_id, i| { + routes[i] = (try table_router.resolveGroupRoute(alloc, self.catalog, self.router, group_id, .prefer_leader)) orelse + return error.LeaderUnavailable; + routes_initialized += 1; + } + + var aggregate: ?db_mod.types.GraphMetricStatus = null; + errdefer if (aggregate) |*status| status.deinit(alloc); + + var api_lane: ?db_mod.background_runtime.BackendRuntime.ApiLaneLease = if (self.backend_runtime) |runtime| + runtime.acquireApiLane() catch |err| switch (err) { + error.BackendRuntimeUnavailable => null, + else => return err, + } + else + null; + defer if (api_lane) |*lane| lane.release(); + if (api_lane == null or group_ids.len == 1) { + var accepted_groups: usize = 0; + for (group_ids, routes) |group_id, route| { + var shard_status = self.graphMetricActionForRoute(alloc, route, group_id, table_name, body, cancellation) catch |err| { + if (accepted_groups == 0) return err; + std.log.warn( + "graph metric action partially accepted table={s} index={s} metric={s} action={s} accepted_groups={} failed_group_id={} err={s}; retry is safe", + .{ table_name, index_name, metric_name, action, accepted_groups, group_id, @errorName(err) }, + ); + return error.GraphMetricActionPartialOutcome; + }; + accepted_groups += 1; + if (aggregate) |*status| { + query_api.mergeCompatibleGraphMetricStatusInto(alloc, status, shard_status) catch |err| { + shard_status.deinit(alloc); + std.log.warn( + "graph metric action accepted but shard status aggregation failed table={s} index={s} metric={s} action={s} accepted_groups={} err={s}; retry is safe", + .{ table_name, index_name, metric_name, action, accepted_groups, @errorName(err) }, + ); + return error.GraphMetricActionPartialOutcome; + }; + shard_status.deinit(alloc); + } else aggregate = shard_status; + } + return aggregate; + } + + const Slot = struct { + arena: std.heap.ArenaAllocator = std.heap.ArenaAllocator.init(std.heap.page_allocator), + status: ?db_mod.types.GraphMetricStatus = null, + err: ?anyerror = null, + }; + const slots = try alloc.alloc(Slot, group_ids.len); + defer { + for (slots) |*slot| slot.arena.deinit(); + alloc.free(slots); + } + for (slots) |*slot| slot.* = .{}; + + const Fiber = struct { + fn run( + hosted_source: *HostedProvisionedTableWriteSource, + slot: *Slot, + route: table_router.GroupRoute, + group_id: u64, + table_name_inner: []const u8, + body_inner: []const u8, + cancellation_inner: db_mod.types.CancellationToken, + ) void { + slot.status = hosted_source.graphMetricActionForRoute( + slot.arena.allocator(), + route, + group_id, + table_name_inner, + body_inner, + cancellation_inner, + ) catch |err| { + slot.err = err; + return; + }; + } + }; + const io = api_lane.?.io(); + const width = @max(@as(usize, 1), @min( + group_ids.len, + @min(graph_metric_action_fanout_max, @as(usize, @intCast(api_lane.?.concurrentCapacity()))), + )); + var start: usize = 0; + while (start < group_ids.len) : (start += width) { + const end = @min(start + width, group_ids.len); + var group: Io.Group = .init; + for (group_ids[start..end], routes[start..end], start..end) |group_id, route, i| { + group.async(io, Fiber.run, .{ self, &slots[i], route, group_id, table_name, body, cancellation }); + } + group.await(io) catch {}; + } + var accepted_groups: usize = 0; + var first_failed_group: ?u64 = null; + var first_error: ?anyerror = null; + for (slots, group_ids) |slot, group_id| { + if (slot.err) |err| { + if (first_error == null) { + first_error = err; + first_failed_group = group_id; + } + } else if (slot.status != null) { + accepted_groups += 1; + } else if (first_error == null) { + first_error = error.UnknownGroup; + first_failed_group = group_id; + } + } + if (first_error) |err| { + if (accepted_groups == 0) return err; + std.log.warn( + "graph metric action partially accepted table={s} index={s} metric={s} action={s} accepted_groups={} total_groups={} first_failed_group_id={} err={s}; retry is safe", + .{ table_name, index_name, metric_name, action, accepted_groups, group_ids.len, first_failed_group.?, @errorName(err) }, + ); + return error.GraphMetricActionPartialOutcome; + } + for (slots) |slot| { + const shard_status = slot.status orelse return error.UnknownGroup; + if (aggregate) |*status| { + query_api.mergeCompatibleGraphMetricStatusInto(alloc, status, shard_status) catch |err| { + std.log.warn( + "graph metric action accepted by all shards but status aggregation failed table={s} index={s} metric={s} action={s} total_groups={} err={s}; retry is safe", + .{ table_name, index_name, metric_name, action, group_ids.len, @errorName(err) }, + ); + return error.GraphMetricActionPartialOutcome; + }; + } else { + aggregate = try query_api.cloneGraphMetricStatus(alloc, shard_status); + } + } + return aggregate; + } + fn persistentDropCleanupSource( self: *HostedProvisionedTableWriteSource, cache: *HostedManagedDbCache, diff --git a/zig/pkg/antfly/src/api_graph_metric_test_root.zig b/zig/pkg/antfly/src/api_graph_metric_test_root.zig new file mode 100644 index 0000000000..eb5d410cbc --- /dev/null +++ b/zig/pkg/antfly/src/api_graph_metric_test_root.zig @@ -0,0 +1,37 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Focused discovery root for graph-metric public contracts and distributed +//! fan-in. This keeps the fail-closed matrix independently runnable without +//! coupling it to the monolithic storage test artifact. + +const query = @import("api/query.zig"); +const distributed_graph = @import("api/distributed_graph.zig"); +const openapi_contract = @import("api/openapi_contract.zig"); +const indexes = @import("api/indexes.zig"); +const public_table_http = @import("api/public_table_http.zig"); +const graph_exec = @import("storage/db/query/graph_exec.zig"); + +// Storage adapters resolve these declarations through their discovery root. +pub const storage_backend_erased = @import("storage/backend_erased.zig"); +pub const lsm_backend = @import("storage/lsm_backend.zig"); + +test { + _ = query; + _ = distributed_graph; + _ = openapi_contract; + _ = indexes; + _ = public_table_http; + _ = graph_exec; +} diff --git a/zig/pkg/antfly/src/cli_root.zig b/zig/pkg/antfly/src/cli_root.zig index 50c54e9ce9..3873f1047d 100644 --- a/zig/pkg/antfly/src/cli_root.zig +++ b/zig/pkg/antfly/src/cli_root.zig @@ -21,6 +21,8 @@ pub const build_options = @import("build_options"); pub const admin = @import("admin/mod.zig"); pub const common = @import("common/mod.zig"); pub const data = @import("data/mod.zig"); +pub const graph = @import("graph/graph.zig"); +pub const graph_query = @import("graph/query.zig"); pub const metadata = @import("metadata/mod.zig"); pub const public_api = @import("api/mod.zig"); pub const raft = @import("raft/mod.zig"); @@ -34,6 +36,8 @@ pub const backup_bundle = @import("storage/backup_bundle.zig"); pub const backup_bundle_io = @import("storage/backup_bundle_io.zig"); pub const backup_repository = @import("storage/backup_repository.zig"); pub const portable_backup = @import("storage/portable_backup.zig"); +pub const lmdb_engine = @import("lmdb_engine"); +pub const platform_clock = @import("antfly_platform").clock; pub const platform_time = @import("antfly_platform").time; // usermgr/storage_imports.zig depends back on these through antfly_root. diff --git a/zig/pkg/antfly/src/cmd/graph_metric_maintenance.zig b/zig/pkg/antfly/src/cmd/graph_metric_maintenance.zig new file mode 100644 index 0000000000..96e2c323ec --- /dev/null +++ b/zig/pkg/antfly/src/cmd/graph_metric_maintenance.zig @@ -0,0 +1,4396 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const antfly = @import("../cli_root.zig"); +const platform = @import("antfly_platform"); +const platform_clock = antfly.platform_clock; +const graph_metric_runtime_mod = antfly.db.graph_metric_runtime; +const graph_query_mod = antfly.graph_query; +const writer_lock_mod = antfly.lmdb_engine.writer_lock; +const internal_service_auth = @import("../api/internal_service_auth.zig"); + +const RuntimeRole = graph_metric_runtime_mod.Role; +const SweepResult = antfly.db.IndexManager.GraphMetricPlannedSchedulerSweepResult; +const HttpQueryResponse = antfly.public_api.http_client.QueryResponse; +const local_db_writer_lock_retries: usize = 10_000; +const local_db_writer_lock_sleep_ms: u64 = 5; +const process_subcommand = "__graph-metric-maintenance"; + +const ExitReason = enum { + max_ticks, + idle, +}; + +const SupervisorExitReason = enum { + idle, + max_rounds, + restart_limit, +}; + +const CliConfig = struct { + db_path: ?[]const u8 = null, + service_base_uri: ?[]const u8 = null, + service_group_id: ?u64 = null, + service_table_name: ?[]const u8 = null, + role: RuntimeRole = .combined, + runtime_id: []const u8 = "graph-metric-maintenance", + owner_id: []const u8 = "", + worker_id: []const u8 = "graph-metric-worker", + worker_ids: std.ArrayListUnmanaged([]const u8) = .empty, + lease_owned: bool = true, + lease_ttl_ms: u64 = 30_000, + coordinator_start_background_builds: bool = true, + max_ticks: usize = 1, + ticks_set: bool = false, + until_idle: bool = false, + max_idle_ticks: ?usize = null, + tick_interval_ms: u64 = 0, + max_rounds: usize = 1, + max_metrics_per_round: usize = 8, + max_pages_per_round: usize = 1, + test_now_ms: ?u64 = null, + test_ready_file: ?[]const u8 = null, + test_hold_after_run_ms: u64 = 0, + summary_file: ?[]const u8 = null, + local_db_writer_lock: bool = false, + help: bool = false, + + fn deinit(self: *CliConfig, alloc: std.mem.Allocator) void { + self.worker_ids.deinit(alloc); + self.* = undefined; + } + + fn runtimeConfig(self: CliConfig, clock: platform_clock.Clock) graph_metric_runtime_mod.Config { + return .{ + .enabled = true, + .start_background_loop = false, + .role = self.role, + .runtime_id = self.runtime_id, + .lease_owned = self.lease_owned, + .owner_id = self.owner_id, + .lease_ttl_ms = self.lease_ttl_ms, + .coordinator_start_background_builds = self.coordinator_start_background_builds, + .planned_options = .{ + .worker_id = self.worker_id, + .worker_ids = self.worker_ids.items, + .max_rounds = self.max_rounds, + .max_metrics_per_round = self.max_metrics_per_round, + .max_pages_per_round = self.max_pages_per_round, + }, + .clock = clock, + }; + } + + fn serviceTarget(self: CliConfig) !?ServiceTarget { + const has_any = self.service_base_uri != null or self.service_group_id != null or self.service_table_name != null; + if (!has_any) return null; + if (self.service_base_uri == null or self.service_group_id == null or self.service_table_name == null) { + return error.InvalidArguments; + } + return .{ + .base_uri = self.service_base_uri.?, + .group_id = self.service_group_id.?, + .table_name = self.service_table_name.?, + }; + } +}; + +const ServiceTarget = struct { + base_uri: []const u8, + group_id: u64, + table_name: []const u8, +}; + +const ServiceMaintenanceRequestWire = struct { + action: ServiceMaintenanceAction = .tick, + role: RuntimeRole, + runtime_id: []const u8, + owner_id: []const u8, + lease_owned: bool = false, + lease_ttl_ms: u64 = 30_000, + worker_id: ?[]const u8 = null, + worker_ids: ?[]const []const u8 = null, + start_background_builds: bool = true, + max_rounds: usize = 1, + max_metrics_per_round: usize = 8, + max_pages_per_round: usize = 1, + preserve_lease_after_tick: bool = false, + now_ms: ?u64 = null, +}; + +const ServiceMaintenanceAction = enum { + tick, + status, + release, +}; + +const ServiceMaintenanceResponseWire = struct { + result: SweepResult = .{}, + stats: ?graph_metric_runtime_mod.Stats = null, + released: bool = false, + lease_owner_id_hash: u64 = 0, + lease_expires_at_ms: u64 = 0, +}; + +const ServiceBoundaryRequester = struct { + ptr: *anyopaque, + request: *const fn (*anyopaque, std.mem.Allocator, ServiceTarget, []const u8) anyerror!HttpQueryResponse, +}; + +fn ServiceRequestAdapter( + comptime Context: type, + comptime request: fn (*Context, std.mem.Allocator, ServiceTarget, []const u8) anyerror!HttpQueryResponse, +) type { + return struct { + fn call( + ptr: *anyopaque, + alloc: std.mem.Allocator, + target: ServiceTarget, + body: []const u8, + ) !HttpQueryResponse { + const context: *Context = @ptrCast(@alignCast(ptr)); + return try request(context, alloc, target, body); + } + }; +} + +const ServiceMaintenanceBoundary = struct { + alloc: std.mem.Allocator, + target: ServiceTarget, + base_cli: CliConfig, + requester: ServiceBoundaryRequester, + stats: *graph_metric_runtime_mod.Stats, + last_response_had_stats: bool = false, + + fn boundary(self: *ServiceMaintenanceBoundary) graph_metric_runtime_mod.MaintenanceBoundary { + return graph_metric_runtime_mod.MaintenanceBoundary.init(self, &service_boundary_vtable); + } + + fn requestRole( + self: *ServiceMaintenanceBoundary, + role: RuntimeRole, + worker_id: []const u8, + worker_ids: []const []const u8, + start_background_builds: bool, + max_rounds: usize, + max_metrics: usize, + max_pages: usize, + now_ms: ?u64, + ) !SweepResult { + const body = try serviceRequestJsonFromFieldsAlloc( + self.alloc, + self.base_cli, + .tick, + role, + worker_id, + worker_ids, + start_background_builds, + max_rounds, + max_metrics, + max_pages, + now_ms, + ); + defer self.alloc.free(body); + + var response = try self.requester.request(self.requester.ptr, self.alloc, self.target, body); + defer response.deinit(self.alloc); + const parsed = try parseServiceMaintenanceResponse(self.alloc, response.body); + self.last_response_had_stats = parsed.stats != null; + if (parsed.stats) |server_stats| { + mergeServiceStats(self.stats, server_stats); + } + return parsed.result; + } +}; + +const service_boundary_vtable = graph_metric_runtime_mod.MaintenanceBoundary.VTable{ + .run_combined = serviceBoundaryRunCombined, + .run_coordinator = serviceBoundaryRunCoordinator, + .run_worker = serviceBoundaryRunWorker, + .run_worker_pool = serviceBoundaryRunWorkerPool, +}; + +fn serviceBoundaryContext(ptr: *anyopaque) *ServiceMaintenanceBoundary { + return @ptrCast(@alignCast(ptr)); +} + +fn serviceBoundaryRunCombined( + ptr: *anyopaque, + options: antfly.db.IndexManager.GraphMetricPlannedMaintenanceOptions, +) !SweepResult { + const self = serviceBoundaryContext(ptr); + return try self.requestRole( + .combined, + options.worker_id, + options.worker_ids, + true, + options.max_rounds, + options.max_metrics_per_round, + options.max_pages_per_round, + options.now_ms, + ); +} + +fn serviceBoundaryRunCoordinator( + ptr: *anyopaque, + options: antfly.db.IndexManager.GraphMetricPlannedSchedulerSweepOptions, +) !SweepResult { + const self = serviceBoundaryContext(ptr); + return try self.requestRole( + .coordinator, + self.base_cli.worker_id, + &.{}, + options.start_background_builds, + self.base_cli.max_rounds, + options.max_metrics, + self.base_cli.max_pages_per_round, + options.now_ms, + ); +} + +fn serviceBoundaryRunWorker( + ptr: *anyopaque, + options: antfly.db.IndexManager.GraphMetricPlannedWorkerSweepOptions, +) !SweepResult { + const self = serviceBoundaryContext(ptr); + return try self.requestRole( + .worker, + options.worker_id, + &.{}, + false, + self.base_cli.max_rounds, + self.base_cli.max_metrics_per_round, + options.max_pages, + options.now_ms, + ); +} + +fn serviceBoundaryRunWorkerPool( + ptr: *anyopaque, + options: graph_metric_runtime_mod.MaintenanceBoundary.WorkerPoolSweepOptions, +) !SweepResult { + const self = serviceBoundaryContext(ptr); + return try self.requestRole( + .worker_pool, + options.worker_id, + options.worker_ids, + false, + self.base_cli.max_rounds, + self.base_cli.max_metrics_per_round, + options.max_pages, + options.now_ms, + ); +} + +const RunSummary = struct { + role: RuntimeRole, + ticks_executed: usize, + idle_streak: usize, + exit_reason: ExitReason, + durable_progressed: bool, + result: SweepResult, + stats: graph_metric_runtime_mod.Stats, +}; + +const SupervisorConfig = struct { + db_path: ?[]const u8 = null, + service_base_uri: ?[]const u8 = null, + service_group_id: ?u64 = null, + service_table_name: ?[]const u8 = null, + executable: ?[]const u8 = null, + coordinator_owner_id: []const u8 = "graph-metric-coordinator", + worker_pool_owner_id: []const u8 = "graph-metric-worker-pool", + worker_ids: std.ArrayListUnmanaged([]const u8) = .empty, + lease_ttl_ms: u64 = 30_000, + max_ticks: usize = 1024, + max_idle_ticks: usize = 8, + max_supervisor_rounds: usize = 1024, + max_supervisor_idle_rounds: usize = 1, + tick_interval_ms: u64 = 25, + max_restarts: usize = 1, + max_rounds: usize = 1, + max_metrics_per_round: usize = 8, + max_pages_per_round: usize = 2, + summary_dir: []const u8 = ".zig-cache/tmp", + help: bool = false, + + fn deinit(self: *SupervisorConfig, alloc: std.mem.Allocator) void { + self.worker_ids.deinit(alloc); + self.* = undefined; + } + + fn ensureDefaultWorkers(self: *SupervisorConfig, alloc: std.mem.Allocator) !void { + if (self.worker_ids.items.len != 0) return; + try self.worker_ids.append(alloc, "graph-metric-worker-a"); + try self.worker_ids.append(alloc, "graph-metric-worker-b"); + } + + fn serviceTarget(self: SupervisorConfig) !?ServiceTarget { + const has_any = self.service_base_uri != null or self.service_group_id != null or self.service_table_name != null; + if (!has_any) return null; + if (self.service_base_uri == null or self.service_group_id == null or self.service_table_name == null) { + return error.InvalidArguments; + } + return .{ + .base_uri = self.service_base_uri.?, + .group_id = self.service_group_id.?, + .table_name = self.service_table_name.?, + }; + } + + fn target(self: SupervisorConfig) !?SupervisorTarget { + const service_target = try self.serviceTarget(); + if (self.db_path != null and service_target != null) return error.InvalidArguments; + if (service_target) |target_value| return .{ .service = target_value }; + if (self.db_path) |db_path| return .{ .db_path = db_path }; + return null; + } +}; + +const SupervisorTarget = union(enum) { + db_path: []const u8, + service: ServiceTarget, +}; + +const ChildRole = enum { + coordinator, + worker_pool, +}; + +const ChildExitSummary = struct { + exited: bool, + code: ?u8, +}; + +const ChildRuntimeTelemetry = struct { + role: RuntimeRole, + runtime_id_hash: u64, + owner_id_hash: u64, + lease_key_hash: u64, + worker_id_hash: u64, + worker_count: usize, + lease_owned: bool, + has_lease: bool, + acquisition_count: u64, + takeover_count: u64, + lost_leases: u64, + ticks_started: u64, + ticks_completed: u64, + idle_ticks: u64, + error_ticks: u64, + has_last_error: bool, +}; + +const ChildRunSummary = struct { + exit: ChildExitSummary, + durable_progressed: ?bool, + telemetry: ?ChildRuntimeTelemetry = null, + stdout_bytes: usize, + stderr_bytes: usize, +}; + +const SupervisorSummary = struct { + rounds_executed: usize, + restarts: usize, + idle_rounds: usize, + exit_reason: SupervisorExitReason, + succeeded: bool, + coordinator: ChildRunSummary, + worker_pool: ChildRunSummary, +}; + +const ChildArgv = struct { + argv: std.ArrayListUnmanaged([]const u8) = .empty, + owned: std.ArrayListUnmanaged([]const u8) = .empty, + + fn deinit(self: *ChildArgv, alloc: std.mem.Allocator) void { + for (self.owned.items) |item| alloc.free(item); + self.owned.deinit(alloc); + self.argv.deinit(alloc); + self.* = undefined; + } + + fn append(self: *ChildArgv, alloc: std.mem.Allocator, value: []const u8) !void { + try self.argv.append(alloc, value); + } + + fn appendOwned(self: *ChildArgv, alloc: std.mem.Allocator, comptime fmt: []const u8, args: anytype) !void { + const value = try std.fmt.allocPrint(alloc, fmt, args); + errdefer alloc.free(value); + try self.owned.append(alloc, value); + try self.argv.append(alloc, value); + } +}; + +pub fn run(init: std.process.Init) !void { + var args = try std.process.Args.Iterator.initAllocator(init.minimal.args, init.gpa); + defer args.deinit(); + + const argv0 = args.next() orelse "antfly"; + return try runFromIterator(init, argv0, &args); +} + +pub fn runFromIterator(init: std.process.Init, argv0: []const u8, args: *std.process.Args.Iterator) !void { + const alloc = init.gpa; + const first = args.next(); + if (first) |arg| { + if (std.mem.eql(u8, arg, "supervise")) { + var supervisor = try parseSupervisorCli(alloc, args); + defer supervisor.deinit(alloc); + if (supervisor.help) { + printSupervisorUsage(argv0); + return; + } + const target = (try supervisor.target()) orelse return error.InvalidArguments; + if (target == .service) _ = try serviceCredentials(init.environ_map); + const summary = try runSupervisorConfigured(init.io, alloc, argv0, target, supervisor); + try writeJson(init.io, alloc, summary); + return; + } + if (std.mem.eql(u8, arg, "launch")) { + var launcher = try parseSupervisorCli(alloc, args); + defer launcher.deinit(alloc); + if (launcher.help) { + printLaunchUsage(argv0); + return; + } + const target = (try launcher.target()) orelse return error.InvalidArguments; + if (target == .service) _ = try serviceCredentials(init.environ_map); + const summary = try runLaunchedConfigured(init.io, alloc, argv0, target, launcher); + try writeJson(init.io, alloc, summary); + return; + } + } + + var cli = try parseCliWithFirst(alloc, args, first); + defer cli.deinit(alloc); + if (cli.help) { + printUsage(argv0); + return; + } + const owner_incarnation = try ownerIncarnationAlloc(alloc, if (cli.owner_id.len != 0) cli.owner_id else cli.runtime_id); + defer alloc.free(owner_incarnation); + cli.owner_id = owner_incarnation; + const summary = if (try cli.serviceTarget()) |target| blk: { + if (cli.db_path != null) return error.InvalidArguments; + break :blk try runServiceConfigured(init, target, cli); + } else blk: { + const db_path = cli.db_path orelse return error.InvalidArguments; + break :blk try runConfigured(alloc, db_path, cli); + }; + try writeJson(init.io, alloc, summary); + if (cli.summary_file) |path| { + try writeJsonFile(init.io, alloc, path, summary); + } +} + +fn ownerIncarnationAlloc(alloc: std.mem.Allocator, logical_owner_id: []const u8) ![]u8 { + if (logical_owner_id.len == 0) return error.InvalidArguments; + return try std.fmt.allocPrint(alloc, "{s}:pid-{d}:start-{x}", .{ + logical_owner_id, + platform.process.currentId() orelse 0, + platform.time.monotonicNs(), + }); +} + +fn runConfigured(alloc: std.mem.Allocator, db_path: []const u8, cli: CliConfig) !RunSummary { + var local_db_writer_lock = if (cli.local_db_writer_lock) + try acquireLocalDbWriterLock(alloc, db_path) + else + null; + defer if (local_db_writer_lock) |*lock| lock.release(); + + var manual_clock = platform_clock.ManualClock{}; + if (cli.test_now_ms) |now_ms| manual_clock.setRealtimeNs(now_ms * std.time.ns_per_ms); + const clock = if (cli.test_now_ms != null) manual_clock.clock() else platform_clock.Clock.real(); + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = cli.runtimeConfig(clock), + }); + defer db.close(); + + const runtime = db.graph_metric_runtime orelse return error.GraphMetricRuntimeNotInitialized; + var total = SweepResult{}; + var ticks_executed: usize = 0; + var idle_streak: usize = 0; + var exit_reason: ExitReason = .max_ticks; + while (ticks_executed < cli.max_ticks) { + const tick = try runtime.runOnceDetailed(); + total.add(tick); + ticks_executed += 1; + if (tick.durableProgressed()) { + idle_streak = 0; + } else { + idle_streak += 1; + if (cli.max_idle_ticks) |max_idle_ticks| { + if (idle_streak >= max_idle_ticks) { + exit_reason = .idle; + break; + } + } + } + if (ticks_executed < cli.max_ticks and cli.tick_interval_ms > 0) { + platform.time.sleepNs(cli.tick_interval_ms * std.time.ns_per_ms); + } + } + + if (cli.test_ready_file) |path| { + writeReadyFile(path); + } + if (cli.test_hold_after_run_ms > 0) { + platform.time.sleepNs(cli.test_hold_after_run_ms * std.time.ns_per_ms); + } + + return .{ + .role = cli.role, + .ticks_executed = ticks_executed, + .idle_streak = idle_streak, + .exit_reason = exit_reason, + .durable_progressed = total.durableProgressed(), + .result = total, + .stats = runtime.stats(), + }; +} + +const RealServiceMaintenanceClient = struct { + client: *antfly.public_api.ApiHttpClient, + + fn request( + self: *RealServiceMaintenanceClient, + alloc: std.mem.Allocator, + target: ServiceTarget, + body: []const u8, + ) !HttpQueryResponse { + _ = alloc; + return try self.client.fetchGroupGraphMetricMaintenance(target.base_uri, target.group_id, target.table_name, body); + } +}; + +fn serviceCredentials(environ: *const std.process.Environ.Map) !internal_service_auth.Config { + const secret = environ.get("ANTFLY_INTERNAL_SERVICE_SECRET"); + const issuer = environ.get("ANTFLY_INTERNAL_SERVICE_ISSUER"); + try internal_service_auth.validateRuntimeConfig(secret, null, issuer); + return .{ .secret = secret.?, .issuer = issuer.? }; +} + +fn runServiceConfigured(init: std.process.Init, target: ServiceTarget, cli: CliConfig) !RunSummary { + const alloc = init.gpa; + // Use the same secret-key environment mapping as data/metadata runtimes. + // Children inherit credentials without exposing secrets in process argv. + const credentials = serviceCredentials(init.environ_map) catch |err| { + std.log.err("graph metric service roles require ANTFLY_INTERNAL_SERVICE_SECRET (at least 32 bytes) and ANTFLY_INTERNAL_SERVICE_ISSUER; err={s}", .{@errorName(err)}); + return err; + }; + var executor = antfly.raft.transport.StdHttpExecutor.init(alloc, .{}); + defer executor.deinit(); + var client = antfly.public_api.ApiHttpClient.init(alloc, executor.executor()); + _ = client.withInternalServiceAuth(credentials.secret, credentials.issuer); + var context = RealServiceMaintenanceClient{ .client = &client }; + return try runServiceConfiguredWithRequester( + RealServiceMaintenanceClient, + &context, + RealServiceMaintenanceClient.request, + alloc, + target, + cli, + ); +} + +fn runServiceConfiguredWithRequester( + comptime Context: type, + context: *Context, + comptime request: fn (*Context, std.mem.Allocator, ServiceTarget, []const u8) anyerror!HttpQueryResponse, + alloc: std.mem.Allocator, + target: ServiceTarget, + cli: CliConfig, +) !RunSummary { + var total = SweepResult{}; + var stats = try serviceInitialStats(alloc, cli); + const RequestAdapter = ServiceRequestAdapter(Context, request); + var boundary_context = ServiceMaintenanceBoundary{ + .alloc = alloc, + .target = target, + .base_cli = cli, + .requester = .{ + .ptr = context, + .request = RequestAdapter.call, + }, + .stats = &stats, + }; + const boundary = boundary_context.boundary(); + var manual_clock = platform_clock.ManualClock{}; + if (cli.test_now_ms) |now_ms| manual_clock.setRealtimeNs(now_ms * std.time.ns_per_ms); + const clock = if (cli.test_now_ms != null) manual_clock.clock() else platform_clock.Clock.real(); + const runtime_config = cli.runtimeConfig(clock); + var ticks_executed: usize = 0; + var idle_streak: usize = 0; + var exit_reason: ExitReason = .max_ticks; + while (ticks_executed < cli.max_ticks) { + stats.ticks_started += 1; + boundary_context.last_response_had_stats = false; + const tick = graph_metric_runtime_mod.runBoundaryTick( + boundary, + runtime_config, + clock.nowRealtimeMs(), + ) catch |err| { + stats.error_ticks += 1; + stats.last_error_name = @errorName(err); + return err; + }; + stats.ticks_completed += 1; + if (!boundary_context.last_response_had_stats) { + stats.last_result = tick; + stats.total_result.add(tick); + } + total.add(tick); + ticks_executed += 1; + if (!boundary_context.last_response_had_stats and tick.durableProgressed()) { + stats.durable_progress_ticks += 1; + idle_streak = 0; + } else if (boundary_context.last_response_had_stats and tick.durableProgressed()) { + idle_streak = 0; + } else { + if (!boundary_context.last_response_had_stats) stats.idle_ticks += 1; + idle_streak += 1; + if (cli.max_idle_ticks) |max_idle_ticks| { + if (idle_streak >= max_idle_ticks) { + exit_reason = .idle; + break; + } + } + } + if (ticks_executed < cli.max_ticks and cli.tick_interval_ms > 0) { + platform.time.sleepNs(cli.tick_interval_ms * std.time.ns_per_ms); + } + } + + if (cli.test_ready_file) |path| { + writeReadyFile(path); + } + if (cli.test_hold_after_run_ms > 0) { + platform.time.sleepNs(cli.test_hold_after_run_ms * std.time.ns_per_ms); + } + + if (cli.lease_owned) { + try releaseServiceRuntimeOwner(Context, context, request, alloc, target, cli, &stats); + } + + return .{ + .role = cli.role, + .ticks_executed = ticks_executed, + .idle_streak = idle_streak, + .exit_reason = exit_reason, + .durable_progressed = total.durableProgressed(), + .result = total, + .stats = stats, + }; +} + +fn serviceRequestJsonAlloc(alloc: std.mem.Allocator, cli: CliConfig) ![]u8 { + return try serviceRequestJsonFromFieldsAlloc( + alloc, + cli, + .tick, + cli.role, + cli.worker_id, + cli.worker_ids.items, + cli.coordinator_start_background_builds, + cli.max_rounds, + cli.max_metrics_per_round, + cli.max_pages_per_round, + cli.test_now_ms, + ); +} + +fn serviceRequestJsonFromFieldsAlloc( + alloc: std.mem.Allocator, + cli: CliConfig, + action: ServiceMaintenanceAction, + role: RuntimeRole, + worker_id: []const u8, + worker_ids_slice: []const []const u8, + start_background_builds: bool, + max_rounds: usize, + max_metrics_per_round: usize, + max_pages_per_round: usize, + now_ms: ?u64, +) ![]u8 { + const request_worker_ids: ?[]const []const u8 = if (worker_ids_slice.len == 0) null else worker_ids_slice; + return try std.json.Stringify.valueAlloc(alloc, ServiceMaintenanceRequestWire{ + .action = action, + .role = role, + .runtime_id = cli.runtime_id, + .owner_id = if (cli.owner_id.len != 0) cli.owner_id else cli.runtime_id, + .lease_owned = cli.lease_owned, + .lease_ttl_ms = cli.lease_ttl_ms, + .worker_id = worker_id, + .worker_ids = request_worker_ids, + .start_background_builds = start_background_builds, + .max_rounds = max_rounds, + .max_metrics_per_round = max_metrics_per_round, + .max_pages_per_round = max_pages_per_round, + .preserve_lease_after_tick = action == .tick and cli.test_hold_after_run_ms > 0, + .now_ms = now_ms, + }, .{}); +} + +fn releaseServiceRuntimeOwner( + comptime Context: type, + context: *Context, + comptime request: fn (*Context, std.mem.Allocator, ServiceTarget, []const u8) anyerror!HttpQueryResponse, + alloc: std.mem.Allocator, + target: ServiceTarget, + cli: CliConfig, + stats: *graph_metric_runtime_mod.Stats, +) !void { + const body = try serviceRequestJsonFromFieldsAlloc( + alloc, + cli, + .release, + cli.role, + cli.worker_id, + cli.worker_ids.items, + cli.coordinator_start_background_builds, + cli.max_rounds, + cli.max_metrics_per_round, + cli.max_pages_per_round, + cli.test_now_ms, + ); + defer alloc.free(body); + + var response = request(context, alloc, target, body) catch |err| { + stats.error_ticks += 1; + stats.last_error_name = @errorName(err); + return err; + }; + defer response.deinit(alloc); + const parsed = parseServiceMaintenanceResponse(alloc, response.body) catch |err| { + stats.error_ticks += 1; + stats.last_error_name = @errorName(err); + return err; + }; + if (parsed.stats) |server_stats| { + stats.lease_owned = server_stats.lease_owned; + stats.has_lease = server_stats.has_lease; + stats.last_acquired_ms = server_stats.last_acquired_ms; + stats.lease_expires_at_ms = server_stats.lease_expires_at_ms; + stats.lease_renew_after_ms = server_stats.lease_renew_after_ms; + stats.renewal_count = server_stats.renewal_count; + } else if (parsed.released) { + stats.has_lease = false; + } + stats.shutdown = true; +} + +fn parseServiceMaintenanceResponse(alloc: std.mem.Allocator, body: []const u8) !ServiceMaintenanceResponseWire { + var parsed_value = try std.json.parseFromSlice(std.json.Value, alloc, body, .{}); + defer parsed_value.deinit(); + if (parsed_value.value == .object and + (parsed_value.value.object.get("result") != null or + parsed_value.value.object.get("stats") != null or + parsed_value.value.object.get("released") != null)) + { + var parsed = try std.json.parseFromSlice(ServiceMaintenanceResponseWire, alloc, body, .{ + .ignore_unknown_fields = true, + }); + defer parsed.deinit(); + return parsed.value; + } + var parsed = try std.json.parseFromSlice(SweepResult, alloc, body, .{ + .ignore_unknown_fields = true, + }); + defer parsed.deinit(); + return .{ .result = parsed.value, .stats = null }; +} + +fn mergeServiceStats( + stats: *graph_metric_runtime_mod.Stats, + server_stats: graph_metric_runtime_mod.Stats, +) void { + stats.enabled = server_stats.enabled; + stats.role = server_stats.role; + stats.runtime_id_hash = server_stats.runtime_id_hash; + stats.owner_id_hash = server_stats.owner_id_hash; + stats.lease_key_hash = server_stats.lease_key_hash; + stats.worker_id_hash = server_stats.worker_id_hash; + stats.worker_count = server_stats.worker_count; + stats.lease_owned = server_stats.lease_owned; + stats.has_lease = server_stats.has_lease; + stats.acquisition_count += server_stats.acquisition_count; + stats.takeover_count += server_stats.takeover_count; + stats.lease_acquire_failures += server_stats.lease_acquire_failures; + stats.lost_leases += server_stats.lost_leases; + stats.last_acquired_ms = server_stats.last_acquired_ms; + stats.lease_expires_at_ms = server_stats.lease_expires_at_ms; + stats.lease_renew_after_ms = server_stats.lease_renew_after_ms; + stats.renewal_count += server_stats.renewal_count; + stats.durable_progress_ticks += server_stats.durable_progress_ticks; + stats.idle_ticks += server_stats.idle_ticks; + stats.error_ticks += server_stats.error_ticks; + stats.last_error_name = server_stats.last_error_name; + stats.last_result = server_stats.last_result; + stats.total_result.add(server_stats.total_result); +} + +fn serviceInitialStats(alloc: std.mem.Allocator, cli: CliConfig) !graph_metric_runtime_mod.Stats { + const worker_hash = serviceWorkerIdentityHash(cli); + const lease_key_hash = try serviceRuntimeLeaseKeyHash(alloc, cli, worker_hash); + return .{ + .enabled = true, + .role = cli.role, + .runtime_id_hash = identityHash(cli.runtime_id), + .owner_id_hash = identityHash(if (cli.owner_id.len != 0) cli.owner_id else cli.runtime_id), + .lease_key_hash = lease_key_hash, + .worker_id_hash = worker_hash, + .worker_count = configuredWorkerCount(cli), + .lease_owned = cli.lease_owned, + .has_lease = !cli.lease_owned, + }; +} + +fn identityHash(value: []const u8) u64 { + if (value.len == 0) return 0; + return std.hash.Wyhash.hash(0, value); +} + +fn configuredWorkerCount(cli: CliConfig) usize { + if (cli.role == .coordinator) return 0; + if (cli.worker_ids.items.len != 0) return cli.worker_ids.items.len; + return if (cli.worker_id.len == 0) 0 else 1; +} + +fn serviceWorkerIdentityHash(cli: CliConfig) u64 { + if (cli.role == .coordinator) return 0; + if (cli.worker_ids.items.len == 0) return identityHash(cli.worker_id); + var xor_hash: u64 = 0; + var sum_hash: u64 = 0; + for (cli.worker_ids.items) |worker_id| { + const item_hash = identityHash(worker_id); + xor_hash ^= item_hash; + sum_hash +%= item_hash; + } + const fingerprint_words = [_]u64{ + @intCast(cli.worker_ids.items.len), + xor_hash, + sum_hash, + }; + return std.hash.Wyhash.hash(0, std.mem.asBytes(&fingerprint_words)); +} + +fn serviceRuntimeLeaseKeyHash(alloc: std.mem.Allocator, cli: CliConfig, worker_hash: u64) !u64 { + const base_key = graph_metric_runtime_mod.defaultLeaseKey(cli.role); + switch (cli.role) { + .combined, .coordinator => return identityHash(base_key), + .worker, .worker_pool => { + if (configuredWorkerCount(cli) == 0) return identityHash(base_key); + const lease_key = try std.fmt.allocPrint(alloc, "{s}:{x}", .{ base_key, worker_hash }); + defer alloc.free(lease_key); + return identityHash(lease_key); + }, + } +} + +fn parseCli(alloc: std.mem.Allocator, args: *std.process.Args.Iterator) !CliConfig { + return parseCliWithFirst(alloc, args, null); +} + +fn parseCliWithFirst(alloc: std.mem.Allocator, args: *std.process.Args.Iterator, first_arg: ?[]const u8) !CliConfig { + var cfg = CliConfig{}; + errdefer cfg.deinit(alloc); + var pending_arg = first_arg; + while (true) { + const arg = blk: { + if (pending_arg) |value| { + pending_arg = null; + break :blk value; + } + break :blk args.next() orelse break; + }; + if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) { + cfg.help = true; + continue; + } + if (std.mem.eql(u8, arg, "--db-path")) { + cfg.db_path = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--base-uri") or std.mem.eql(u8, arg, "--service-base-uri")) { + cfg.service_base_uri = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--group-id")) { + cfg.service_group_id = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--table-name")) { + cfg.service_table_name = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--base-uri") or std.mem.eql(u8, arg, "--service-base-uri")) { + cfg.service_base_uri = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--group-id")) { + cfg.service_group_id = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--table-name")) { + cfg.service_table_name = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--role")) { + cfg.role = try parseRole(args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--runtime-id")) { + cfg.runtime_id = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--owner-id")) { + cfg.owner_id = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--worker-id")) { + cfg.worker_id = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--worker-ids")) { + try parseWorkerIds(alloc, &cfg.worker_ids, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--lease-owned")) { + cfg.lease_owned = try parseBool(args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--lease-ttl-ms")) { + cfg.lease_ttl_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--coordinator-start-background-builds")) { + cfg.coordinator_start_background_builds = try parseBool(args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--ticks")) { + cfg.max_ticks = try parseInt(usize, args.next() orelse return error.InvalidArguments); + cfg.ticks_set = true; + continue; + } + if (std.mem.eql(u8, arg, "--until-idle")) { + cfg.until_idle = true; + continue; + } + if (std.mem.eql(u8, arg, "--max-idle-ticks")) { + cfg.max_idle_ticks = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--tick-ms")) { + cfg.tick_interval_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-rounds")) { + cfg.max_rounds = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-metrics")) { + cfg.max_metrics_per_round = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-pages")) { + cfg.max_pages_per_round = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--test-now-ms")) { + cfg.test_now_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--test-ready-file")) { + cfg.test_ready_file = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--test-hold-after-run-ms")) { + cfg.test_hold_after_run_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--summary-file")) { + cfg.summary_file = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--local-db-writer-lock")) { + cfg.local_db_writer_lock = try parseBool(args.next() orelse return error.InvalidArguments); + continue; + } + return error.InvalidArguments; + } + if (cfg.until_idle and cfg.max_idle_ticks == null) cfg.max_idle_ticks = 1; + if (cfg.until_idle and !cfg.ticks_set) cfg.max_ticks = 1024; + if (cfg.max_ticks == 0) return error.InvalidArguments; + if (cfg.max_idle_ticks != null and cfg.max_idle_ticks.? == 0) return error.InvalidArguments; + if (cfg.lease_ttl_ms == 0) return error.InvalidArguments; + if (cfg.max_rounds == 0) return error.InvalidArguments; + if (cfg.max_metrics_per_round == 0) return error.InvalidArguments; + if (cfg.max_pages_per_round == 0) return error.InvalidArguments; + if ((cfg.role == .coordinator or cfg.role == .worker) and cfg.worker_ids.items.len != 0) { + return error.InvalidArguments; + } + const service_target = try cfg.serviceTarget(); + if (cfg.db_path != null and service_target != null) return error.InvalidArguments; + return cfg; +} + +fn writeReadyFile(path: []const u8) void { + std.Io.Dir.cwd().writeFile(std.Io.Threaded.global_single_threaded.io(), .{ + .sub_path = path, + .data = "ready\n", + }) catch {}; +} + +fn parseSupervisorCli(alloc: std.mem.Allocator, args: *std.process.Args.Iterator) !SupervisorConfig { + var cfg = SupervisorConfig{}; + errdefer cfg.deinit(alloc); + while (args.next()) |arg| { + if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) { + cfg.help = true; + continue; + } + if (std.mem.eql(u8, arg, "--db-path")) { + cfg.db_path = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--executable")) { + cfg.executable = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--base-uri") or std.mem.eql(u8, arg, "--service-base-uri")) { + cfg.service_base_uri = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--group-id")) { + cfg.service_group_id = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--table-name")) { + cfg.service_table_name = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--coordinator-owner-id")) { + cfg.coordinator_owner_id = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--worker-pool-owner-id")) { + cfg.worker_pool_owner_id = args.next() orelse return error.InvalidArguments; + continue; + } + if (std.mem.eql(u8, arg, "--worker-ids")) { + try parseWorkerIds(alloc, &cfg.worker_ids, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--lease-ttl-ms")) { + cfg.lease_ttl_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--ticks")) { + cfg.max_ticks = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-idle-ticks")) { + cfg.max_idle_ticks = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--supervisor-rounds")) { + cfg.max_supervisor_rounds = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--supervisor-idle-rounds")) { + cfg.max_supervisor_idle_rounds = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--tick-ms")) { + cfg.tick_interval_ms = try parseInt(u64, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-restarts")) { + cfg.max_restarts = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-rounds")) { + cfg.max_rounds = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-metrics")) { + cfg.max_metrics_per_round = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--max-pages")) { + cfg.max_pages_per_round = try parseInt(usize, args.next() orelse return error.InvalidArguments); + continue; + } + if (std.mem.eql(u8, arg, "--summary-dir")) { + cfg.summary_dir = args.next() orelse return error.InvalidArguments; + continue; + } + return error.InvalidArguments; + } + + if (cfg.max_ticks == 0) return error.InvalidArguments; + if (cfg.max_idle_ticks == 0) return error.InvalidArguments; + if (cfg.max_supervisor_rounds == 0) return error.InvalidArguments; + if (cfg.max_supervisor_idle_rounds == 0) return error.InvalidArguments; + if (cfg.lease_ttl_ms == 0) return error.InvalidArguments; + if (cfg.max_rounds == 0) return error.InvalidArguments; + if (cfg.max_metrics_per_round == 0) return error.InvalidArguments; + if (cfg.max_pages_per_round == 0) return error.InvalidArguments; + _ = try cfg.target(); + try cfg.ensureDefaultWorkers(alloc); + return cfg; +} + +fn parseRole(raw: []const u8) !RuntimeRole { + if (std.mem.eql(u8, raw, "combined")) return .combined; + if (std.mem.eql(u8, raw, "coordinator")) return .coordinator; + if (std.mem.eql(u8, raw, "worker")) return .worker; + if (std.mem.eql(u8, raw, "worker_pool") or std.mem.eql(u8, raw, "worker-pool")) return .worker_pool; + return error.InvalidArguments; +} + +fn parseBool(raw: []const u8) !bool { + if (std.mem.eql(u8, raw, "true") or std.mem.eql(u8, raw, "1") or std.mem.eql(u8, raw, "yes")) return true; + if (std.mem.eql(u8, raw, "false") or std.mem.eql(u8, raw, "0") or std.mem.eql(u8, raw, "no")) return false; + return error.InvalidArguments; +} + +fn parseInt(comptime T: type, raw: []const u8) !T { + return std.fmt.parseInt(T, raw, 10) catch return error.InvalidArguments; +} + +fn parseWorkerIds( + alloc: std.mem.Allocator, + out: *std.ArrayListUnmanaged([]const u8), + raw: []const u8, +) !void { + out.clearRetainingCapacity(); + var it = std.mem.splitScalar(u8, raw, ','); + while (it.next()) |worker_id| { + if (worker_id.len == 0) return error.InvalidArguments; + for (out.items) |prior_worker_id| { + if (std.mem.eql(u8, worker_id, prior_worker_id)) return error.InvalidArguments; + } + try out.append(alloc, worker_id); + } + if (out.items.len == 0) return error.InvalidArguments; +} + +fn runSupervisorConfigured( + io: std.Io, + alloc: std.mem.Allocator, + argv0: []const u8, + target: SupervisorTarget, + cfg: SupervisorConfig, +) !SupervisorSummary { + var context = RealSupervisorChildRunner{}; + return runSupervisorConfiguredWithRunner( + RealSupervisorChildRunner, + &context, + RealSupervisorChildRunner.run, + io, + alloc, + argv0, + target, + cfg, + ); +} + +fn runLaunchedConfigured( + io: std.Io, + alloc: std.mem.Allocator, + argv0: []const u8, + target: SupervisorTarget, + cfg: SupervisorConfig, +) !SupervisorSummary { + try std.Io.Dir.cwd().createDirPath(io, cfg.summary_dir); + + var rounds_executed: usize = 0; + var restarts: usize = 0; + var idle_rounds: usize = 0; + var coordinator = ChildRunSummary{ + .exit = .{ .exited = false, .code = null }, + .durable_progressed = null, + .telemetry = null, + .stdout_bytes = 0, + .stderr_bytes = 0, + }; + var worker_pool = coordinator; + + while (rounds_executed < cfg.max_supervisor_rounds) { + rounds_executed += 1; + const coordinator_summary_path = try launchSummaryPathAlloc(alloc, cfg.summary_dir, .coordinator, rounds_executed); + defer alloc.free(coordinator_summary_path); + const worker_pool_summary_path = try launchSummaryPathAlloc(alloc, cfg.summary_dir, .worker_pool, rounds_executed); + defer alloc.free(worker_pool_summary_path); + const coordinator_stderr_path = try launchStderrPathAlloc(alloc, cfg.summary_dir, .coordinator, rounds_executed); + defer alloc.free(coordinator_stderr_path); + const worker_pool_stderr_path = try launchStderrPathAlloc(alloc, cfg.summary_dir, .worker_pool, rounds_executed); + defer alloc.free(worker_pool_stderr_path); + std.Io.Dir.cwd().deleteFile(io, coordinator_summary_path) catch {}; + std.Io.Dir.cwd().deleteFile(io, worker_pool_summary_path) catch {}; + std.Io.Dir.cwd().deleteFile(io, coordinator_stderr_path) catch {}; + std.Io.Dir.cwd().deleteFile(io, worker_pool_stderr_path) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_summary_path) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_pool_summary_path) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_stderr_path) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_pool_stderr_path) catch {}; + + var coordinator_argv = try buildSupervisorChildArgv(alloc, argv0, target, cfg, .coordinator, coordinator_summary_path); + defer coordinator_argv.deinit(alloc); + var worker_pool_argv = try buildSupervisorChildArgv(alloc, argv0, target, cfg, .worker_pool, worker_pool_summary_path); + defer worker_pool_argv.deinit(alloc); + + var coordinator_stderr = try std.Io.Dir.cwd().createFile(io, coordinator_stderr_path, .{ .truncate = true }); + defer coordinator_stderr.close(io); + var worker_pool_stderr = try std.Io.Dir.cwd().createFile(io, worker_pool_stderr_path, .{ .truncate = true }); + defer worker_pool_stderr.close(io); + + var coordinator_child = try std.process.spawn(io, .{ + .argv = coordinator_argv.argv.items, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .{ .file = coordinator_stderr }, + }); + errdefer coordinator_child.kill(io); + var worker_pool_child = try std.process.spawn(io, .{ + .argv = worker_pool_argv.argv.items, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .{ .file = worker_pool_stderr }, + }); + errdefer worker_pool_child.kill(io); + + const coordinator_term = try coordinator_child.wait(io); + const worker_pool_term = try worker_pool_child.wait(io); + coordinator = try childRunSummaryFromFile(alloc, io, coordinator_term, coordinator_summary_path, coordinator_stderr_path); + worker_pool = try childRunSummaryFromFile(alloc, io, worker_pool_term, worker_pool_summary_path, worker_pool_stderr_path); + + const failed = !childRunSucceeded(coordinator) or !childRunSucceeded(worker_pool); + if (failed) { + reportLaunchChildFailure(alloc, io, "coordinator", coordinator, coordinator_stderr_path); + reportLaunchChildFailure(alloc, io, "worker_pool", worker_pool, worker_pool_stderr_path); + if (!shouldRestartSupervisorAttempt(true, restarts, cfg.max_restarts)) { + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .restart_limit, + .succeeded = false, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; + } + restarts += 1; + idle_rounds = 0; + continue; + } + + const progressed = childRunDurableProgressed(coordinator) or childRunDurableProgressed(worker_pool); + if (progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= cfg.max_supervisor_idle_rounds) { + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .idle, + .succeeded = true, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; + } + } + if (cfg.tick_interval_ms > 0) { + platform.time.sleepNs(cfg.tick_interval_ms * std.time.ns_per_ms); + } + } + + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .max_rounds, + .succeeded = false, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; +} + +fn runSupervisorConfiguredWithRunner( + comptime Context: type, + context: *Context, + comptime runChild: fn (*Context, std.mem.Allocator, std.Io, []const []const u8) anyerror!ChildRunSummary, + io: std.Io, + alloc: std.mem.Allocator, + argv0: []const u8, + target: SupervisorTarget, + cfg: SupervisorConfig, +) !SupervisorSummary { + var rounds_executed: usize = 0; + var restarts: usize = 0; + var idle_rounds: usize = 0; + var coordinator = ChildRunSummary{ + .exit = .{ .exited = false, .code = null }, + .durable_progressed = null, + .telemetry = null, + .stdout_bytes = 0, + .stderr_bytes = 0, + }; + var worker_pool = coordinator; + + while (rounds_executed < cfg.max_supervisor_rounds) { + rounds_executed += 1; + var coordinator_argv = try buildSupervisorChildArgv(alloc, argv0, target, cfg, .coordinator, null); + defer coordinator_argv.deinit(alloc); + var worker_pool_argv = try buildSupervisorChildArgv(alloc, argv0, target, cfg, .worker_pool, null); + defer worker_pool_argv.deinit(alloc); + + coordinator = try runChild(context, alloc, io, coordinator_argv.argv.items); + worker_pool = try runChild(context, alloc, io, worker_pool_argv.argv.items); + const failed = !childRunSucceeded(coordinator) or !childRunSucceeded(worker_pool); + if (failed) { + if (!shouldRestartSupervisorAttempt(true, restarts, cfg.max_restarts)) { + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .restart_limit, + .succeeded = false, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; + } + restarts += 1; + idle_rounds = 0; + continue; + } + + const progressed = childRunDurableProgressed(coordinator) or childRunDurableProgressed(worker_pool); + if (progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= cfg.max_supervisor_idle_rounds) { + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .idle, + .succeeded = true, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; + } + } + if (cfg.tick_interval_ms > 0) { + platform.time.sleepNs(cfg.tick_interval_ms * std.time.ns_per_ms); + } + } + + return .{ + .rounds_executed = rounds_executed, + .restarts = restarts, + .idle_rounds = idle_rounds, + .exit_reason = .max_rounds, + .succeeded = false, + .coordinator = coordinator, + .worker_pool = worker_pool, + }; +} + +const RealSupervisorChildRunner = struct { + fn run( + self: *RealSupervisorChildRunner, + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, + ) !ChildRunSummary { + _ = self; + const result = try std.process.run(alloc, io, .{ + .argv = argv, + .reserve_amount = 512, + }); + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + return .{ + .exit = childExitFromTerm(result.term), + .durable_progressed = parseChildDurableProgressed(alloc, result.stdout) catch null, + .telemetry = parseChildRuntimeTelemetry(alloc, result.stdout) catch null, + .stdout_bytes = result.stdout.len, + .stderr_bytes = result.stderr.len, + }; + } +}; + +fn buildSupervisorChildArgv( + alloc: std.mem.Allocator, + argv0: []const u8, + target: SupervisorTarget, + cfg: SupervisorConfig, + role: ChildRole, + summary_file: ?[]const u8, +) !ChildArgv { + var out = ChildArgv{}; + errdefer out.deinit(alloc); + + try out.append(alloc, cfg.executable orelse argv0); + try out.append(alloc, process_subcommand); + try appendSupervisorTargetArgs(&out, alloc, target); + try out.append(alloc, "--role"); + try out.append(alloc, switch (role) { + .coordinator => "coordinator", + .worker_pool => "worker_pool", + }); + + const owner_id = switch (role) { + .coordinator => cfg.coordinator_owner_id, + .worker_pool => cfg.worker_pool_owner_id, + }; + try out.append(alloc, "--runtime-id"); + try out.append(alloc, owner_id); + try out.append(alloc, "--owner-id"); + try out.append(alloc, owner_id); + try out.append(alloc, "--worker-id"); + try out.append(alloc, switch (role) { + .coordinator => "graph-metric-coordinator-unused", + .worker_pool => "graph-metric-worker-pool-unused", + }); + if (role == .worker_pool) { + const worker_ids = try std.mem.join(alloc, ",", cfg.worker_ids.items); + errdefer alloc.free(worker_ids); + try out.owned.append(alloc, worker_ids); + try out.append(alloc, "--worker-ids"); + try out.append(alloc, worker_ids); + } + + try out.append(alloc, "--lease-owned"); + try out.append(alloc, "true"); + try out.append(alloc, "--lease-ttl-ms"); + try out.appendOwned(alloc, "{d}", .{cfg.lease_ttl_ms}); + try out.append(alloc, "--coordinator-start-background-builds"); + try out.append(alloc, if (role == .coordinator) "true" else "false"); + try out.append(alloc, "--ticks"); + try out.appendOwned(alloc, "{d}", .{cfg.max_ticks}); + try out.append(alloc, "--until-idle"); + try out.append(alloc, "--max-idle-ticks"); + try out.appendOwned(alloc, "{d}", .{cfg.max_idle_ticks}); + try out.append(alloc, "--tick-ms"); + try out.appendOwned(alloc, "{d}", .{cfg.tick_interval_ms}); + try out.append(alloc, "--max-rounds"); + try out.appendOwned(alloc, "{d}", .{cfg.max_rounds}); + try out.append(alloc, "--max-metrics"); + try out.appendOwned(alloc, "{d}", .{cfg.max_metrics_per_round}); + try out.append(alloc, "--max-pages"); + try out.appendOwned(alloc, "{d}", .{cfg.max_pages_per_round}); + if (summary_file) |path| { + try out.append(alloc, "--summary-file"); + try out.append(alloc, path); + if (std.meta.activeTag(target) == .db_path) { + try out.append(alloc, "--local-db-writer-lock"); + try out.append(alloc, "true"); + } + } + + return out; +} + +fn appendSupervisorTargetArgs(out: *ChildArgv, alloc: std.mem.Allocator, target: SupervisorTarget) !void { + switch (target) { + .db_path => |db_path| { + try out.append(alloc, "--db-path"); + try out.append(alloc, db_path); + }, + .service => |service| { + try out.append(alloc, "--base-uri"); + try out.append(alloc, service.base_uri); + try out.append(alloc, "--group-id"); + try out.appendOwned(alloc, "{d}", .{service.group_id}); + try out.append(alloc, "--table-name"); + try out.append(alloc, service.table_name); + }, + } +} + +fn acquireLocalDbWriterLock(alloc: std.mem.Allocator, db_path: []const u8) !writer_lock_mod.WriterLock { + const lock_path = try std.fmt.allocPrint(alloc, "{s}-graph-metric-maintenance", .{db_path}); + defer alloc.free(lock_path); + + var attempts: usize = 0; + while (attempts < local_db_writer_lock_retries) : (attempts += 1) { + return writer_lock_mod.acquire(lock_path) catch |err| switch (err) { + error.WriterLocked => { + platform.time.sleepNs(local_db_writer_lock_sleep_ms * std.time.ns_per_ms); + continue; + }, + else => return err, + }; + } + return error.WriterLocked; +} + +fn launchSummaryPathAlloc( + alloc: std.mem.Allocator, + summary_dir: []const u8, + role: ChildRole, + round: usize, +) ![]u8 { + return try std.fmt.allocPrint(alloc, "{s}/graph-metric-launch-{s}-{d}.json", .{ + summary_dir, + switch (role) { + .coordinator => "coordinator", + .worker_pool => "worker-pool", + }, + round, + }); +} + +fn launchStderrPathAlloc( + alloc: std.mem.Allocator, + summary_dir: []const u8, + role: ChildRole, + round: usize, +) ![]u8 { + return try std.fmt.allocPrint(alloc, "{s}/graph-metric-launch-{s}-{d}.stderr", .{ + summary_dir, + switch (role) { + .coordinator => "coordinator", + .worker_pool => "worker-pool", + }, + round, + }); +} + +fn childRunSummaryFromFile( + alloc: std.mem.Allocator, + io: std.Io, + term: std.process.Child.Term, + summary_path: []const u8, + stderr_path: []const u8, +) !ChildRunSummary { + const stderr_bytes = try childOutputFileSize(alloc, io, stderr_path); + const raw = std.Io.Dir.cwd().readFileAlloc(io, summary_path, alloc, .limited(64 * 1024)) catch |err| switch (err) { + error.FileNotFound => return .{ + .exit = childExitFromTerm(term), + .durable_progressed = null, + .telemetry = null, + .stdout_bytes = 0, + .stderr_bytes = stderr_bytes, + }, + else => return err, + }; + defer alloc.free(raw); + return .{ + .exit = childExitFromTerm(term), + .durable_progressed = parseChildDurableProgressed(alloc, raw) catch null, + .telemetry = parseChildRuntimeTelemetry(alloc, raw) catch null, + .stdout_bytes = raw.len, + .stderr_bytes = stderr_bytes, + }; +} + +fn childOutputFileSize(alloc: std.mem.Allocator, io: std.Io, path: []const u8) !usize { + const raw = std.Io.Dir.cwd().readFileAlloc(io, path, alloc, .limited(64 * 1024)) catch |err| switch (err) { + error.FileNotFound => return 0, + else => return err, + }; + defer alloc.free(raw); + return raw.len; +} + +fn reportLaunchChildFailure( + alloc: std.mem.Allocator, + io: std.Io, + label: []const u8, + summary: ChildRunSummary, + stderr_path: []const u8, +) void { + if (childRunSucceeded(summary)) return; + const raw = std.Io.Dir.cwd().readFileAlloc(io, stderr_path, alloc, .limited(64 * 1024)) catch return; + defer alloc.free(raw); + if (raw.len == 0) return; + std.debug.print("graph metric launched {s} stderr:\n{s}\n", .{ label, raw }); +} + +fn childExitFromTerm(term: std.process.Child.Term) ChildExitSummary { + return switch (term) { + .exited => |code| .{ .exited = true, .code = code }, + else => .{ .exited = false, .code = null }, + }; +} + +fn childExitSucceeded(summary: ChildExitSummary) bool { + return summary.exited and summary.code != null and summary.code.? == 0; +} + +fn childRunSucceeded(summary: ChildRunSummary) bool { + return childExitSucceeded(summary.exit); +} + +fn childRunDurableProgressed(summary: ChildRunSummary) bool { + return summary.durable_progressed orelse false; +} + +fn parseChildDurableProgressed(alloc: std.mem.Allocator, stdout: []const u8) !bool { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, stdout, .{}); + defer parsed.deinit(); + const object = switch (parsed.value) { + .object => |object| object, + else => return error.InvalidArguments, + }; + const value = object.get("durable_progressed") orelse return error.InvalidArguments; + return switch (value) { + .bool => |b| b, + else => error.InvalidArguments, + }; +} + +fn parseChildRuntimeTelemetry(alloc: std.mem.Allocator, stdout: []const u8) !ChildRuntimeTelemetry { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, stdout, .{}); + defer parsed.deinit(); + const object = switch (parsed.value) { + .object => |object| object, + else => return error.InvalidArguments, + }; + const stats = switch (object.get("stats") orelse return error.InvalidArguments) { + .object => |stats| stats, + else => return error.InvalidArguments, + }; + return .{ + .role = try parseJsonRole(stats.get("role") orelse return error.InvalidArguments), + .runtime_id_hash = try parseJsonU64(stats.get("runtime_id_hash") orelse return error.InvalidArguments), + .owner_id_hash = try parseJsonU64(stats.get("owner_id_hash") orelse return error.InvalidArguments), + .lease_key_hash = try parseJsonU64(stats.get("lease_key_hash") orelse return error.InvalidArguments), + .worker_id_hash = try parseJsonU64(stats.get("worker_id_hash") orelse return error.InvalidArguments), + .worker_count = try parseJsonUsize(stats.get("worker_count") orelse return error.InvalidArguments), + .lease_owned = try parseJsonBool(stats.get("lease_owned") orelse return error.InvalidArguments), + .has_lease = try parseJsonBool(stats.get("has_lease") orelse return error.InvalidArguments), + .acquisition_count = try parseJsonU64(stats.get("acquisition_count") orelse return error.InvalidArguments), + .takeover_count = try parseJsonU64(stats.get("takeover_count") orelse return error.InvalidArguments), + .lost_leases = try parseJsonU64(stats.get("lost_leases") orelse return error.InvalidArguments), + .ticks_started = try parseJsonU64(stats.get("ticks_started") orelse return error.InvalidArguments), + .ticks_completed = try parseJsonU64(stats.get("ticks_completed") orelse return error.InvalidArguments), + .idle_ticks = try parseJsonU64(stats.get("idle_ticks") orelse return error.InvalidArguments), + .error_ticks = try parseJsonU64(stats.get("error_ticks") orelse return error.InvalidArguments), + .has_last_error = switch (stats.get("last_error_name") orelse return error.InvalidArguments) { + .null => false, + else => true, + }, + }; +} + +fn telemetryFromRunSummary(summary: RunSummary) ChildRuntimeTelemetry { + return .{ + .role = summary.stats.role, + .runtime_id_hash = summary.stats.runtime_id_hash, + .owner_id_hash = summary.stats.owner_id_hash, + .lease_key_hash = summary.stats.lease_key_hash, + .worker_id_hash = summary.stats.worker_id_hash, + .worker_count = summary.stats.worker_count, + .lease_owned = summary.stats.lease_owned, + .has_lease = summary.stats.has_lease, + .acquisition_count = summary.stats.acquisition_count, + .takeover_count = summary.stats.takeover_count, + .lost_leases = summary.stats.lost_leases, + .ticks_started = summary.stats.ticks_started, + .ticks_completed = summary.stats.ticks_completed, + .idle_ticks = summary.stats.idle_ticks, + .error_ticks = summary.stats.error_ticks, + .has_last_error = summary.stats.last_error_name != null, + }; +} + +fn parseJsonRole(value: std.json.Value) !RuntimeRole { + return switch (value) { + .string => |raw| parseRole(raw), + else => error.InvalidArguments, + }; +} + +fn parseJsonUsize(value: std.json.Value) !usize { + return @intCast(try parseJsonU64(value)); +} + +fn parseJsonU64(value: std.json.Value) !u64 { + return switch (value) { + .integer => |int| if (int >= 0) @intCast(int) else error.InvalidArguments, + .number_string => |raw| std.fmt.parseInt(u64, raw, 10) catch return error.InvalidArguments, + else => error.InvalidArguments, + }; +} + +fn parseJsonBool(value: std.json.Value) !bool { + return switch (value) { + .bool => |b| b, + else => error.InvalidArguments, + }; +} + +fn shouldRestartSupervisorAttempt(failed: bool, restarts: usize, max_restarts: usize) bool { + return failed and restarts < max_restarts; +} + +fn writeJson(io: std.Io, alloc: std.mem.Allocator, value: anytype) !void { + const json = try std.json.Stringify.valueAlloc(alloc, value, .{ .whitespace = .indent_2 }); + defer alloc.free(json); + std.Io.File.stdout().writeStreamingAll(io, json) catch {}; + std.Io.File.stdout().writeStreamingAll(io, "\n") catch {}; +} + +fn writeJsonFile(io: std.Io, alloc: std.mem.Allocator, path: []const u8, value: anytype) !void { + const json = try std.json.Stringify.valueAlloc(alloc, value, .{ .whitespace = .indent_2 }); + defer alloc.free(json); + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = path, + .data = json, + }); +} + +fn printUsage(argv0: []const u8) void { + std.debug.print( + \\usage: {s} graph-metric-maintenance (--db-path | --base-uri --group-id --table-name ) [options] + \\ + \\options: + \\ --base-uri + \\ --service-base-uri + \\ --group-id + \\ --table-name
+ \\ --role + \\ --runtime-id + \\ --owner-id logical owner label (a process-incarnation fence is appended) + \\ --worker-id + \\ --worker-ids + \\ --lease-owned + \\ --lease-ttl-ms + \\ --coordinator-start-background-builds + \\ --ticks + \\ --until-idle + \\ --max-idle-ticks + \\ --tick-ms + \\ --max-rounds + \\ --max-metrics + \\ --max-pages + \\ --test-now-ms + \\ --test-ready-file + \\ --test-hold-after-run-ms + \\ --summary-file + \\ --local-db-writer-lock + \\ + \\subcommands: + \\ supervise + \\ launch + \\ + , .{argv0}); +} + +fn printSupervisorUsage(argv0: []const u8) void { + std.debug.print( + \\usage: {s} graph-metric-maintenance supervise (--db-path | --base-uri --group-id --table-name
) [options] + \\ + \\options: + \\ --base-uri + \\ --service-base-uri + \\ --group-id + \\ --table-name
+ \\ --executable + \\ --coordinator-owner-id + \\ --worker-pool-owner-id + \\ --worker-ids + \\ --lease-ttl-ms + \\ --ticks + \\ --max-idle-ticks + \\ --supervisor-rounds + \\ --supervisor-idle-rounds + \\ --tick-ms + \\ --max-restarts + \\ --max-rounds + \\ --max-metrics + \\ --max-pages + \\ --summary-dir + \\ + , .{argv0}); +} + +fn printLaunchUsage(argv0: []const u8) void { + std.debug.print( + \\usage: {s} graph-metric-maintenance launch (--db-path | --base-uri --group-id --table-name
) [options] + \\ + \\options: + \\ --base-uri + \\ --service-base-uri + \\ --group-id + \\ --table-name
+ \\ --executable + \\ --coordinator-owner-id + \\ --worker-pool-owner-id + \\ --worker-ids + \\ --lease-ttl-ms + \\ --ticks + \\ --max-idle-ticks + \\ --supervisor-rounds + \\ --supervisor-idle-rounds + \\ --tick-ms + \\ --max-restarts + \\ --max-rounds + \\ --max-metrics + \\ --max-pages + \\ --summary-dir + \\ + , .{argv0}); +} + +fn argvContains(argv: []const []const u8, needle: []const u8) bool { + for (argv) |arg| { + if (std.mem.eql(u8, arg, needle)) return true; + } + return false; +} + +fn argvValueEquals(argv: []const []const u8, flag: []const u8, value: []const u8) bool { + for (argv, 0..) |arg, i| { + if (std.mem.eql(u8, arg, flag)) { + return i + 1 < argv.len and std.mem.eql(u8, argv[i + 1], value); + } + } + return false; +} + +fn argvContainsAny(argv: []const []const u8, needles: []const []const u8) bool { + for (needles) |needle| { + if (argvContains(argv, needle)) return true; + } + return false; +} + +const FakeSupervisorChildRunner = struct { + sequence: []const ChildRunSummary, + calls: usize = 0, + + fn run( + self: *FakeSupervisorChildRunner, + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, + ) !ChildRunSummary { + _ = alloc; + _ = io; + _ = argv; + if (self.calls >= self.sequence.len) return error.UnexpectedEndOfStream; + const result = self.sequence[self.calls]; + self.calls += 1; + return result; + } +}; + +const FakeServiceMaintenanceClient = struct { + responses: []const []const u8, + calls: usize = 0, + first_captured_body: ?[]u8 = null, + captured_body: ?[]u8 = null, + captured_target: ?ServiceTarget = null, + expect_ready_file_before_release: ?[]const u8 = null, + + fn deinit(self: *FakeServiceMaintenanceClient, alloc: std.mem.Allocator) void { + if (self.first_captured_body) |body| alloc.free(body); + if (self.captured_body) |body| alloc.free(body); + self.* = undefined; + } + + fn request( + self: *FakeServiceMaintenanceClient, + alloc: std.mem.Allocator, + target: ServiceTarget, + body: []const u8, + ) !HttpQueryResponse { + if (self.calls >= self.responses.len) return error.UnexpectedEndOfStream; + if (self.first_captured_body == null) { + self.first_captured_body = try alloc.dupe(u8, body); + } + if (self.captured_body) |prior| alloc.free(prior); + self.captured_body = try alloc.dupe(u8, body); + self.captured_target = target; + if (self.expect_ready_file_before_release) |ready_file| { + if (std.mem.indexOf(u8, body, "\"action\":\"release\"") != null) { + try std.Io.Dir.cwd().access(std.Io.Threaded.global_single_threaded.io(), ready_file, .{}); + } + } + const response_body = try alloc.dupe(u8, self.responses[self.calls]); + self.calls += 1; + return .{ .body = response_body }; + } +}; + +fn fakeChild(progressed: ?bool, code: u8) ChildRunSummary { + return .{ + .exit = .{ .exited = true, .code = code }, + .durable_progressed = progressed, + .telemetry = null, + .stdout_bytes = if (progressed == null) 0 else 64, + .stderr_bytes = 0, + }; +} + +const LoopbackSupervisorChildRunner = struct { + calls: usize = 0, + + fn run( + self: *LoopbackSupervisorChildRunner, + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, + ) !ChildRunSummary { + _ = io; + if (argv.len < 3) return error.InvalidArguments; + if (!std.mem.eql(u8, argv[1], process_subcommand)) return error.InvalidArguments; + const argv_z = try alloc.alloc([*:0]const u8, argv.len - 2); + defer alloc.free(argv_z); + var initialized: usize = 0; + defer for (argv_z[0..initialized]) |arg| alloc.free(std.mem.span(arg)); + for (argv[2..], argv_z) |arg, *arg_z| { + arg_z.* = (try alloc.dupeZ(u8, arg)).ptr; + initialized += 1; + } + var args = std.process.Args.Iterator.init(.{ .vector = argv_z }); + var cli = try parseCli(alloc, &args); + defer cli.deinit(alloc); + const db_path = cli.db_path orelse return error.InvalidArguments; + const summary = try runConfigured(alloc, db_path, cli); + self.calls += 1; + return .{ + .exit = .{ .exited = true, .code = 0 }, + .durable_progressed = summary.durable_progressed, + .telemetry = telemetryFromRunSummary(summary), + .stdout_bytes = 64, + .stderr_bytes = 0, + }; + } +}; + +const InProcessGraphMetricService = struct { + db: *antfly.db.DB, + bound: antfly.public_api.BoundTableWriteSource, + calls: usize = 0, + + fn init(db: *antfly.db.DB) InProcessGraphMetricService { + return .{ + .db = db, + .bound = antfly.public_api.BoundTableWriteSource.init("docs", db), + }; + } + + fn request( + self: *InProcessGraphMetricService, + alloc: std.mem.Allocator, + target: ServiceTarget, + body: []const u8, + ) !HttpQueryResponse { + const response_body = (try self.bound.source().graphMetricMaintenanceGroupLocal( + alloc, + target.group_id, + target.table_name, + body, + )) orelse return error.InvalidArguments; + self.calls += 1; + return .{ + .content_type = try alloc.dupe(u8, "application/json"), + .body = response_body, + }; + } + + fn validateBatch(_: *anyopaque, _: []const u8, _: []const antfly.db.types.BatchWrite) !void {} + + fn validateTxn(_: *anyopaque, _: []const u8, _: []const antfly.db.types.TransactionWrite) !void {} +}; + +fn requestInProcessServiceMaintenanceForTest( + alloc: std.mem.Allocator, + service: *InProcessGraphMetricService, + target: ServiceTarget, + cli: CliConfig, + action: ServiceMaintenanceAction, +) !ServiceMaintenanceResponseWire { + const body = try serviceRequestJsonFromFieldsAlloc( + alloc, + cli, + action, + cli.role, + cli.worker_id, + cli.worker_ids.items, + cli.coordinator_start_background_builds, + cli.max_rounds, + cli.max_metrics_per_round, + cli.max_pages_per_round, + cli.test_now_ms, + ); + defer alloc.free(body); + var response = try service.request(alloc, target, body); + defer response.deinit(alloc); + return try parseServiceMaintenanceResponse(alloc, response.body); +} + +// Cold iterative admission is a coordinator/worker protocol, not one tick. +// Stop immediately after numerical admission so freshness and takeover tests +// still observe an active, unexecuted numerical job. +fn startInProcessServiceBuildForTest( + alloc: std.mem.Allocator, + service: *InProcessGraphMetricService, + target: ServiceTarget, + coordinator: CliConfig, +) !ServiceMaintenanceResponseWire { + var worker = CliConfig{ + .role = .worker_pool, + .runtime_id = "test-topology-preparation", + .owner_id = "test-topology-preparation", + .coordinator_start_background_builds = false, + .max_rounds = 1, + .max_pages_per_round = 1, + .test_now_ms = coordinator.test_now_ms, + }; + defer worker.deinit(alloc); + try worker.worker_ids.append(alloc, "test-topology-worker"); + for (0..256) |_| { + const response = try requestInProcessServiceMaintenanceForTest(alloc, service, target, coordinator, .tick); + if (response.result.builds_started > 0) return response; + _ = try requestInProcessServiceMaintenanceForTest(alloc, service, target, worker, .tick); + } + return error.GraphMetricBuildNotStarted; +} + +fn expectParseCliInvalid(alloc: std.mem.Allocator, argv: []const [*:0]const u8) !void { + var args = std.process.Args.Iterator.init(.{ .vector = argv }); + try std.testing.expectError(error.InvalidArguments, parseCli(alloc, &args)); +} + +fn expectParseSupervisorInvalid(alloc: std.mem.Allocator, argv: []const [*:0]const u8) !void { + var args = std.process.Args.Iterator.init(.{ .vector = argv }); + try std.testing.expectError(error.InvalidArguments, parseSupervisorCli(alloc, &args)); +} + +test "graph metric maintenance service credentials fail closed before requesting work" { + var environ = std.process.Environ.Map.init(std.testing.allocator); + defer environ.deinit(); + try std.testing.expectError(error.InternalServiceSecretMissing, serviceCredentials(&environ)); + try environ.put("ANTFLY_INTERNAL_SERVICE_SECRET", "short"); + try std.testing.expectError(error.InternalServiceSecretTooShort, serviceCredentials(&environ)); + try environ.put("ANTFLY_INTERNAL_SERVICE_SECRET", "0123456789abcdef0123456789abcdef"); + try std.testing.expectError(error.InternalServiceIssuerMissing, serviceCredentials(&environ)); + try environ.put("ANTFLY_INTERNAL_SERVICE_ISSUER", "cluster-a"); + const credentials = try serviceCredentials(&environ); + try std.testing.expectEqualStrings("cluster-a", credentials.issuer); + try std.testing.expectEqualStrings("0123456789abcdef0123456789abcdef", credentials.secret); +} + +test "graph metric maintenance command parses worker pool config" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--role", "worker-pool", + "--runtime-id", "runtime-a", + "--owner-id", "owner-a", + "--worker-ids", "worker-a,worker-b", + "--ticks", "7", + "--until-idle", "--max-idle-ticks", + "3", "--tick-ms", + "5", "--max-pages", + "2", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + var parsed = try parseCli(alloc, &args); + defer parsed.deinit(alloc); + + try std.testing.expectEqual(RuntimeRole.worker_pool, parsed.role); + try std.testing.expectEqual(@as(usize, 7), parsed.max_ticks); + try std.testing.expect(parsed.until_idle); + try std.testing.expectEqual(@as(usize, 3), parsed.max_idle_ticks.?); + try std.testing.expectEqual(@as(u64, 5), parsed.tick_interval_ms); + try std.testing.expectEqual(@as(usize, 2), parsed.worker_ids.items.len); + try std.testing.expectEqualStrings("worker-a", parsed.worker_ids.items[0]); + try std.testing.expectEqualStrings("worker-b", parsed.worker_ids.items[1]); + const runtime_cfg = parsed.runtimeConfig(platform_clock.Clock.real()); + try std.testing.expect(runtime_cfg.enabled); + try std.testing.expect(!runtime_cfg.start_background_loop); + try std.testing.expectEqual(RuntimeRole.worker_pool, runtime_cfg.role); + try std.testing.expectEqual(@as(usize, 2), runtime_cfg.planned_options.worker_ids.len); + try std.testing.expectEqual(@as(usize, 2), runtime_cfg.planned_options.max_pages_per_round); +} + +test "graph metric maintenance command parses service target config" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--base-uri", "http://127.0.0.1:8080", + "--group-id", "7", + "--table-name", "docs", + "--role", "worker-pool", + "--runtime-id", "runtime-a", + "--owner-id", "owner-a", + "--worker-ids", "worker-a,worker-b", + "--max-pages", "3", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + var parsed = try parseCli(alloc, &args); + defer parsed.deinit(alloc); + + const target = (try parsed.serviceTarget()) orelse return error.MissingServiceTarget; + try std.testing.expectEqualStrings("http://127.0.0.1:8080", target.base_uri); + try std.testing.expectEqual(@as(u64, 7), target.group_id); + try std.testing.expectEqualStrings("docs", target.table_name); + try std.testing.expectEqual(RuntimeRole.worker_pool, parsed.role); + try std.testing.expectEqual(@as(usize, 2), parsed.worker_ids.items.len); + try std.testing.expectEqual(@as(usize, 3), parsed.max_pages_per_round); +} + +test "graph metric maintenance command rejects invalid service target combinations" { + const alloc = std.testing.allocator; + + try expectParseCliInvalid(alloc, &.{ + "--db-path", "/tmp/db", + "--base-uri", "http://127.0.0.1:8080", + "--group-id", "7", + "--table-name", "docs", + }); + try expectParseCliInvalid(alloc, &.{ + "--base-uri", "http://127.0.0.1:8080", + "--group-id", "7", + }); + try expectParseCliInvalid(alloc, &.{ + "--base-uri", "http://127.0.0.1:8080", + "--table-name", "docs", + }); +} + +test "graph metric maintenance service request stays owner and budget scoped" { + const alloc = std.testing.allocator; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + + const body = try serviceRequestJsonAlloc(alloc, .{ + .role = .worker_pool, + .runtime_id = "runtime-a", + .owner_id = "owner-a", + .worker_id = "unused-worker", + .worker_ids = worker_ids, + .coordinator_start_background_builds = false, + .max_rounds = 2, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + .test_now_ms = 42, + }); + defer alloc.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, body, .{}); + defer parsed.deinit(); + const object = switch (parsed.value) { + .object => |object| object, + else => return error.InvalidArguments, + }; + try std.testing.expectEqualStrings("worker_pool", object.get("role").?.string); + try std.testing.expectEqualStrings("tick", object.get("action").?.string); + try std.testing.expectEqualStrings("unused-worker", object.get("worker_id").?.string); + try std.testing.expectEqual(@as(usize, 2), object.get("worker_ids").?.array.items.len); + try std.testing.expect(!object.get("start_background_builds").?.bool); + try std.testing.expectEqual(@as(i64, 2), object.get("max_rounds").?.integer); + try std.testing.expectEqual(@as(i64, 4), object.get("max_metrics_per_round").?.integer); + try std.testing.expectEqual(@as(i64, 3), object.get("max_pages_per_round").?.integer); + try std.testing.expectEqual(@as(i64, 42), object.get("now_ms").?.integer); + try std.testing.expectEqualStrings("runtime-a", object.get("runtime_id").?.string); + try std.testing.expectEqualStrings("owner-a", object.get("owner_id").?.string); + try std.testing.expect(object.get("lease_owned").?.bool); + try std.testing.expectEqual(@as(i64, 30_000), object.get("lease_ttl_ms").?.integer); + try std.testing.expect(object.get("metric_name") == null); + try std.testing.expect(object.get("target_generation") == null); + try std.testing.expect(object.get("job_id") == null); + try std.testing.expect(object.get("page_id") == null); +} + +test "graph metric maintenance service runner aggregates remote ticks" { + const alloc = std.testing.allocator; + const runtime_hash = std.hash.Wyhash.hash(0, "service-runtime"); + const owner_hash = std.hash.Wyhash.hash(0, "service-owner"); + const response_a = try std.fmt.allocPrint( + alloc, + "{{\"result\":{{\"metrics_scanned\":1,\"builds_started\":1,\"budget_exhausted\":true}},\"stats\":{{\"enabled\":true,\"role\":\"coordinator\",\"runtime_id_hash\":{d},\"owner_id_hash\":{d},\"lease_key_hash\":1,\"worker_id_hash\":0,\"worker_count\":0,\"lease_owned\":true,\"has_lease\":true,\"acquisition_count\":1,\"takeover_count\":0,\"lease_acquire_failures\":0,\"lost_leases\":0,\"last_acquired_ms\":1000,\"started\":false,\"shutdown\":false,\"notified\":false,\"ticks_started\":1,\"ticks_completed\":1,\"durable_progress_ticks\":1,\"idle_ticks\":0,\"error_ticks\":0,\"last_error_name\":null,\"total_result\":{{\"metrics_scanned\":1,\"builds_started\":1,\"budget_exhausted\":true}},\"last_result\":{{\"metrics_scanned\":1,\"builds_started\":1,\"budget_exhausted\":true}}}}}}", + .{ runtime_hash, owner_hash }, + ); + defer alloc.free(response_a); + const response_b = try std.fmt.allocPrint( + alloc, + "{{\"result\":{{\"metrics_scanned\":1}},\"stats\":{{\"enabled\":true,\"role\":\"coordinator\",\"runtime_id_hash\":{d},\"owner_id_hash\":{d},\"lease_key_hash\":1,\"worker_id_hash\":0,\"worker_count\":0,\"lease_owned\":true,\"has_lease\":true,\"acquisition_count\":1,\"takeover_count\":0,\"lease_acquire_failures\":0,\"lost_leases\":0,\"last_acquired_ms\":1010,\"started\":false,\"shutdown\":false,\"notified\":false,\"ticks_started\":1,\"ticks_completed\":1,\"durable_progress_ticks\":0,\"idle_ticks\":1,\"error_ticks\":0,\"last_error_name\":null,\"total_result\":{{\"metrics_scanned\":1}},\"last_result\":{{\"metrics_scanned\":1}}}}}}", + .{ runtime_hash, owner_hash }, + ); + defer alloc.free(response_b); + const responses = [_][]const u8{ + response_a, + response_b, + "{\"released\":true,\"stats\":{\"enabled\":true,\"role\":\"coordinator\",\"lease_owned\":true,\"has_lease\":false}}", + }; + var client = FakeServiceMaintenanceClient{ .responses = responses[0..] }; + defer client.deinit(alloc); + + const target = ServiceTarget{ + .base_uri = "http://127.0.0.1:8080", + .group_id = 7, + .table_name = "docs", + }; + const summary = try runServiceConfiguredWithRequester( + FakeServiceMaintenanceClient, + &client, + FakeServiceMaintenanceClient.request, + alloc, + target, + .{ + .role = .coordinator, + .runtime_id = "service-runtime", + .owner_id = "service-owner", + .lease_owned = true, + .worker_id = "service-unused-worker", + .max_ticks = 10, + .until_idle = true, + .max_idle_ticks = 1, + .max_metrics_per_round = 4, + }, + ); + + try std.testing.expectEqual(@as(usize, 3), client.calls); + try std.testing.expectEqualStrings(target.base_uri, client.captured_target.?.base_uri); + try std.testing.expectEqual(target.group_id, client.captured_target.?.group_id); + try std.testing.expectEqualStrings(target.table_name, client.captured_target.?.table_name); + try std.testing.expectEqual(RuntimeRole.coordinator, summary.role); + try std.testing.expectEqual(@as(usize, 2), summary.ticks_executed); + try std.testing.expectEqual(@as(usize, 1), summary.idle_streak); + try std.testing.expectEqual(ExitReason.idle, summary.exit_reason); + try std.testing.expect(summary.durable_progressed); + try std.testing.expectEqual(@as(usize, 2), summary.result.metrics_scanned); + try std.testing.expectEqual(@as(usize, 1), summary.result.builds_started); + try std.testing.expect(summary.result.budget_exhausted); + try std.testing.expect(summary.stats.enabled); + try std.testing.expectEqual(RuntimeRole.coordinator, summary.stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "service-runtime"), summary.stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "service-owner"), summary.stats.owner_id_hash); + try std.testing.expect(summary.stats.lease_owned); + try std.testing.expect(!summary.stats.has_lease); + try std.testing.expect(summary.stats.shutdown); + try std.testing.expectEqual(@as(u64, 2), summary.stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 2), summary.stats.ticks_started); + try std.testing.expectEqual(@as(u64, 2), summary.stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), summary.stats.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), summary.stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), summary.stats.error_ticks); +} + +test "graph metric maintenance service boundary preserves worker pool owner request" { + const alloc = std.testing.allocator; + const responses = [_][]const u8{ + "{\"worker_steps\":2,\"pages_completed\":2}", + "{\"released\":true,\"stats\":{\"enabled\":true,\"role\":\"worker_pool\",\"worker_count\":2,\"lease_owned\":true,\"has_lease\":false}}", + }; + var client = FakeServiceMaintenanceClient{ .responses = responses[0..] }; + defer client.deinit(alloc); + + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + + const target = ServiceTarget{ + .base_uri = "http://127.0.0.1:8080", + .group_id = 7, + .table_name = "docs", + }; + const summary = try runServiceConfiguredWithRequester( + FakeServiceMaintenanceClient, + &client, + FakeServiceMaintenanceClient.request, + alloc, + target, + .{ + .role = .worker_pool, + .runtime_id = "service-worker-pool-runtime", + .owner_id = "service-worker-pool-owner", + .lease_owned = true, + .worker_id = "unused-worker", + .worker_ids = worker_ids, + .max_ticks = 1, + .max_pages_per_round = 5, + .test_now_ms = 700, + }, + ); + + try std.testing.expectEqual(@as(usize, 2), client.calls); + try std.testing.expectEqual(RuntimeRole.worker_pool, summary.role); + try std.testing.expectEqual(@as(usize, 2), summary.result.worker_steps); + try std.testing.expectEqual(@as(usize, 2), summary.result.pages_completed); + try std.testing.expectEqual(RuntimeRole.worker_pool, summary.stats.role); + try std.testing.expectEqual(@as(usize, 2), summary.stats.worker_count); + try std.testing.expect(summary.stats.lease_owned); + try std.testing.expect(!summary.stats.has_lease); + try std.testing.expect(summary.stats.shutdown); + + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, client.first_captured_body.?, .{}); + defer parsed.deinit(); + const object = switch (parsed.value) { + .object => |object| object, + else => return error.InvalidArguments, + }; + try std.testing.expectEqualStrings("tick", object.get("action").?.string); + try std.testing.expectEqualStrings("worker_pool", object.get("role").?.string); + try std.testing.expectEqualStrings("unused-worker", object.get("worker_id").?.string); + try std.testing.expectEqual(@as(usize, 2), object.get("worker_ids").?.array.items.len); + try std.testing.expectEqual(@as(i64, 5), object.get("max_pages_per_round").?.integer); + try std.testing.expectEqual(@as(i64, 700), object.get("now_ms").?.integer); + try std.testing.expect(object.get("metric_name") == null); + try std.testing.expect(object.get("job_id") == null); + try std.testing.expect(object.get("page_id") == null); + + var release_parsed = try std.json.parseFromSlice(std.json.Value, alloc, client.captured_body.?, .{}); + defer release_parsed.deinit(); + const release_object = switch (release_parsed.value) { + .object => |release_object| release_object, + else => return error.InvalidArguments, + }; + try std.testing.expectEqualStrings("release", release_object.get("action").?.string); + try std.testing.expectEqualStrings("worker_pool", release_object.get("role").?.string); + try std.testing.expectEqual(@as(usize, 2), release_object.get("worker_ids").?.array.items.len); +} + +test "graph metric maintenance service test ready marker is written before clean release" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var ready_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const ready_path = try std.fmt.bufPrint(&ready_path_buf, ".zig-cache/tmp/{s}/graph-metric-service-ready-before-release", .{tmp.sub_path}); + std.Io.Dir.cwd().deleteFile(std.Io.Threaded.global_single_threaded.io(), ready_path) catch {}; + defer std.Io.Dir.cwd().deleteFile(std.Io.Threaded.global_single_threaded.io(), ready_path) catch {}; + + const responses = [_][]const u8{ + "{\"metrics_scanned\":1}", + "{\"released\":true,\"stats\":{\"enabled\":true,\"role\":\"coordinator\",\"lease_owned\":true,\"has_lease\":false}}", + }; + var client = FakeServiceMaintenanceClient{ + .responses = responses[0..], + .expect_ready_file_before_release = ready_path, + }; + defer client.deinit(alloc); + + const target = ServiceTarget{ + .base_uri = "http://127.0.0.1:8080", + .group_id = 7, + .table_name = "docs", + }; + const summary = try runServiceConfiguredWithRequester( + FakeServiceMaintenanceClient, + &client, + FakeServiceMaintenanceClient.request, + alloc, + target, + .{ + .role = .coordinator, + .runtime_id = "service-ready-coordinator", + .owner_id = "service-ready-coordinator", + .lease_owned = true, + .worker_id = "service-ready-unused", + .max_ticks = 1, + .max_metrics_per_round = 4, + .test_ready_file = ready_path, + }, + ); + + try std.testing.expectEqual(@as(usize, 2), client.calls); + try std.testing.expect(summary.stats.shutdown); + try std.testing.expect(!summary.stats.has_lease); + + var release_parsed = try std.json.parseFromSlice(std.json.Value, alloc, client.captured_body.?, .{}); + defer release_parsed.deinit(); + const release_object = switch (release_parsed.value) { + .object => |release_object| release_object, + else => return error.InvalidArguments, + }; + try std.testing.expectEqualStrings("release", release_object.get("action").?.string); +} + +test "graph metric maintenance service owners drain degree through internal route" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-degree-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "service-worker-a", "service-worker-b" }); + + var idle_rounds: usize = 0; + var coordinator_summary = RunSummary{ + .role = .coordinator, + .ticks_executed = 0, + .idle_streak = 0, + .exit_reason = .max_ticks, + .durable_progressed = false, + .result = .{}, + .stats = .{}, + }; + var worker_summary = coordinator_summary; + for (0..80) |_| { + coordinator_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + .{ + .role = .coordinator, + .runtime_id = "service-degree-coordinator", + .owner_id = "service-degree-coordinator", + .lease_owned = true, + .worker_id = "service-degree-coordinator-unused", + .max_ticks = 1, + .max_idle_ticks = 1, + .tick_interval_ms = 0, + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + }, + ); + worker_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + .{ + .role = .worker_pool, + .runtime_id = "service-degree-worker-pool", + .owner_id = "service-degree-worker-pool", + .lease_owned = true, + .worker_id = "service-degree-worker-unused", + .worker_ids = worker_ids, + .coordinator_start_background_builds = false, + .max_ticks = 1, + .max_idle_ticks = 1, + .tick_interval_ms = 0, + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + }, + ); + + const progressed = coordinator_summary.durable_progressed or worker_summary.durable_progressed; + idle_rounds = if (progressed) 0 else idle_rounds + 1; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(coordinator_summary.stats.shutdown); + try std.testing.expect(worker_summary.stats.shutdown); + try std.testing.expect(!coordinator_summary.stats.has_lease); + try std.testing.expect(!worker_summary.stats.has_lease); + try std.testing.expect(service.calls >= 4); + return; + } + if (idle_rounds >= 4) break; + } + + return error.GraphMetricBuildNotComplete; +} + +test "graph metric maintenance service owners preserve degree freshness while rebuild is active" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-degree-active-freshness-db", .{tmp.sub_path}); + + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try db.runUntilIdle(); + + var initial = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer initial.deinit(); + try std.testing.expectEqual(@as(usize, 1), initial.graph_metric_results.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[0].status.state); + const published_generation = initial.graph_metric_results[0].status.published_generation; + try std.testing.expect(published_generation > 0); + try std.testing.expectEqualStrings("doc:b", initial.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), initial.graph_metric_results[0].scores[0].score, 0.001); + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:d", + .value = "{\"title\":\"delta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + const start = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + .{ + .role = .coordinator, + .runtime_id = "service-degree-freshness-coordinator", + .owner_id = "service-degree-freshness-coordinator", + .worker_id = "service-degree-freshness-unused", + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 1, + }, + .tick, + ); + try std.testing.expect(start.result.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), start.result.builds_started); + + var published = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published.deinit(); + try std.testing.expectEqual(@as(usize, 1), published.graph_metric_results.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, published.graph_metric_results[0].status.state); + try std.testing.expectEqual(published_generation, published.graph_metric_results[0].status.published_generation); + try std.testing.expect(published.graph_metric_results[0].status.building_generation > published_generation); + try std.testing.expectEqualStrings("doc:b", published.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), published.graph_metric_results[0].scores[0].score, 0.001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .published, + }}; + const traversal_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = traversal_query }}, + .limit = 0, + }); + defer traversal.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", traversal.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes[0].metrics.len); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), traversal.graph_results[0].nodes[0].metrics[0].score orelse return error.TestUnexpectedResult, 0.001); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].metric_status.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, traversal.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(published_generation, traversal.graph_results[0].metric_status[0].published_generation); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .fresh, + }}; + var fresh_traversal_query = traversal_query; + fresh_traversal_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_traversal_query }}, + .limit = 0, + })); + + var rerank = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank.deinit(); + try std.testing.expectEqualStrings("doc:b", rerank.hits[0].id); + const rerank_status = rerank.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, rerank_status.state); + try std.testing.expectEqual(published_generation, rerank_status.published_generation); + const rerank_details = rerank.hits[0].score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("degree", rerank_details.metric_name); + try std.testing.expectEqual(published_generation, rerank_details.published_generation); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), rerank_details.metric_score orelse return error.TestUnexpectedResult, 0.001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +test "graph metric maintenance service owners fence abandoned degree leases and recover after ttl" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-degree-restart-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + + var coordinator_a = CliConfig{ + .role = .coordinator, + .runtime_id = "service-degree-restart-coordinator", + .owner_id = "coordinator-a", + .worker_id = "coordinator-unused", + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + .test_hold_after_run_ms = 1, + .test_now_ms = 1000, + }; + const coordinator_a_start = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_a, + .tick, + ); + const coordinator_a_start_stats = coordinator_a_start.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_a_start_stats.lease_owned); + try std.testing.expect(coordinator_a_start_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_a_start_stats.acquisition_count); + try std.testing.expect(coordinator_a_start.result.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), coordinator_a_start.result.builds_started); + + var coordinator_b = coordinator_a; + coordinator_b.owner_id = "coordinator-b"; + coordinator_b.test_now_ms = 1100; + const coordinator_b_fenced = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_b, + .tick, + ); + const coordinator_b_fenced_stats = coordinator_b_fenced.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_b_fenced_stats.lease_owned); + try std.testing.expect(!coordinator_b_fenced_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_b_fenced_stats.lease_acquire_failures); + try std.testing.expect(!coordinator_b_fenced.result.durableProgressed()); + + coordinator_b.test_now_ms = 1301; + const coordinator_b_takeover = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_b, + .tick, + ); + const coordinator_b_takeover_stats = coordinator_b_takeover.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_b_takeover_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_b_takeover_stats.takeover_count); + try std.testing.expectEqual(identityHash("coordinator-b"), coordinator_b_takeover_stats.owner_id_hash); + + coordinator_a.test_now_ms = 1302; + const stale_coordinator_release = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_a, + .release, + ); + try std.testing.expect(!stale_coordinator_release.released); + try std.testing.expect(stale_coordinator_release.lease_owner_id_hash != 0); + const stale_coordinator_release_stats = stale_coordinator_release.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!stale_coordinator_release_stats.has_lease); + + var worker_pool_a = CliConfig{ + .role = .worker_pool, + .runtime_id = "service-degree-restart-worker-pool", + .owner_id = "worker-pool-a", + .worker_id = "worker-pool-unused", + .coordinator_start_background_builds = false, + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + .test_hold_after_run_ms = 1, + .test_now_ms = 2000, + }; + defer worker_pool_a.deinit(alloc); + try worker_pool_a.worker_ids.appendSlice(alloc, &.{ "restart-worker-a", "restart-worker-b" }); + + const worker_pool_a_start = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_a, + .tick, + ); + const worker_pool_a_start_stats = worker_pool_a_start.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(worker_pool_a_start_stats.has_lease); + try std.testing.expectEqual(@as(usize, 2), worker_pool_a_start_stats.worker_count); + try std.testing.expect(worker_pool_a_start.result.durableProgressed()); + + var worker_pool_b = CliConfig{ + .role = .worker_pool, + .runtime_id = "service-degree-restart-worker-pool", + .owner_id = "worker-pool-b", + .worker_id = "worker-pool-unused", + .coordinator_start_background_builds = false, + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + .test_hold_after_run_ms = 1, + .test_now_ms = 2100, + }; + defer worker_pool_b.deinit(alloc); + try worker_pool_b.worker_ids.appendSlice(alloc, &.{ "restart-worker-a", "restart-worker-b" }); + + const worker_pool_b_fenced = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_b, + .tick, + ); + const worker_pool_b_fenced_stats = worker_pool_b_fenced.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!worker_pool_b_fenced_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), worker_pool_b_fenced_stats.lease_acquire_failures); + try std.testing.expect(!worker_pool_b_fenced.result.durableProgressed()); + + worker_pool_b.test_now_ms = 2301; + const worker_pool_b_takeover = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_b, + .tick, + ); + const worker_pool_b_takeover_stats = worker_pool_b_takeover.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(worker_pool_b_takeover_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), worker_pool_b_takeover_stats.takeover_count); + try std.testing.expectEqual(identityHash("worker-pool-b"), worker_pool_b_takeover_stats.owner_id_hash); + + worker_pool_a.test_now_ms = 2302; + const stale_worker_pool_release = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_a, + .release, + ); + try std.testing.expect(!stale_worker_pool_release.released); + try std.testing.expect(stale_worker_pool_release.lease_owner_id_hash != 0); + const stale_worker_pool_release_stats = stale_worker_pool_release.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!stale_worker_pool_release_stats.has_lease); + + var idle_rounds: usize = 0; + for (0..80) |round| { + coordinator_b.test_now_ms = 2400 + @as(u64, @intCast(round)); + const coordinator_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + coordinator_b, + ); + worker_pool_b.test_now_ms = 2500 + @as(u64, @intCast(round)); + const worker_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + worker_pool_b, + ); + + const progressed = coordinator_summary.durable_progressed or worker_summary.durable_progressed; + idle_rounds = if (progressed) 0 else idle_rounds + 1; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(coordinator_summary.stats.shutdown); + try std.testing.expect(worker_summary.stats.shutdown); + try std.testing.expect(!coordinator_summary.stats.has_lease); + try std.testing.expect(!worker_summary.stats.has_lease); + return; + } + if (idle_rounds >= 4) break; + } + + return error.GraphMetricBuildNotComplete; +} + +test "graph metric maintenance service owners drain pagerank through internal route" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-pagerank-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "service-pagerank-worker-a", "service-pagerank-worker-b" }); + + var idle_rounds: usize = 0; + var coordinator_summary = RunSummary{ + .role = .coordinator, + .ticks_executed = 0, + .idle_streak = 0, + .exit_reason = .max_ticks, + .durable_progressed = false, + .result = .{}, + .stats = .{}, + }; + var worker_summary = coordinator_summary; + for (0..160) |_| { + coordinator_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + .{ + .role = .coordinator, + .runtime_id = "service-pagerank-coordinator", + .owner_id = "service-pagerank-coordinator", + .lease_owned = true, + .worker_id = "service-pagerank-coordinator-unused", + .max_ticks = 1, + .max_idle_ticks = 1, + .tick_interval_ms = 0, + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + }, + ); + worker_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + .{ + .role = .worker_pool, + .runtime_id = "service-pagerank-worker-pool", + .owner_id = "service-pagerank-worker-pool", + .lease_owned = true, + .worker_id = "service-pagerank-worker-unused", + .worker_ids = worker_ids, + .coordinator_start_background_builds = false, + .max_ticks = 1, + .max_idle_ticks = 1, + .tick_interval_ms = 0, + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + }, + ); + + const progressed = coordinator_summary.durable_progressed or worker_summary.durable_progressed; + idle_rounds = if (progressed) 0 else idle_rounds + 1; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(coordinator_summary.stats.shutdown); + try std.testing.expect(worker_summary.stats.shutdown); + try std.testing.expect(!coordinator_summary.stats.has_lease); + try std.testing.expect(!worker_summary.stats.has_lease); + try std.testing.expect(service.calls >= 4); + const top = try graph_entry.index.graphMetricTopK("pagerank", 3); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expect(top.len > 0); + return; + } + if (idle_rounds >= 6) break; + } + + return error.GraphMetricBuildNotComplete; +} + +test "graph metric maintenance service owners preserve pagerank freshness while rebuild is active" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-pagerank-active-freshness-db", .{tmp.sub_path}); + + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try db.runUntilIdle(); + + var initial = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer initial.deinit(); + try std.testing.expectEqual(@as(usize, 1), initial.graph_metric_results.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[0].status.state); + const published_generation = initial.graph_metric_results[0].status.published_generation; + try std.testing.expect(published_generation > 0); + try std.testing.expect(initial.graph_metric_results[0].scores.len > 0); + const initial_top_node = initial.graph_metric_results[0].scores[0].node; + const initial_top_score = initial.graph_metric_results[0].scores[0].score; + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:e", + .value = "{\"title\":\"epsilon\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + const start = try startInProcessServiceBuildForTest( + alloc, + &service, + target, + .{ + .role = .coordinator, + .runtime_id = "service-pagerank-freshness-coordinator", + .owner_id = "service-pagerank-freshness-coordinator", + .worker_id = "service-pagerank-freshness-unused", + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 1, + }, + ); + try std.testing.expect(start.result.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), start.result.builds_started); + + var published = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published.deinit(); + try std.testing.expectEqual(@as(usize, 1), published.graph_metric_results.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, published.graph_metric_results[0].status.state); + try std.testing.expectEqual(published_generation, published.graph_metric_results[0].status.published_generation); + try std.testing.expect(published.graph_metric_results[0].status.building_generation > published_generation); + try std.testing.expectEqualStrings(initial_top_node, published.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(initial_top_score, published.graph_metric_results[0].scores[0].score, 0.000001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const traversal_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = traversal_query }}, + .limit = 0, + }); + defer traversal.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", traversal.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("pagerank", traversal.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(traversal.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].metric_status.len); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, traversal.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(published_generation, traversal.graph_results[0].metric_status[0].published_generation); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + var fresh_traversal_query = traversal_query; + fresh_traversal_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_traversal_query }}, + .limit = 0, + })); + + var rerank = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .weight = 1.0, + }, + .limit = 5, + .include_stored = false, + }); + defer rerank.deinit(); + const rerank_status = rerank.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.building, rerank_status.state); + try std.testing.expectEqual(published_generation, rerank_status.published_generation); + const rerank_details = rerank.hits[0].score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("pagerank", rerank_details.metric_name); + try std.testing.expectEqual(published_generation, rerank_details.published_generation); + try std.testing.expect(rerank_details.metric_score != null); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 5, + .include_stored = false, + })); +} + +test "graph metric maintenance service owners fence abandoned pagerank leases and recover after ttl" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-service-pagerank-restart-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + var service = InProcessGraphMetricService.init(&db); + const target = ServiceTarget{ + .base_uri = "in-process://graph-metric-maintenance", + .group_id = 7, + .table_name = "docs", + }; + + var coordinator_a = CliConfig{ + .role = .coordinator, + .runtime_id = "service-pagerank-restart-coordinator", + .owner_id = "coordinator-a", + .worker_id = "coordinator-unused", + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + .test_hold_after_run_ms = 1, + .test_now_ms = 1000, + }; + const coordinator_a_start = try startInProcessServiceBuildForTest( + alloc, + &service, + target, + coordinator_a, + ); + const coordinator_a_start_stats = coordinator_a_start.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_a_start_stats.lease_owned); + try std.testing.expect(coordinator_a_start_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_a_start_stats.acquisition_count); + try std.testing.expect(coordinator_a_start.result.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), coordinator_a_start.result.builds_started); + + var coordinator_b = coordinator_a; + coordinator_b.owner_id = "coordinator-b"; + coordinator_b.test_now_ms = 1100; + const coordinator_b_fenced = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_b, + .tick, + ); + const coordinator_b_fenced_stats = coordinator_b_fenced.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_b_fenced_stats.lease_owned); + try std.testing.expect(!coordinator_b_fenced_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_b_fenced_stats.lease_acquire_failures); + try std.testing.expect(!coordinator_b_fenced.result.durableProgressed()); + + coordinator_b.test_now_ms = 1301; + const coordinator_b_takeover = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_b, + .tick, + ); + const coordinator_b_takeover_stats = coordinator_b_takeover.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(coordinator_b_takeover_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_b_takeover_stats.takeover_count); + try std.testing.expectEqual(identityHash("coordinator-b"), coordinator_b_takeover_stats.owner_id_hash); + + coordinator_a.test_now_ms = 1302; + const stale_coordinator_release = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + coordinator_a, + .release, + ); + try std.testing.expect(!stale_coordinator_release.released); + try std.testing.expect(stale_coordinator_release.lease_owner_id_hash != 0); + const stale_coordinator_release_stats = stale_coordinator_release.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!stale_coordinator_release_stats.has_lease); + + var worker_pool_a = CliConfig{ + .role = .worker_pool, + .runtime_id = "service-pagerank-restart-worker-pool", + .owner_id = "worker-pool-a", + .worker_id = "worker-pool-unused", + .coordinator_start_background_builds = false, + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + .test_hold_after_run_ms = 1, + .test_now_ms = 2000, + }; + defer worker_pool_a.deinit(alloc); + try worker_pool_a.worker_ids.appendSlice(alloc, &.{ "restart-pagerank-worker-a", "restart-pagerank-worker-b" }); + + const worker_pool_a_start = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_a, + .tick, + ); + const worker_pool_a_start_stats = worker_pool_a_start.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(worker_pool_a_start_stats.has_lease); + try std.testing.expectEqual(@as(usize, 2), worker_pool_a_start_stats.worker_count); + try std.testing.expect(worker_pool_a_start.result.durableProgressed()); + + var worker_pool_b = CliConfig{ + .role = .worker_pool, + .runtime_id = "service-pagerank-restart-worker-pool", + .owner_id = "worker-pool-b", + .worker_id = "worker-pool-unused", + .coordinator_start_background_builds = false, + .lease_ttl_ms = 200, + .max_metrics_per_round = 4, + .max_pages_per_round = 3, + .test_hold_after_run_ms = 1, + .test_now_ms = 2100, + }; + defer worker_pool_b.deinit(alloc); + try worker_pool_b.worker_ids.appendSlice(alloc, &.{ "restart-pagerank-worker-a", "restart-pagerank-worker-b" }); + + const worker_pool_b_fenced = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_b, + .tick, + ); + const worker_pool_b_fenced_stats = worker_pool_b_fenced.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!worker_pool_b_fenced_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), worker_pool_b_fenced_stats.lease_acquire_failures); + try std.testing.expect(!worker_pool_b_fenced.result.durableProgressed()); + + worker_pool_b.test_now_ms = 2301; + const worker_pool_b_takeover = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_b, + .tick, + ); + const worker_pool_b_takeover_stats = worker_pool_b_takeover.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(worker_pool_b_takeover_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), worker_pool_b_takeover_stats.takeover_count); + try std.testing.expectEqual(identityHash("worker-pool-b"), worker_pool_b_takeover_stats.owner_id_hash); + + worker_pool_a.test_now_ms = 2302; + const stale_worker_pool_release = try requestInProcessServiceMaintenanceForTest( + alloc, + &service, + target, + worker_pool_a, + .release, + ); + try std.testing.expect(!stale_worker_pool_release.released); + try std.testing.expect(stale_worker_pool_release.lease_owner_id_hash != 0); + const stale_worker_pool_release_stats = stale_worker_pool_release.stats orelse return error.MissingRuntimeStats; + try std.testing.expect(!stale_worker_pool_release_stats.has_lease); + + var idle_rounds: usize = 0; + for (0..160) |round| { + coordinator_b.test_now_ms = 2400 + @as(u64, @intCast(round)); + const coordinator_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + coordinator_b, + ); + worker_pool_b.test_now_ms = 2500 + @as(u64, @intCast(round)); + const worker_summary = try runServiceConfiguredWithRequester( + InProcessGraphMetricService, + &service, + InProcessGraphMetricService.request, + alloc, + target, + worker_pool_b, + ); + + const progressed = coordinator_summary.durable_progressed or worker_summary.durable_progressed; + idle_rounds = if (progressed) 0 else idle_rounds + 1; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(coordinator_summary.stats.shutdown); + try std.testing.expect(worker_summary.stats.shutdown); + try std.testing.expect(!coordinator_summary.stats.has_lease); + try std.testing.expect(!worker_summary.stats.has_lease); + const top = try graph_entry.index.graphMetricTopK("pagerank", 3); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expect(top.len > 0); + return; + } + if (idle_rounds >= 6) break; + } + + return error.GraphMetricBuildNotComplete; +} + +test "graph metric maintenance command rejects duplicate worker pool ids" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--role", "worker-pool", + "--runtime-id", "runtime-a", + "--owner-id", "owner-a", + "--worker-ids", "worker-a,worker-a", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + try std.testing.expectError(error.InvalidArguments, parseCli(alloc, &args)); +} + +test "graph metric maintenance command rejects worker id lists for single-owner roles" { + const alloc = std.testing.allocator; + const coordinator_argv = [_][*:0]const u8{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--role", "coordinator", + "--runtime-id", "runtime-a", + "--owner-id", "owner-a", + "--worker-ids", "worker-a,worker-b", + }; + var coordinator_args = std.process.Args.Iterator.init(.{ .vector = coordinator_argv[0..] }); + try std.testing.expectError(error.InvalidArguments, parseCli(alloc, &coordinator_args)); + + const worker_argv = [_][*:0]const u8{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--role", "worker", + "--runtime-id", "runtime-a", + "--owner-id", "owner-a", + "--worker-ids", "worker-a,worker-b", + }; + var worker_args = std.process.Args.Iterator.init(.{ .vector = worker_argv[0..] }); + try std.testing.expectError(error.InvalidArguments, parseCli(alloc, &worker_args)); +} + +test "graph metric maintenance command rejects zero lease and maintenance budgets" { + const alloc = std.testing.allocator; + + try expectParseCliInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--lease-ttl-ms", "0", + }); + try expectParseCliInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--max-rounds", "0", + }); + try expectParseCliInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--max-metrics", "0", + }); + try expectParseCliInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-command-test", + "--max-pages", "0", + }); +} + +test "graph metric maintenance supervisor parses config and defaults workers" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--db-path", + "/tmp/antfly-graph-metric-supervisor-test", + "--executable", + "/tmp/antfly", + "--coordinator-owner-id", + "coord-a", + "--worker-pool-owner-id", + "pool-a", + "--ticks", + "11", + "--max-idle-ticks", + "3", + "--supervisor-rounds", + "17", + "--supervisor-idle-rounds", + "2", + "--tick-ms", + "5", + "--max-restarts", + "4", + "--summary-dir", + "/tmp/summaries", + "--max-pages", + "7", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + var parsed = try parseSupervisorCli(alloc, &args); + defer parsed.deinit(alloc); + + try std.testing.expectEqualStrings("/tmp/antfly-graph-metric-supervisor-test", parsed.db_path.?); + try std.testing.expectEqualStrings("/tmp/antfly", parsed.executable.?); + try std.testing.expectEqualStrings("coord-a", parsed.coordinator_owner_id); + try std.testing.expectEqualStrings("pool-a", parsed.worker_pool_owner_id); + try std.testing.expectEqual(@as(usize, 11), parsed.max_ticks); + try std.testing.expectEqual(@as(usize, 3), parsed.max_idle_ticks); + try std.testing.expectEqual(@as(usize, 17), parsed.max_supervisor_rounds); + try std.testing.expectEqual(@as(usize, 2), parsed.max_supervisor_idle_rounds); + try std.testing.expectEqual(@as(u64, 5), parsed.tick_interval_ms); + try std.testing.expectEqual(@as(usize, 4), parsed.max_restarts); + try std.testing.expectEqualStrings("/tmp/summaries", parsed.summary_dir); + try std.testing.expectEqual(@as(usize, 7), parsed.max_pages_per_round); + try std.testing.expectEqual(@as(usize, 2), parsed.worker_ids.items.len); + try std.testing.expectEqualStrings("graph-metric-worker-a", parsed.worker_ids.items[0]); + try std.testing.expectEqualStrings("graph-metric-worker-b", parsed.worker_ids.items[1]); +} + +test "graph metric maintenance supervisor parses service target config" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--base-uri", + "http://127.0.0.1:8080", + "--group-id", + "7", + "--table-name", + "docs", + "--worker-ids", + "worker-a,worker-b", + "--max-pages", + "5", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + var parsed = try parseSupervisorCli(alloc, &args); + defer parsed.deinit(alloc); + + const target = (try parsed.target()) orelse return error.MissingServiceTarget; + const service = switch (target) { + .service => |service| service, + .db_path => return error.InvalidArguments, + }; + try std.testing.expectEqualStrings("http://127.0.0.1:8080", service.base_uri); + try std.testing.expectEqual(@as(u64, 7), service.group_id); + try std.testing.expectEqualStrings("docs", service.table_name); + try std.testing.expectEqual(@as(usize, 2), parsed.worker_ids.items.len); + try std.testing.expectEqual(@as(usize, 5), parsed.max_pages_per_round); +} + +test "graph metric maintenance supervisor rejects duplicate worker pool ids" { + const alloc = std.testing.allocator; + const argv = [_][*:0]const u8{ + "--db-path", + "/tmp/antfly-graph-metric-supervisor-test", + "--worker-ids", + "worker-a,worker-b,worker-a", + }; + var args = std.process.Args.Iterator.init(.{ .vector = argv[0..] }); + try std.testing.expectError(error.InvalidArguments, parseSupervisorCli(alloc, &args)); +} + +test "graph metric maintenance supervisor rejects invalid service target combinations" { + const alloc = std.testing.allocator; + + try expectParseSupervisorInvalid(alloc, &.{ + "--db-path", "/tmp/db", + "--base-uri", "http://127.0.0.1:8080", + "--group-id", "7", + "--table-name", "docs", + }); + try expectParseSupervisorInvalid(alloc, &.{ + "--base-uri", "http://127.0.0.1:8080", + "--group-id", "7", + }); + try expectParseSupervisorInvalid(alloc, &.{ + "--base-uri", "http://127.0.0.1:8080", + "--table-name", "docs", + }); +} + +test "graph metric maintenance supervisor rejects zero lease and maintenance budgets" { + const alloc = std.testing.allocator; + + try expectParseSupervisorInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-supervisor-test", + "--lease-ttl-ms", "0", + }); + try expectParseSupervisorInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-supervisor-test", + "--max-rounds", "0", + }); + try expectParseSupervisorInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-supervisor-test", + "--max-metrics", "0", + }); + try expectParseSupervisorInvalid(alloc, &.{ + "--db-path", "/tmp/antfly-graph-metric-supervisor-test", + "--max-pages", "0", + }); +} + +test "graph metric maintenance supervisor builds coordinator and worker pool argv" { + const alloc = std.testing.allocator; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + const cfg = SupervisorConfig{ + .executable = "/tmp/antfly", + .coordinator_owner_id = "coord-owner", + .worker_pool_owner_id = "pool-owner", + .worker_ids = worker_ids, + .lease_ttl_ms = 1234, + .max_ticks = 11, + .max_idle_ticks = 3, + .tick_interval_ms = 5, + .max_rounds = 2, + .max_metrics_per_round = 4, + .max_pages_per_round = 7, + }; + + const db_target = SupervisorTarget{ .db_path = "/tmp/db" }; + var coordinator = try buildSupervisorChildArgv(alloc, "fallback-antfly", db_target, cfg, .coordinator, "/tmp/coordinator-summary.json"); + defer coordinator.deinit(alloc); + try std.testing.expectEqualStrings("/tmp/antfly", coordinator.argv.items[0]); + try std.testing.expectEqualStrings(process_subcommand, coordinator.argv.items[1]); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--db-path", "/tmp/db")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--role", "coordinator")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--runtime-id", "coord-owner")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--owner-id", "coord-owner")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--coordinator-start-background-builds", "true")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--summary-file", "/tmp/coordinator-summary.json")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--local-db-writer-lock", "true")); + try std.testing.expect(!argvContains(coordinator.argv.items, "--worker-ids")); + + var worker_pool = try buildSupervisorChildArgv(alloc, "fallback-antfly", db_target, cfg, .worker_pool, null); + defer worker_pool.deinit(alloc); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--role", "worker_pool")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--runtime-id", "pool-owner")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--owner-id", "pool-owner")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--worker-ids", "worker-a,worker-b")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--coordinator-start-background-builds", "false")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--lease-ttl-ms", "1234")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--ticks", "11")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-idle-ticks", "3")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--tick-ms", "5")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-rounds", "2")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-metrics", "4")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-pages", "7")); +} + +test "graph metric maintenance supervisor builds service child argv without local writer guard" { + const alloc = std.testing.allocator; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + const cfg = SupervisorConfig{ + .executable = "/tmp/antfly", + .coordinator_owner_id = "coord-owner", + .worker_pool_owner_id = "pool-owner", + .worker_ids = worker_ids, + .max_ticks = 11, + .max_idle_ticks = 3, + .max_pages_per_round = 7, + }; + const service_target = SupervisorTarget{ .service = .{ + .base_uri = "http://127.0.0.1:8080", + .group_id = 7, + .table_name = "docs", + } }; + + var coordinator = try buildSupervisorChildArgv(alloc, "fallback-antfly", service_target, cfg, .coordinator, "/tmp/coordinator-summary.json"); + defer coordinator.deinit(alloc); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--base-uri", "http://127.0.0.1:8080")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--group-id", "7")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--table-name", "docs")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--role", "coordinator")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--summary-file", "/tmp/coordinator-summary.json")); + try std.testing.expect(!argvContains(coordinator.argv.items, "--db-path")); + try std.testing.expect(!argvContains(coordinator.argv.items, "--local-db-writer-lock")); + + var worker_pool = try buildSupervisorChildArgv(alloc, "fallback-antfly", service_target, cfg, .worker_pool, "/tmp/worker-summary.json"); + defer worker_pool.deinit(alloc); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--base-uri", "http://127.0.0.1:8080")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--role", "worker_pool")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--worker-ids", "worker-a,worker-b")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-pages", "7")); + try std.testing.expect(!argvContains(worker_pool.argv.items, "--db-path")); + try std.testing.expect(!argvContains(worker_pool.argv.items, "--local-db-writer-lock")); +} + +test "graph metric maintenance launched child argv stays owner and budget scoped" { + const alloc = std.testing.allocator; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + const cfg = SupervisorConfig{ + .executable = "/tmp/antfly", + .coordinator_owner_id = "coord-owner", + .worker_pool_owner_id = "pool-owner", + .worker_ids = worker_ids, + .lease_ttl_ms = 1234, + .max_ticks = 11, + .max_idle_ticks = 3, + .tick_interval_ms = 5, + .max_rounds = 2, + .max_metrics_per_round = 4, + .max_pages_per_round = 7, + }; + + const forbidden = [_][]const u8{ + "--index", + "--index-name", + "--metric", + "--metric-name", + "--metric-config", + "--target-generation", + "--job-id", + "--page-id", + "--phase", + "--summary-file", + "--local-db-writer-lock", + }; + + const db_target = SupervisorTarget{ .db_path = "/tmp/db" }; + var coordinator = try buildSupervisorChildArgv(alloc, "fallback-antfly", db_target, cfg, .coordinator, null); + defer coordinator.deinit(alloc); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--role", "coordinator")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--runtime-id", "coord-owner")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--owner-id", "coord-owner")); + try std.testing.expect(argvValueEquals(coordinator.argv.items, "--worker-id", "graph-metric-coordinator-unused")); + try std.testing.expect(!argvContains(coordinator.argv.items, "--worker-ids")); + try std.testing.expect(!argvContainsAny(coordinator.argv.items, forbidden[0..])); + + var worker_pool = try buildSupervisorChildArgv(alloc, "fallback-antfly", db_target, cfg, .worker_pool, null); + defer worker_pool.deinit(alloc); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--role", "worker_pool")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--runtime-id", "pool-owner")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--owner-id", "pool-owner")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--worker-id", "graph-metric-worker-pool-unused")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--worker-ids", "worker-a,worker-b")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--lease-ttl-ms", "1234")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-rounds", "2")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-metrics", "4")); + try std.testing.expect(argvValueEquals(worker_pool.argv.items, "--max-pages", "7")); + try std.testing.expect(!argvContainsAny(worker_pool.argv.items, forbidden[0..])); +} + +test "graph metric maintenance supervisor restart policy is bounded" { + try std.testing.expect(shouldRestartSupervisorAttempt(true, 0, 1)); + try std.testing.expect(!shouldRestartSupervisorAttempt(true, 1, 1)); + try std.testing.expect(!shouldRestartSupervisorAttempt(false, 0, 1)); + try std.testing.expect(childExitSucceeded(.{ .exited = true, .code = 0 })); + try std.testing.expect(!childExitSucceeded(.{ .exited = true, .code = 1 })); + try std.testing.expect(!childExitSucceeded(.{ .exited = false, .code = null })); +} + +test "graph metric maintenance supervisor loops until global idle" { + const alloc = std.testing.allocator; + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + const sequence = [_]ChildRunSummary{ + fakeChild(false, 0), + fakeChild(true, 0), + fakeChild(false, 0), + fakeChild(false, 0), + }; + var runner = FakeSupervisorChildRunner{ .sequence = sequence[0..] }; + var workers = std.ArrayListUnmanaged([]const u8).empty; + defer workers.deinit(alloc); + try workers.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + const summary = try runSupervisorConfiguredWithRunner( + FakeSupervisorChildRunner, + &runner, + FakeSupervisorChildRunner.run, + io_impl.io(), + alloc, + "antfly", + .{ .db_path = "/tmp/db" }, + .{ + .worker_ids = workers, + .max_ticks = 1, + .max_idle_ticks = 1, + .max_supervisor_rounds = 4, + .max_supervisor_idle_rounds = 1, + .tick_interval_ms = 0, + .max_restarts = 0, + }, + ); + try std.testing.expectEqual(@as(usize, 4), runner.calls); + try std.testing.expectEqual(@as(usize, 2), summary.rounds_executed); + try std.testing.expectEqual(@as(usize, 0), summary.restarts); + try std.testing.expectEqual(@as(usize, 1), summary.idle_rounds); + try std.testing.expectEqual(SupervisorExitReason.idle, summary.exit_reason); + try std.testing.expect(summary.succeeded); +} + +test "graph metric maintenance supervisor drives degree through child role argv" { + const alloc = std.testing.allocator; + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-supervised-degree-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + { + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "supervised-worker-a", "supervised-worker-b" }; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, workers[0..]); + + var runner = LoopbackSupervisorChildRunner{}; + const summary = try runSupervisorConfiguredWithRunner( + LoopbackSupervisorChildRunner, + &runner, + LoopbackSupervisorChildRunner.run, + io_impl.io(), + alloc, + "antfly", + .{ .db_path = path }, + .{ + .coordinator_owner_id = "supervised-degree-coordinator", + .worker_pool_owner_id = "supervised-degree-worker-pool", + .worker_ids = worker_ids, + .max_ticks = 8, + .max_idle_ticks = 2, + .max_supervisor_rounds = 80, + .max_supervisor_idle_rounds = 1, + .tick_interval_ms = 0, + .max_restarts = 0, + .max_rounds = 1, + .max_metrics_per_round = 4, + .max_pages_per_round = 2, + }, + ); + try std.testing.expect(summary.succeeded); + try std.testing.expectEqual(SupervisorExitReason.idle, summary.exit_reason); + try std.testing.expect(runner.calls >= 2); + const coordinator_telemetry = summary.coordinator.telemetry orelse return error.MissingChildTelemetry; + try std.testing.expectEqual(RuntimeRole.coordinator, coordinator_telemetry.role); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "supervised-degree-coordinator"), + coordinator_telemetry.runtime_id_hash, + ); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "supervised-degree-coordinator"), + coordinator_telemetry.owner_id_hash, + ); + try std.testing.expect(coordinator_telemetry.lease_key_hash != 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_telemetry.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_telemetry.worker_count); + try std.testing.expect(coordinator_telemetry.lease_owned); + try std.testing.expect(coordinator_telemetry.has_lease); + try std.testing.expectEqual(@as(u64, 1), coordinator_telemetry.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), coordinator_telemetry.error_ticks); + try std.testing.expect(!coordinator_telemetry.has_last_error); + + const worker_pool_telemetry = summary.worker_pool.telemetry orelse return error.MissingChildTelemetry; + try std.testing.expectEqual(RuntimeRole.worker_pool, worker_pool_telemetry.role); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "supervised-degree-worker-pool"), + worker_pool_telemetry.runtime_id_hash, + ); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "supervised-degree-worker-pool"), + worker_pool_telemetry.owner_id_hash, + ); + try std.testing.expect(worker_pool_telemetry.lease_key_hash != 0); + try std.testing.expect(worker_pool_telemetry.worker_id_hash != 0); + try std.testing.expectEqual(@as(usize, 2), worker_pool_telemetry.worker_count); + try std.testing.expect(worker_pool_telemetry.lease_owned); + try std.testing.expect(worker_pool_telemetry.has_lease); + try std.testing.expectEqual(@as(u64, 1), worker_pool_telemetry.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), worker_pool_telemetry.error_ticks); + try std.testing.expect(!worker_pool_telemetry.has_last_error); + + var reader = try antfly.db.DB.open(alloc, path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(antfly.graph.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(target_generation, status.published_generation); +} + +test "graph metric maintenance supervisor stops at restart limit" { + const alloc = std.testing.allocator; + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + const sequence = [_]ChildRunSummary{ + fakeChild(null, 17), + fakeChild(false, 0), + fakeChild(false, 0), + fakeChild(null, 42), + }; + var runner = FakeSupervisorChildRunner{ .sequence = sequence[0..] }; + var workers = std.ArrayListUnmanaged([]const u8).empty; + defer workers.deinit(alloc); + try workers.appendSlice(alloc, &.{ "worker-a", "worker-b" }); + const summary = try runSupervisorConfiguredWithRunner( + FakeSupervisorChildRunner, + &runner, + FakeSupervisorChildRunner.run, + io_impl.io(), + alloc, + "antfly", + .{ .db_path = "/tmp/db" }, + .{ + .worker_ids = workers, + .max_ticks = 1, + .max_idle_ticks = 1, + .max_supervisor_rounds = 4, + .max_supervisor_idle_rounds = 1, + .tick_interval_ms = 0, + .max_restarts = 1, + }, + ); + try std.testing.expectEqual(@as(usize, 4), runner.calls); + try std.testing.expectEqual(@as(usize, 2), summary.rounds_executed); + try std.testing.expectEqual(@as(usize, 1), summary.restarts); + try std.testing.expectEqual(SupervisorExitReason.restart_limit, summary.exit_reason); + try std.testing.expect(!summary.succeeded); + try std.testing.expectEqual(@as(?u8, 42), summary.worker_pool.exit.code); +} + +test "graph metric maintenance supervisor parses child durable progress" { + const alloc = std.testing.allocator; + try std.testing.expect(try parseChildDurableProgressed( + alloc, + "{ \"role\": \"worker_pool\", \"durable_progressed\": true }", + )); + try std.testing.expect(!try parseChildDurableProgressed( + alloc, + "{ \"role\": \"coordinator\", \"durable_progressed\": false }", + )); + try std.testing.expectError(error.InvalidArguments, parseChildDurableProgressed( + alloc, + "{ \"role\": \"coordinator\" }", + )); +} + +test "graph metric maintenance supervisor parses child runtime telemetry" { + const alloc = std.testing.allocator; + const telemetry = try parseChildRuntimeTelemetry(alloc, + \\{ + \\ "role": "worker_pool", + \\ "durable_progressed": false, + \\ "stats": { + \\ "enabled": true, + \\ "role": "worker_pool", + \\ "runtime_id_hash": 11, + \\ "owner_id_hash": 12, + \\ "lease_key_hash": 13, + \\ "worker_id_hash": 14, + \\ "worker_count": 2, + \\ "lease_owned": true, + \\ "has_lease": true, + \\ "acquisition_count": 1, + \\ "takeover_count": 0, + \\ "lease_acquire_failures": 0, + \\ "lost_leases": 0, + \\ "last_acquired_ms": 123, + \\ "started": false, + \\ "shutdown": false, + \\ "notified": false, + \\ "ticks_started": 3, + \\ "ticks_completed": 3, + \\ "durable_progress_ticks": 1, + \\ "idle_ticks": 2, + \\ "error_ticks": 0, + \\ "last_error_name": null + \\ } + \\} + ); + try std.testing.expectEqual(RuntimeRole.worker_pool, telemetry.role); + try std.testing.expectEqual(@as(u64, 11), telemetry.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 12), telemetry.owner_id_hash); + try std.testing.expectEqual(@as(u64, 13), telemetry.lease_key_hash); + try std.testing.expectEqual(@as(u64, 14), telemetry.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), telemetry.worker_count); + try std.testing.expect(telemetry.lease_owned); + try std.testing.expect(telemetry.has_lease); + try std.testing.expectEqual(@as(u64, 1), telemetry.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), telemetry.takeover_count); + try std.testing.expectEqual(@as(u64, 0), telemetry.lost_leases); + try std.testing.expectEqual(@as(u64, 3), telemetry.ticks_started); + try std.testing.expectEqual(@as(u64, 3), telemetry.ticks_completed); + try std.testing.expectEqual(@as(u64, 2), telemetry.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), telemetry.error_ticks); + try std.testing.expect(!telemetry.has_last_error); +} + +test "graph metric maintenance command exits after configured idle streak" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-command-idle-db", .{tmp.sub_path}); + { + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + } + + const summary = try runConfigured(alloc, path, .{ + .role = .coordinator, + .runtime_id = "command-idle-coordinator", + .owner_id = "command-idle-coordinator", + .worker_id = "command-idle-unused", + .max_ticks = 10, + .until_idle = true, + .max_idle_ticks = 2, + }); + try std.testing.expectEqual(RuntimeRole.coordinator, summary.role); + try std.testing.expectEqual(@as(usize, 2), summary.ticks_executed); + try std.testing.expectEqual(@as(usize, 2), summary.idle_streak); + try std.testing.expectEqual(ExitReason.idle, summary.exit_reason); + try std.testing.expect(!summary.durable_progressed); + try std.testing.expect(summary.stats.enabled); + try std.testing.expect(!summary.stats.started); +} + +test "graph metric maintenance command summary exposes ownership telemetry" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-command-telemetry-db", .{tmp.sub_path}); + { + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + } + + const coordinator_summary = try runConfigured(alloc, path, .{ + .role = .coordinator, + .runtime_id = "command-telemetry-coordinator-runtime", + .owner_id = "command-telemetry-coordinator-owner", + .worker_id = "command-telemetry-unused-worker", + .max_ticks = 1, + }); + try std.testing.expectEqual(RuntimeRole.coordinator, coordinator_summary.role); + try std.testing.expectEqual(RuntimeRole.coordinator, coordinator_summary.stats.role); + try std.testing.expect(coordinator_summary.stats.enabled); + try std.testing.expect(coordinator_summary.stats.lease_owned); + try std.testing.expect(coordinator_summary.stats.has_lease); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "command-telemetry-coordinator-runtime"), + coordinator_summary.stats.runtime_id_hash, + ); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "command-telemetry-coordinator-owner"), + coordinator_summary.stats.owner_id_hash, + ); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, graph_metric_runtime_mod.defaultLeaseKey(.coordinator)), + coordinator_summary.stats.lease_key_hash, + ); + try std.testing.expectEqual(@as(u64, 0), coordinator_summary.stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_summary.stats.worker_count); + try std.testing.expectEqual(@as(u64, 1), coordinator_summary.stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), coordinator_summary.stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), coordinator_summary.stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), coordinator_summary.stats.ticks_started); + try std.testing.expectEqual(@as(u64, 1), coordinator_summary.stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), coordinator_summary.stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), coordinator_summary.stats.error_ticks); + try std.testing.expectEqual(@as(?[]const u8, null), coordinator_summary.stats.last_error_name); + + const workers = [_][]const u8{ "command-telemetry-worker-a", "command-telemetry-worker-b" }; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, workers[0..]); + + const worker_pool_summary = try runConfigured(alloc, path, .{ + .role = .worker_pool, + .runtime_id = "command-telemetry-worker-runtime", + .owner_id = "command-telemetry-worker-owner", + .worker_id = "command-telemetry-unused-single-worker", + .worker_ids = worker_ids, + .max_ticks = 1, + }); + try std.testing.expectEqual(RuntimeRole.worker_pool, worker_pool_summary.role); + try std.testing.expectEqual(RuntimeRole.worker_pool, worker_pool_summary.stats.role); + try std.testing.expect(worker_pool_summary.stats.enabled); + try std.testing.expect(worker_pool_summary.stats.lease_owned); + try std.testing.expect(worker_pool_summary.stats.has_lease); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "command-telemetry-worker-runtime"), + worker_pool_summary.stats.runtime_id_hash, + ); + try std.testing.expectEqual( + std.hash.Wyhash.hash(0, "command-telemetry-worker-owner"), + worker_pool_summary.stats.owner_id_hash, + ); + try std.testing.expect(worker_pool_summary.stats.lease_key_hash != 0); + try std.testing.expect(worker_pool_summary.stats.worker_id_hash != 0); + try std.testing.expectEqual(@as(usize, 2), worker_pool_summary.stats.worker_count); + try std.testing.expectEqual(@as(u64, 1), worker_pool_summary.stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), worker_pool_summary.stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), worker_pool_summary.stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), worker_pool_summary.stats.ticks_started); + try std.testing.expectEqual(@as(u64, 1), worker_pool_summary.stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), worker_pool_summary.stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), worker_pool_summary.stats.error_ticks); + try std.testing.expectEqual(@as(?[]const u8, null), worker_pool_summary.stats.last_error_name); +} + +test "graph metric maintenance command drives split degree through db-open runtime" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/graph-metric-command-db", .{tmp.sub_path}); + + var target_generation: u64 = 0; + { + var db = try antfly.db.DB.open(alloc, path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "command-worker-a", "command-worker-b" }; + var worker_ids = std.ArrayListUnmanaged([]const u8).empty; + defer worker_ids.deinit(alloc); + try worker_ids.appendSlice(alloc, workers[0..]); + + const coordinator = CliConfig{ + .role = .coordinator, + .runtime_id = "command-coordinator", + .owner_id = "command-coordinator", + .worker_id = "command-coordinator-unused", + .max_ticks = 1, + .max_pages_per_round = 1, + }; + const worker_pool = CliConfig{ + .role = .worker_pool, + .runtime_id = "command-worker-pool", + .owner_id = "command-worker-pool", + .worker_id = "command-worker-unused", + .worker_ids = worker_ids, + .max_ticks = 1, + .max_pages_per_round = 2, + }; + + var fresh = false; + for (0..80) |_| { + _ = try runConfigured(alloc, path, coordinator); + const worker_summary = try runConfigured(alloc, path, worker_pool); + try std.testing.expectEqual(RuntimeRole.worker_pool, worker_summary.role); + try std.testing.expect(!worker_summary.stats.started); + _ = try runConfigured(alloc, path, coordinator); + + var reader = try antfly.db.DB.open(alloc, path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + fresh = true; + break; + } + } + try std.testing.expect(fresh); +} diff --git a/zig/pkg/antfly/src/cmd/graph_metric_process_harness.zig b/zig/pkg/antfly/src/cmd/graph_metric_process_harness.zig new file mode 100644 index 0000000000..3bff14a727 --- /dev/null +++ b/zig/pkg/antfly/src/cmd/graph_metric_process_harness.zig @@ -0,0 +1,10333 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const antfly = @import("antfly-zig"); +const httpx = @import("httpx"); +const platform = @import("antfly_platform"); +const graph_query_mod = antfly.graph_query; + +const harness_internal_service_secret = "graph-metric-process-harness-internal-service-secret"; +const harness_internal_service_issuer = "graph-metric-process-harness"; +// Scoped to main's lifetime; every child gets an owned environment snapshot. +var child_environ: ?*const std.process.Environ.Map = null; + +const HarnessProfile = enum { + smoke, + promotion, +}; + +const RuntimeRole = enum { + combined, + coordinator, + worker, + worker_pool, +}; + +const ChildRuntimeTelemetry = struct { + role: RuntimeRole = .combined, + runtime_id_hash: u64 = 0, + owner_id_hash: u64 = 0, + lease_key_hash: u64 = 0, + worker_id_hash: u64 = 0, + worker_count: usize = 0, + lease_owned: bool = false, + has_lease: bool = false, + acquisition_count: u64 = 0, + takeover_count: u64 = 0, + lost_leases: u64 = 0, + ticks_started: u64 = 0, + ticks_completed: u64 = 0, + idle_ticks: u64 = 0, + error_ticks: u64 = 0, + has_last_error: bool = false, +}; + +const ChildRunSummary = struct { + telemetry: ?ChildRuntimeTelemetry = null, +}; + +const SupervisorSummary = struct { + rounds_executed: usize = 0, + exit_reason: []const u8 = "", + succeeded: bool = false, + coordinator: ChildRunSummary = .{}, + worker_pool: ChildRunSummary = .{}, +}; + +const RuntimeStats = struct { + role: RuntimeRole = .combined, + runtime_id_hash: u64 = 0, + owner_id_hash: u64 = 0, + lease_key_hash: u64 = 0, + worker_id_hash: u64 = 0, + worker_count: usize = 0, + lease_owned: bool = false, + has_lease: bool = false, + acquisition_count: u64 = 0, + takeover_count: u64 = 0, + lease_acquire_failures: u64 = 0, + lost_leases: u64 = 0, + ticks_started: u64 = 0, + ticks_completed: u64 = 0, + durable_progress_ticks: u64 = 0, + idle_ticks: u64 = 0, + error_ticks: u64 = 0, +}; + +const SchedulerResult = struct { + pages_claimed: usize = 0, + pages_completed: usize = 0, + phases_advanced: usize = 0, + published: usize = 0, + failed_builds: usize = 0, +}; + +const RoleRunSummary = struct { + durable_progressed: bool = false, + result: SchedulerResult = .{}, + stats: RuntimeStats = .{}, +}; + +const PageLeaseSnapshot = struct { + job_id: u64 = 0, + page_id: u64 = 0, + iteration: u32 = 0, + attempt: u64 = 0, + lease_expires_at_ms: u64 = 0, + total_units: u64 = 0, +}; + +const ProcessHarnessReleaseSummary = struct { + launch_families: usize = 0, + service_owner_restart_families: usize = 0, + service_publish_cleanup_families: usize = 0, + service_publish_failure_families: usize = 0, + service_multipage_worker_pool_families: usize = 0, + service_multipage_coordinator_takeover_families: usize = 0, + service_multipage_worker_pool_takeover_families: usize = 0, + service_multipage_worker_phase_proofs: usize = 0, + service_multipage_coordinator_phase_proofs: usize = 0, + service_multipage_takeover_phase_proofs: usize = 0, + service_cleanup_takeover_families: usize = 0, + service_active_public_read_families: usize = 0, + direct_publish_cleanup_families: usize = 0, + direct_publish_failure_families: usize = 0, + direct_active_public_read_families: usize = 0, + direct_page_reclaim_phase_proofs: usize = 0, + direct_reclaimed_attempt_completion_phase_proofs: usize = 0, + direct_stale_attempt_rejection_phase_proofs: usize = 0, + fixed_iteration_families: usize = 0, + exhausted_attempt_families: usize = 0, + same_worker_fencing_proofs: usize = 0, +}; + +const required_process_harness_release_summary = ProcessHarnessReleaseSummary{ + .launch_families = 4, + .service_owner_restart_families = 4, + .service_publish_cleanup_families = 4, + .service_publish_failure_families = 3, + .service_multipage_worker_pool_families = 4, + .service_multipage_coordinator_takeover_families = 4, + .service_multipage_worker_pool_takeover_families = 4, + .service_multipage_worker_phase_proofs = 27, + .service_multipage_coordinator_phase_proofs = 31, + .service_multipage_takeover_phase_proofs = 8, + .service_cleanup_takeover_families = 4, + .service_active_public_read_families = 4, + .direct_publish_cleanup_families = 4, + .direct_publish_failure_families = 3, + .direct_active_public_read_families = 4, + .direct_page_reclaim_phase_proofs = 20, + .direct_reclaimed_attempt_completion_phase_proofs = 20, + .direct_stale_attempt_rejection_phase_proofs = 20, + .fixed_iteration_families = 3, + .exhausted_attempt_families = 3, + .same_worker_fencing_proofs = 2, +}; + +fn recordDirectPageReclaimProof(summary: *ProcessHarnessReleaseSummary) void { + summary.direct_page_reclaim_phase_proofs += 1; + summary.direct_reclaimed_attempt_completion_phase_proofs += 1; + summary.direct_stale_attempt_rejection_phase_proofs += 1; +} + +fn recordServiceMultipagePhaseProofs( + summary: *ProcessHarnessReleaseSummary, + worker_phase_proofs: usize, + coordinator_phase_proofs: usize, + takeover_phase_proofs: usize, +) void { + summary.service_multipage_worker_phase_proofs += worker_phase_proofs; + summary.service_multipage_coordinator_phase_proofs += coordinator_phase_proofs; + summary.service_multipage_takeover_phase_proofs += takeover_phase_proofs; +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + var environ = try init.environ_map.clone(alloc); + defer environ.deinit(); + try environ.put("ANTFLY_INTERNAL_SERVICE_SECRET", harness_internal_service_secret); + try environ.put("ANTFLY_INTERNAL_SERVICE_ISSUER", harness_internal_service_issuer); + child_environ = &environ; + defer child_environ = null; + + var arena_impl = std.heap.ArenaAllocator.init(alloc); + defer arena_impl.deinit(); + const arena = arena_impl.allocator(); + + const argv = try init.minimal.args.toSlice(arena); + if (argv.len >= 2 and std.mem.eql(u8, argv[1], "claim-degree-page-hold")) { + try runClaimDegreePageHoldMode(alloc, init.io, argv); + return; + } + if (argv.len >= 2 and std.mem.eql(u8, argv[1], "claim-metric-page-hold")) { + try runClaimMetricPageHoldMode(alloc, init.io, argv); + return; + } + if (!(argv.len == 2 or (argv.len == 4 and std.mem.eql(u8, argv[2], "--profile")))) { + std.debug.print("usage: graph_metric_process_harness [--profile smoke|promotion]\n", .{}); + std.debug.print(" graph_metric_process_harness claim-degree-page-hold \n", .{}); + std.debug.print(" graph_metric_process_harness claim-metric-page-hold \n", .{}); + std.process.exit(2); + } + const harness_exe = argv[0]; + const antfly_exe = argv[1]; + const profile = if (argv.len == 4) try parseHarnessProfile(argv[3]) else HarnessProfile.promotion; + var release_summary: ProcessHarnessReleaseSummary = .{}; + + try verifyRoleProcessArgvPreflightSelfTest(); + + const supervisor_db_path = ".zig-cache/tmp/graph-metric-process-degree-db"; + std.Io.Dir.cwd().deleteTree(init.io, supervisor_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, supervisor_db_path) catch {}; + + const launch_db_path = ".zig-cache/tmp/graph-metric-process-launch-degree-db"; + std.Io.Dir.cwd().deleteTree(init.io, launch_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, launch_db_path) catch {}; + + const pagerank_launch_db_path = ".zig-cache/tmp/graph-metric-process-launch-pagerank-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_launch_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_launch_db_path) catch {}; + + const eigenvector_launch_db_path = ".zig-cache/tmp/graph-metric-process-launch-eigenvector-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_launch_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_launch_db_path) catch {}; + + const hits_launch_db_path = ".zig-cache/tmp/graph-metric-process-launch-hits-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_launch_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_launch_db_path) catch {}; + + const lease_db_path = ".zig-cache/tmp/graph-metric-process-lease-db"; + std.Io.Dir.cwd().deleteTree(init.io, lease_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, lease_db_path) catch {}; + + const service_owner_db_path = ".zig-cache/tmp/graph-metric-process-service-owner-db"; + std.Io.Dir.cwd().deleteTree(init.io, service_owner_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, service_owner_db_path) catch {}; + + const degree_service_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-degree-service-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, degree_service_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, degree_service_publish_cleanup_db_path) catch {}; + + const degree_service_multipage_db_path = ".zig-cache/tmp/graph-metric-process-degree-service-multipage-db"; + std.Io.Dir.cwd().deleteTree(init.io, degree_service_multipage_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, degree_service_multipage_db_path) catch {}; + + const pagerank_service_owner_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-service-owner-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_owner_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_owner_db_path) catch {}; + + const pagerank_service_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-service-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_publish_cleanup_db_path) catch {}; + + const pagerank_service_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-service-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_publish_failure_db_path) catch {}; + + const pagerank_service_multipage_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-service-multipage-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_multipage_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_multipage_db_path) catch {}; + + const eigenvector_service_owner_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-service-owner-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_owner_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_owner_db_path) catch {}; + + const eigenvector_service_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-service-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_publish_cleanup_db_path) catch {}; + + const eigenvector_service_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-service-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_publish_failure_db_path) catch {}; + + const eigenvector_service_multipage_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-service-multipage-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_multipage_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_multipage_db_path) catch {}; + + const hits_service_owner_db_path = ".zig-cache/tmp/graph-metric-process-hits-service-owner-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_service_owner_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_service_owner_db_path) catch {}; + + const hits_service_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-hits-service-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_service_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_service_publish_cleanup_db_path) catch {}; + + const hits_service_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-hits-service-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_service_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_service_publish_failure_db_path) catch {}; + + const hits_service_multipage_db_path = ".zig-cache/tmp/graph-metric-process-hits-service-multipage-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_service_multipage_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_service_multipage_db_path) catch {}; + + const degree_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-degree-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, degree_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, degree_active_public_read_db_path) catch {}; + + const pagerank_service_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-service-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_service_active_public_read_db_path) catch {}; + + const degree_service_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-degree-service-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, degree_service_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, degree_service_active_public_read_db_path) catch {}; + + const worker_page_db_path = ".zig-cache/tmp/graph-metric-process-worker-page-db"; + std.Io.Dir.cwd().deleteTree(init.io, worker_page_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, worker_page_db_path) catch {}; + + const worker_runtime_db_path = ".zig-cache/tmp/graph-metric-process-worker-runtime-db"; + std.Io.Dir.cwd().deleteTree(init.io, worker_runtime_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, worker_runtime_db_path) catch {}; + + const publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, publish_cleanup_db_path) catch {}; + + const pagerank_scan_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-scan-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_scan_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_scan_db_path) catch {}; + + const pagerank_initialize_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-initialize-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_initialize_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_initialize_db_path) catch {}; + + const pagerank_contribution_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-contribution-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_contribution_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_contribution_db_path) catch {}; + + const pagerank_reduce_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-reduce-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_reduce_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_reduce_db_path) catch {}; + + const pagerank_convergence_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-convergence-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_convergence_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_convergence_db_path) catch {}; + + const pagerank_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_publish_cleanup_db_path) catch {}; + + const pagerank_cleanup_reclaim_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-cleanup-reclaim-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_cleanup_reclaim_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_cleanup_reclaim_db_path) catch {}; + + const pagerank_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_publish_failure_db_path) catch {}; + + const pagerank_fixed_iteration_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-fixed-iteration-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_fixed_iteration_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_fixed_iteration_db_path) catch {}; + + const pagerank_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_active_public_read_db_path) catch {}; + + const pagerank_same_worker_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-same-worker-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_same_worker_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_same_worker_db_path) catch {}; + + const pagerank_later_contribution_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-later-contribution-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_contribution_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_contribution_db_path) catch {}; + + const pagerank_later_reduce_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-later-reduce-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_reduce_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_reduce_db_path) catch {}; + + const pagerank_later_convergence_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-later-convergence-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_convergence_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_later_convergence_db_path) catch {}; + + const pagerank_exhausted_attempt_db_path = ".zig-cache/tmp/graph-metric-process-pagerank-exhausted-attempt-db"; + std.Io.Dir.cwd().deleteTree(init.io, pagerank_exhausted_attempt_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, pagerank_exhausted_attempt_db_path) catch {}; + + const eigenvector_supervisor_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-supervisor-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_supervisor_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_supervisor_db_path) catch {}; + + const eigenvector_fixed_iteration_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-fixed-iteration-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_fixed_iteration_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_fixed_iteration_db_path) catch {}; + + const hits_fixed_iteration_db_path = ".zig-cache/tmp/graph-metric-process-hits-fixed-iteration-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_fixed_iteration_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_fixed_iteration_db_path) catch {}; + + const eigenvector_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_publish_cleanup_db_path) catch {}; + + const eigenvector_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_publish_failure_db_path) catch {}; + + const eigenvector_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_active_public_read_db_path) catch {}; + + const eigenvector_service_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-service-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_service_active_public_read_db_path) catch {}; + + const eigenvector_scan_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-scan-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_scan_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_scan_db_path) catch {}; + + const eigenvector_initialize_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-initialize-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_initialize_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_initialize_db_path) catch {}; + + const eigenvector_contribution_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-contribution-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_contribution_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_contribution_db_path) catch {}; + + const eigenvector_reduce_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-reduce-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_reduce_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_reduce_db_path) catch {}; + + const eigenvector_convergence_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-convergence-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_convergence_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_convergence_db_path) catch {}; + + const eigenvector_exhausted_attempt_db_path = ".zig-cache/tmp/graph-metric-process-eigenvector-exhausted-attempt-db"; + std.Io.Dir.cwd().deleteTree(init.io, eigenvector_exhausted_attempt_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, eigenvector_exhausted_attempt_db_path) catch {}; + + const hits_supervisor_db_path = ".zig-cache/tmp/graph-metric-process-hits-supervisor-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_supervisor_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_supervisor_db_path) catch {}; + + const hits_publish_cleanup_db_path = ".zig-cache/tmp/graph-metric-process-hits-publish-cleanup-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_publish_cleanup_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_publish_cleanup_db_path) catch {}; + + const hits_publish_failure_db_path = ".zig-cache/tmp/graph-metric-process-hits-publish-failure-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_publish_failure_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_publish_failure_db_path) catch {}; + + const hits_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-hits-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_active_public_read_db_path) catch {}; + + const hits_service_active_public_read_db_path = ".zig-cache/tmp/graph-metric-process-hits-service-active-public-read-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_service_active_public_read_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_service_active_public_read_db_path) catch {}; + + const hits_exhausted_attempt_db_path = ".zig-cache/tmp/graph-metric-process-hits-exhausted-attempt-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_exhausted_attempt_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_exhausted_attempt_db_path) catch {}; + + const hits_authority_contribution_db_path = ".zig-cache/tmp/graph-metric-process-hits-authority-contribution-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_authority_contribution_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_authority_contribution_db_path) catch {}; + + const hits_authority_reduce_db_path = ".zig-cache/tmp/graph-metric-process-hits-authority-reduce-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_authority_reduce_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_authority_reduce_db_path) catch {}; + + const hits_convergence_db_path = ".zig-cache/tmp/graph-metric-process-hits-convergence-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_convergence_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_convergence_db_path) catch {}; + + const hits_hub_contribution_db_path = ".zig-cache/tmp/graph-metric-process-hits-hub-contribution-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_hub_contribution_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_hub_contribution_db_path) catch {}; + + const hits_hub_reduce_db_path = ".zig-cache/tmp/graph-metric-process-hits-hub-reduce-db"; + std.Io.Dir.cwd().deleteTree(init.io, hits_hub_reduce_db_path) catch {}; + defer std.Io.Dir.cwd().deleteTree(init.io, hits_hub_reduce_db_path) catch {}; + + const target_generation = try seedDegreeDb(alloc, supervisor_db_path); + try runSupervisorProcess(alloc, init.io, antfly_exe, supervisor_db_path); + try verifyDegreeFresh(alloc, supervisor_db_path, target_generation); + + const launch_target_generation = try seedDegreeDb(alloc, launch_db_path); + try runLaunchProcess(alloc, init.io, antfly_exe, launch_db_path, "degree"); + try verifyDegreeFresh(alloc, launch_db_path, launch_target_generation); + release_summary.launch_families += 1; + + const pagerank_launch_target_generation = try seedPageRankDb(alloc, pagerank_launch_db_path); + try runLaunchProcess(alloc, init.io, antfly_exe, pagerank_launch_db_path, "pagerank"); + try verifyMetricFresh(alloc, pagerank_launch_db_path, "pagerank", pagerank_launch_target_generation); + release_summary.launch_families += 1; + + const eigenvector_launch_target_generation = try seedEigenvectorDb(alloc, eigenvector_launch_db_path); + try runLaunchProcess(alloc, init.io, antfly_exe, eigenvector_launch_db_path, "eigenvector"); + try verifyMetricFresh(alloc, eigenvector_launch_db_path, "eigenvector", eigenvector_launch_target_generation); + release_summary.launch_families += 1; + + const hits_launch_target_generation = try seedHitsDbWithSources(alloc, hits_launch_db_path, 8); + try runLaunchProcess(alloc, init.io, antfly_exe, hits_launch_db_path, "hits"); + try verifyHitsFresh(alloc, hits_launch_db_path, hits_launch_target_generation); + release_summary.launch_families += 1; + + _ = try seedDegreeDb(alloc, lease_db_path); + try verifyCoordinatorLeaseExpiryTakeover(alloc, init.io, antfly_exe, lease_db_path); + + const service_owner_target_generation = try seedDegreeDb(alloc, service_owner_db_path); + try verifyServiceTargetedMetricOwnerRestartProcess(alloc, init.io, antfly_exe, service_owner_db_path, "degree", service_owner_target_generation); + try verifyDegreeFresh(alloc, service_owner_db_path, service_owner_target_generation); + release_summary.service_owner_restart_families += 1; + + const degree_service_publish_cleanup_target_generation = try seedDegreeDb(alloc, degree_service_publish_cleanup_db_path); + try verifyDegreeServiceTargetedPublishAndCleanupRestartProcess( + alloc, + init.io, + antfly_exe, + degree_service_publish_cleanup_db_path, + degree_service_publish_cleanup_target_generation, + ); + release_summary.service_publish_cleanup_families += 1; + release_summary.service_cleanup_takeover_families += 1; + + const degree_service_multipage_target_generation = try seedDegreeDbWithSources(alloc, degree_service_multipage_db_path, 130); + try verifyDegreeServiceTargetedMultiPageWorkerPoolProcess( + alloc, + init.io, + antfly_exe, + degree_service_multipage_db_path, + degree_service_multipage_target_generation, + ); + release_summary.service_multipage_worker_pool_families += 1; + release_summary.service_multipage_coordinator_takeover_families += 1; + release_summary.service_multipage_worker_pool_takeover_families += 1; + recordServiceMultipagePhaseProofs(&release_summary, 4, 5, 2); + + const pagerank_service_owner_target_generation = try seedPageRankDb(alloc, pagerank_service_owner_db_path); + try verifyServiceTargetedMetricOwnerRestartProcess(alloc, init.io, antfly_exe, pagerank_service_owner_db_path, "pagerank", pagerank_service_owner_target_generation); + try verifyMetricFresh(alloc, pagerank_service_owner_db_path, "pagerank", pagerank_service_owner_target_generation); + release_summary.service_owner_restart_families += 1; + + const pagerank_service_publish_cleanup_target_generation = try seedPageRankDb(alloc, pagerank_service_publish_cleanup_db_path); + try verifyPageRankServiceTargetedPublishAndCleanupRestartProcess( + alloc, + init.io, + antfly_exe, + pagerank_service_publish_cleanup_db_path, + pagerank_service_publish_cleanup_target_generation, + ); + release_summary.service_publish_cleanup_families += 1; + release_summary.service_cleanup_takeover_families += 1; + + const pagerank_service_publish_failure_initial_generation = try seedPageRankDb(alloc, pagerank_service_publish_failure_db_path); + try verifyPageRankServiceTargetedPublishVerifierFailureProcess( + alloc, + init.io, + antfly_exe, + pagerank_service_publish_failure_db_path, + pagerank_service_publish_failure_initial_generation, + ); + release_summary.service_publish_failure_families += 1; + + const pagerank_service_multipage_target_generation = try seedPageRankDbWithSources(alloc, pagerank_service_multipage_db_path, 130); + try verifyPageRankServiceTargetedMultiPageWorkerPoolProcess( + alloc, + init.io, + antfly_exe, + pagerank_service_multipage_db_path, + pagerank_service_multipage_target_generation, + ); + release_summary.service_multipage_worker_pool_families += 1; + release_summary.service_multipage_coordinator_takeover_families += 1; + release_summary.service_multipage_worker_pool_takeover_families += 1; + recordServiceMultipagePhaseProofs(&release_summary, 7, 8, 2); + + const eigenvector_service_owner_target_generation = try seedEigenvectorDb(alloc, eigenvector_service_owner_db_path); + try verifyServiceTargetedMetricOwnerRestartProcess(alloc, init.io, antfly_exe, eigenvector_service_owner_db_path, "eigenvector", eigenvector_service_owner_target_generation); + try verifyMetricFresh(alloc, eigenvector_service_owner_db_path, "eigenvector", eigenvector_service_owner_target_generation); + release_summary.service_owner_restart_families += 1; + + const eigenvector_service_publish_cleanup_target_generation = try seedEigenvectorDbWithSources(alloc, eigenvector_service_publish_cleanup_db_path, 130); + try verifyEigenvectorServiceTargetedPublishAndCleanupRestartProcess( + alloc, + init.io, + antfly_exe, + eigenvector_service_publish_cleanup_db_path, + eigenvector_service_publish_cleanup_target_generation, + ); + release_summary.service_publish_cleanup_families += 1; + release_summary.service_cleanup_takeover_families += 1; + + const eigenvector_service_publish_failure_initial_generation = try seedEigenvectorDb(alloc, eigenvector_service_publish_failure_db_path); + try verifyEigenvectorServiceTargetedPublishVerifierFailureProcess( + alloc, + init.io, + antfly_exe, + eigenvector_service_publish_failure_db_path, + eigenvector_service_publish_failure_initial_generation, + ); + release_summary.service_publish_failure_families += 1; + + const eigenvector_service_multipage_target_generation = try seedEigenvectorDbWithSources(alloc, eigenvector_service_multipage_db_path, 130); + try verifyEigenvectorServiceTargetedMultiPageWorkerPoolProcess( + alloc, + init.io, + antfly_exe, + eigenvector_service_multipage_db_path, + eigenvector_service_multipage_target_generation, + ); + release_summary.service_multipage_worker_pool_families += 1; + release_summary.service_multipage_coordinator_takeover_families += 1; + release_summary.service_multipage_worker_pool_takeover_families += 1; + recordServiceMultipagePhaseProofs(&release_summary, 7, 8, 2); + + const hits_service_owner_target_generation = try seedHitsBackgroundDb(alloc, hits_service_owner_db_path); + try verifyServiceTargetedMetricOwnerRestartProcess(alloc, init.io, antfly_exe, hits_service_owner_db_path, "hits_authority", hits_service_owner_target_generation); + try verifyHitsFresh(alloc, hits_service_owner_db_path, hits_service_owner_target_generation); + release_summary.service_owner_restart_families += 1; + + const hits_service_publish_cleanup_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_service_publish_cleanup_db_path); + try verifyHitsServiceTargetedPublishAndCleanupRestartProcess( + alloc, + init.io, + antfly_exe, + hits_service_publish_cleanup_db_path, + hits_service_publish_cleanup_target_generation, + ); + release_summary.service_publish_cleanup_families += 1; + release_summary.service_cleanup_takeover_families += 1; + + const hits_service_publish_failure_initial_generation = try seedHitsDbWithActiveBuild(alloc, hits_service_publish_failure_db_path); + try verifyHitsServiceTargetedPublishVerifierFailureProcess( + alloc, + init.io, + antfly_exe, + hits_service_publish_failure_db_path, + hits_service_publish_failure_initial_generation, + ); + release_summary.service_publish_failure_families += 1; + + const hits_service_multipage_target_generation = try seedHitsDbWithSources(alloc, hits_service_multipage_db_path, 130); + try verifyHitsServiceTargetedMultiPageWorkerPoolProcess( + alloc, + init.io, + antfly_exe, + hits_service_multipage_db_path, + hits_service_multipage_target_generation, + ); + release_summary.service_multipage_worker_pool_families += 1; + release_summary.service_multipage_coordinator_takeover_families += 1; + release_summary.service_multipage_worker_pool_takeover_families += 1; + recordServiceMultipagePhaseProofs(&release_summary, 9, 10, 2); + + const degree_active_public_read_initial_generation = try seedDegreeSearchDb(alloc, degree_active_public_read_db_path); + try verifyDegreeActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + degree_active_public_read_db_path, + degree_active_public_read_initial_generation, + ); + release_summary.direct_active_public_read_families += 1; + + const degree_service_active_public_read_initial_generation = try seedDegreeSearchDb(alloc, degree_service_active_public_read_db_path); + try verifyDegreeServiceActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + degree_service_active_public_read_db_path, + degree_service_active_public_read_initial_generation, + ); + release_summary.service_active_public_read_families += 1; + + const pagerank_service_active_public_read_initial_generation = try seedPageRankSearchDb(alloc, pagerank_service_active_public_read_db_path); + try verifyPageRankServiceActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + pagerank_service_active_public_read_db_path, + pagerank_service_active_public_read_initial_generation, + ); + release_summary.service_active_public_read_families += 1; + + const worker_page_target_generation = try seedDegreeDb(alloc, worker_page_db_path); + try verifyWorkerPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, worker_page_db_path, worker_page_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const worker_runtime_target_generation = try seedDegreeDbWithSources(alloc, worker_runtime_db_path, 130); + try verifyWorkerRuntimeSameWorkerLeaseFencing(alloc, init.io, antfly_exe, worker_runtime_db_path, worker_runtime_target_generation); + release_summary.same_worker_fencing_proofs += 1; + + const publish_cleanup_target_generation = try seedDegreeDb(alloc, publish_cleanup_db_path); + try verifyPublishAndCleanupRestart(alloc, init.io, antfly_exe, publish_cleanup_db_path, publish_cleanup_target_generation); + release_summary.direct_publish_cleanup_families += 1; + + const pagerank_scan_target_generation = try seedPageRankDb(alloc, pagerank_scan_db_path); + try verifyPageRankScanPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_scan_db_path, pagerank_scan_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_initialize_target_generation = try seedPageRankDb(alloc, pagerank_initialize_db_path); + try verifyPageRankInitializePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_initialize_db_path, pagerank_initialize_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_contribution_target_generation = try seedPageRankDb(alloc, pagerank_contribution_db_path); + try verifyPageRankContributionPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_contribution_db_path, pagerank_contribution_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_reduce_target_generation = try seedPageRankDb(alloc, pagerank_reduce_db_path); + try verifyPageRankReducePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_reduce_db_path, pagerank_reduce_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_convergence_target_generation = try seedPageRankDb(alloc, pagerank_convergence_db_path); + try verifyPageRankConvergencePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_convergence_db_path, pagerank_convergence_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_publish_cleanup_target_generation = try seedPageRankDb(alloc, pagerank_publish_cleanup_db_path); + try verifyPageRankPublishAndCleanupRestart(alloc, init.io, antfly_exe, pagerank_publish_cleanup_db_path, pagerank_publish_cleanup_target_generation); + release_summary.direct_publish_cleanup_families += 1; + + const pagerank_cleanup_reclaim_target_generation = try seedPageRankDb(alloc, pagerank_cleanup_reclaim_db_path); + try verifyPageRankCleanupPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_cleanup_reclaim_db_path, pagerank_cleanup_reclaim_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_publish_failure_initial_generation = try seedPageRankDb(alloc, pagerank_publish_failure_db_path); + try verifyPageRankPublishVerifierFailureProcess(alloc, init.io, antfly_exe, pagerank_publish_failure_db_path, pagerank_publish_failure_initial_generation); + release_summary.direct_publish_failure_families += 1; + + const pagerank_fixed_iteration_target_generation = try seedPageRankDbWithMaxIterations(alloc, pagerank_fixed_iteration_db_path, 2); + try runSupervisorProcess(alloc, init.io, antfly_exe, pagerank_fixed_iteration_db_path); + try verifyPageRankFixedIterationMetadata( + alloc, + pagerank_fixed_iteration_db_path, + pagerank_fixed_iteration_target_generation, + 2, + ); + release_summary.fixed_iteration_families += 1; + + const pagerank_active_public_read_initial_generation = try seedPageRankSearchDb(alloc, pagerank_active_public_read_db_path); + try verifyPageRankActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + pagerank_active_public_read_db_path, + pagerank_active_public_read_initial_generation, + ); + release_summary.direct_active_public_read_families += 1; + + const pagerank_same_worker_target_generation = try seedPageRankDb(alloc, pagerank_same_worker_db_path); + try verifyPageRankSameWorkerReplacementAttemptFence(alloc, init.io, harness_exe, antfly_exe, pagerank_same_worker_db_path, pagerank_same_worker_target_generation); + release_summary.same_worker_fencing_proofs += 1; + + const pagerank_later_contribution_target_generation = try seedPageRankDbWithMaxIterations(alloc, pagerank_later_contribution_db_path, 2); + try verifyPageRankLaterContributionPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_later_contribution_db_path, pagerank_later_contribution_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_later_reduce_target_generation = try seedPageRankDbWithMaxIterations(alloc, pagerank_later_reduce_db_path, 2); + try verifyPageRankLaterReducePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_later_reduce_db_path, pagerank_later_reduce_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_later_convergence_target_generation = try seedPageRankDbWithMaxIterations(alloc, pagerank_later_convergence_db_path, 2); + try verifyPageRankLaterConvergencePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, pagerank_later_convergence_db_path, pagerank_later_convergence_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const pagerank_exhausted_attempt_initial_generation = try seedPageRankDbWithMaxIterations(alloc, pagerank_exhausted_attempt_db_path, 2); + try verifyPageRankExhaustedAttemptProcess(alloc, init.io, harness_exe, antfly_exe, pagerank_exhausted_attempt_db_path, pagerank_exhausted_attempt_initial_generation); + release_summary.exhausted_attempt_families += 1; + + const eigenvector_supervisor_target_generation = try seedEigenvectorDb(alloc, eigenvector_supervisor_db_path); + try runSupervisorProcess(alloc, init.io, antfly_exe, eigenvector_supervisor_db_path); + try verifyMetricFresh(alloc, eigenvector_supervisor_db_path, "eigenvector", eigenvector_supervisor_target_generation); + + const eigenvector_fixed_iteration_target_generation = try seedEigenvectorDbWithMaxIterations(alloc, eigenvector_fixed_iteration_db_path, 1); + try runSupervisorProcess(alloc, init.io, antfly_exe, eigenvector_fixed_iteration_db_path); + try verifyFixedIterationMetadata( + alloc, + eigenvector_fixed_iteration_db_path, + "eigenvector", + eigenvector_fixed_iteration_target_generation, + 1, + ); + release_summary.fixed_iteration_families += 1; + + const eigenvector_publish_cleanup_target_generation = try seedEigenvectorDb(alloc, eigenvector_publish_cleanup_db_path); + try verifyEigenvectorPublishAndCleanupRestart(alloc, init.io, antfly_exe, eigenvector_publish_cleanup_db_path, eigenvector_publish_cleanup_target_generation); + release_summary.direct_publish_cleanup_families += 1; + + const eigenvector_publish_failure_initial_generation = try seedEigenvectorDb(alloc, eigenvector_publish_failure_db_path); + try verifyEigenvectorPublishVerifierFailureProcess(alloc, init.io, antfly_exe, eigenvector_publish_failure_db_path, eigenvector_publish_failure_initial_generation); + release_summary.direct_publish_failure_families += 1; + + const eigenvector_active_public_read_initial_generation = try seedEigenvectorSearchDb(alloc, eigenvector_active_public_read_db_path); + try verifyEigenvectorActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + eigenvector_active_public_read_db_path, + eigenvector_active_public_read_initial_generation, + ); + release_summary.direct_active_public_read_families += 1; + + const eigenvector_service_active_public_read_initial_generation = try seedEigenvectorSearchDb(alloc, eigenvector_service_active_public_read_db_path); + try verifyEigenvectorServiceActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + eigenvector_service_active_public_read_db_path, + eigenvector_service_active_public_read_initial_generation, + ); + release_summary.service_active_public_read_families += 1; + + const eigenvector_scan_target_generation = try seedEigenvectorDb(alloc, eigenvector_scan_db_path); + try verifyEigenvectorScanPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, eigenvector_scan_db_path, eigenvector_scan_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const eigenvector_initialize_target_generation = try seedEigenvectorDb(alloc, eigenvector_initialize_db_path); + try verifyEigenvectorInitializePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, eigenvector_initialize_db_path, eigenvector_initialize_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const eigenvector_contribution_target_generation = try seedEigenvectorDb(alloc, eigenvector_contribution_db_path); + try verifyEigenvectorContributionPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, eigenvector_contribution_db_path, eigenvector_contribution_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const eigenvector_reduce_target_generation = try seedEigenvectorDb(alloc, eigenvector_reduce_db_path); + try verifyEigenvectorReducePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, eigenvector_reduce_db_path, eigenvector_reduce_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const eigenvector_convergence_target_generation = try seedEigenvectorDb(alloc, eigenvector_convergence_db_path); + try verifyEigenvectorConvergencePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, eigenvector_convergence_db_path, eigenvector_convergence_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const eigenvector_exhausted_attempt_initial_generation = try seedEigenvectorDbWithMaxIterations(alloc, eigenvector_exhausted_attempt_db_path, 2); + try verifyEigenvectorExhaustedAttemptProcess(alloc, init.io, harness_exe, antfly_exe, eigenvector_exhausted_attempt_db_path, eigenvector_exhausted_attempt_initial_generation); + release_summary.exhausted_attempt_families += 1; + + const hits_supervisor_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_supervisor_db_path); + try runSupervisorProcess(alloc, init.io, antfly_exe, hits_supervisor_db_path); + try verifyHitsFresh(alloc, hits_supervisor_db_path, hits_supervisor_target_generation); + + const hits_fixed_iteration_target_generation = try seedHitsDbWithActiveBuildMaxIterations(alloc, hits_fixed_iteration_db_path, 1, 0.000001); + try runSupervisorProcess(alloc, init.io, antfly_exe, hits_fixed_iteration_db_path); + try verifyHitsFixedIterationMetadata( + alloc, + hits_fixed_iteration_db_path, + hits_fixed_iteration_target_generation, + 1, + ); + release_summary.fixed_iteration_families += 1; + + const hits_publish_cleanup_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_publish_cleanup_db_path); + try verifyHitsPublishAndCleanupRestart(alloc, init.io, antfly_exe, hits_publish_cleanup_db_path, hits_publish_cleanup_target_generation); + release_summary.direct_publish_cleanup_families += 1; + + const hits_publish_failure_initial_generation = try seedHitsDbWithActiveBuild(alloc, hits_publish_failure_db_path); + try verifyHitsPublishVerifierFailureProcess(alloc, init.io, antfly_exe, hits_publish_failure_db_path, hits_publish_failure_initial_generation); + release_summary.direct_publish_failure_families += 1; + + const hits_active_public_read_initial_generation = try seedHitsBackgroundDb(alloc, hits_active_public_read_db_path); + try verifyHitsActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + hits_active_public_read_db_path, + hits_active_public_read_initial_generation, + ); + release_summary.direct_active_public_read_families += 1; + + const hits_service_active_public_read_initial_generation = try seedHitsBackgroundDb(alloc, hits_service_active_public_read_db_path); + try verifyHitsServiceActiveProcessPublicReadFreshness( + alloc, + init.io, + antfly_exe, + hits_service_active_public_read_db_path, + hits_service_active_public_read_initial_generation, + ); + release_summary.service_active_public_read_families += 1; + + const hits_exhausted_attempt_initial_generation = try seedHitsDbWithActiveBuildMaxIterations(alloc, hits_exhausted_attempt_db_path, 2, 0.000001); + try verifyHitsExhaustedAttemptProcess(alloc, init.io, harness_exe, antfly_exe, hits_exhausted_attempt_db_path, hits_exhausted_attempt_initial_generation); + release_summary.exhausted_attempt_families += 1; + + const hits_authority_contribution_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_authority_contribution_db_path); + try verifyHitsAuthorityContributionPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, hits_authority_contribution_db_path, hits_authority_contribution_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const hits_authority_reduce_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_authority_reduce_db_path); + try verifyHitsAuthorityReducePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, hits_authority_reduce_db_path, hits_authority_reduce_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const hits_convergence_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_convergence_db_path); + try verifyHitsConvergencePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, hits_convergence_db_path, hits_convergence_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const hits_hub_contribution_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_hub_contribution_db_path); + try verifyHitsHubContributionPageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, hits_hub_contribution_db_path, hits_hub_contribution_target_generation); + recordDirectPageReclaimProof(&release_summary); + + const hits_hub_reduce_target_generation = try seedHitsDbWithActiveBuild(alloc, hits_hub_reduce_db_path); + try verifyHitsHubReducePageLeaseReclaim(alloc, init.io, harness_exe, antfly_exe, hits_hub_reduce_db_path, hits_hub_reduce_target_generation); + recordDirectPageReclaimProof(&release_summary); + + try verifyProcessHarnessReleaseSummary(release_summary); + try emitProcessHarnessReleaseSummary(init.io, profile, release_summary); +} + +fn parseHarnessProfile(value: []const u8) !HarnessProfile { + if (std.mem.eql(u8, value, "smoke")) return .smoke; + if (std.mem.eql(u8, value, "promotion")) return .promotion; + return error.InvalidGraphMetricProcessHarnessProfile; +} + +fn verifyProcessHarnessReleaseSummary(summary: ProcessHarnessReleaseSummary) !void { + const required = required_process_harness_release_summary; + if (!hasRemoteOwnerReleaseGate(summary, required)) { + return error.GraphMetricProcessReleaseCoverageMissing; + } +} + +fn hasRemoteOwnerReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasServiceRemoteOwnerReleaseGate(summary, required) and + hasDirectRemoteOwnerReleaseGate(summary, required) and + hasFailureReclaimReleaseGate(summary, required); +} + +fn hasServiceRemoteOwnerReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasServiceLifecycleReleaseGate(summary, required) and + hasServiceMultipageReleaseGate(summary, required) and + hasServiceActiveReadReleaseGate(summary, required); +} + +fn hasServiceLifecycleReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.launch_families == required.launch_families and + summary.service_owner_restart_families == required.service_owner_restart_families and + summary.service_publish_cleanup_families == required.service_publish_cleanup_families and + summary.service_publish_failure_families == required.service_publish_failure_families and + summary.service_cleanup_takeover_families == required.service_cleanup_takeover_families; +} + +fn hasServiceMultipageReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.service_multipage_worker_pool_families == required.service_multipage_worker_pool_families and + summary.service_multipage_coordinator_takeover_families == required.service_multipage_coordinator_takeover_families and + summary.service_multipage_worker_pool_takeover_families == required.service_multipage_worker_pool_takeover_families and + summary.service_multipage_worker_phase_proofs == required.service_multipage_worker_phase_proofs and + summary.service_multipage_coordinator_phase_proofs == required.service_multipage_coordinator_phase_proofs and + summary.service_multipage_takeover_phase_proofs == required.service_multipage_takeover_phase_proofs; +} + +fn hasServiceActiveReadReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.service_active_public_read_families == required.service_active_public_read_families; +} + +fn hasDirectPublishReadReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.direct_publish_cleanup_families == required.direct_publish_cleanup_families and + summary.direct_publish_failure_families == required.direct_publish_failure_families and + summary.direct_active_public_read_families == required.direct_active_public_read_families and + summary.fixed_iteration_families == required.fixed_iteration_families; +} + +fn hasDirectReclaimReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.direct_page_reclaim_phase_proofs == required.direct_page_reclaim_phase_proofs and + summary.direct_reclaimed_attempt_completion_phase_proofs == required.direct_reclaimed_attempt_completion_phase_proofs and + summary.direct_stale_attempt_rejection_phase_proofs == required.direct_stale_attempt_rejection_phase_proofs; +} + +fn hasDirectExhaustionFencingReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return summary.exhausted_attempt_families == required.exhausted_attempt_families and + summary.same_worker_fencing_proofs == required.same_worker_fencing_proofs; +} + +fn hasDirectRemoteOwnerReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasDirectPublishReadReleaseGate(summary, required); +} + +fn hasFailureReclaimReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasDirectReclaimReleaseGate(summary, required) and + hasDirectExhaustionFencingReleaseGate(summary, required); +} + +fn hasPublicReadReleaseGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasServiceActiveReadReleaseGate(summary, required) and + hasDirectPublishReadReleaseGate(summary, required); +} + +fn hasRolloutQualificationGate(summary: ProcessHarnessReleaseSummary, required: ProcessHarnessReleaseSummary) bool { + return hasRemoteOwnerReleaseGate(summary, required) and + hasPublicReadReleaseGate(summary, required); +} + +fn emitProcessHarnessReleaseSummary(io: std.Io, profile: HarnessProfile, summary: ProcessHarnessReleaseSummary) !void { + const required = required_process_harness_release_summary; + var stdout_buffer: [4096]u8 = undefined; + var stdout_writer = std.Io.File.stdout().writer(io, &stdout_buffer); + const out = &stdout_writer.interface; + defer out.flush() catch {}; + + try out.print( + "{{\"event\":\"graph_metric_process_harness_summary\",\"profile\":\"{s}\",\"rollout_qualification_gate\":{},\"promotion_profile_floor_configured\":{},\"all_family_execution_configured\":{},\"public_read_release_gate\":{},\"remote_owner_release_gate\":{},\"service_remote_owner_release_gate\":{},\"service_lifecycle_release_gate\":{},\"service_multipage_release_gate\":{},\"service_active_read_release_gate\":{},\"direct_remote_owner_release_gate\":{},\"direct_publish_read_release_gate\":{},\"failure_reclaim_release_gate\":{},\"direct_reclaim_release_gate\":{},\"direct_exhaustion_fencing_release_gate\":{}", + .{ + @tagName(profile), + hasRolloutQualificationGate(summary, required), + profile == .promotion, + required.launch_families == 4 and summary.launch_families == required.launch_families, + hasPublicReadReleaseGate(summary, required), + hasRemoteOwnerReleaseGate(summary, required), + hasServiceRemoteOwnerReleaseGate(summary, required), + hasServiceLifecycleReleaseGate(summary, required), + hasServiceMultipageReleaseGate(summary, required), + hasServiceActiveReadReleaseGate(summary, required), + hasDirectRemoteOwnerReleaseGate(summary, required), + hasDirectPublishReadReleaseGate(summary, required), + hasFailureReclaimReleaseGate(summary, required), + hasDirectReclaimReleaseGate(summary, required), + hasDirectExhaustionFencingReleaseGate(summary, required), + }, + ); + try out.print( + ",\"required_launch_families\":{d},\"launch_families\":{d},\"required_service_owner_restart_families\":{d},\"service_owner_restart_families\":{d},\"required_service_publish_cleanup_families\":{d},\"service_publish_cleanup_families\":{d},\"required_service_publish_failure_families\":{d},\"service_publish_failure_families\":{d},\"required_service_multipage_worker_pool_families\":{d},\"service_multipage_worker_pool_families\":{d},\"required_service_multipage_coordinator_takeover_families\":{d},\"service_multipage_coordinator_takeover_families\":{d},\"required_service_multipage_worker_pool_takeover_families\":{d},\"service_multipage_worker_pool_takeover_families\":{d},\"required_service_multipage_worker_phase_proofs\":{d},\"service_multipage_worker_phase_proofs\":{d},\"required_service_multipage_coordinator_phase_proofs\":{d},\"service_multipage_coordinator_phase_proofs\":{d},\"required_service_multipage_takeover_phase_proofs\":{d},\"service_multipage_takeover_phase_proofs\":{d},\"required_service_cleanup_takeover_families\":{d},\"service_cleanup_takeover_families\":{d},\"required_service_active_public_read_families\":{d},\"service_active_public_read_families\":{d}", + .{ + required.launch_families, + summary.launch_families, + required.service_owner_restart_families, + summary.service_owner_restart_families, + required.service_publish_cleanup_families, + summary.service_publish_cleanup_families, + required.service_publish_failure_families, + summary.service_publish_failure_families, + required.service_multipage_worker_pool_families, + summary.service_multipage_worker_pool_families, + required.service_multipage_coordinator_takeover_families, + summary.service_multipage_coordinator_takeover_families, + required.service_multipage_worker_pool_takeover_families, + summary.service_multipage_worker_pool_takeover_families, + required.service_multipage_worker_phase_proofs, + summary.service_multipage_worker_phase_proofs, + required.service_multipage_coordinator_phase_proofs, + summary.service_multipage_coordinator_phase_proofs, + required.service_multipage_takeover_phase_proofs, + summary.service_multipage_takeover_phase_proofs, + required.service_cleanup_takeover_families, + summary.service_cleanup_takeover_families, + required.service_active_public_read_families, + summary.service_active_public_read_families, + }, + ); + try out.print( + ",\"required_direct_publish_cleanup_families\":{d},\"direct_publish_cleanup_families\":{d},\"required_direct_publish_failure_families\":{d},\"direct_publish_failure_families\":{d},\"required_direct_active_public_read_families\":{d},\"direct_active_public_read_families\":{d},\"required_direct_page_reclaim_phase_proofs\":{d},\"direct_page_reclaim_phase_proofs\":{d},\"required_direct_reclaimed_attempt_completion_phase_proofs\":{d},\"direct_reclaimed_attempt_completion_phase_proofs\":{d},\"required_direct_stale_attempt_rejection_phase_proofs\":{d},\"direct_stale_attempt_rejection_phase_proofs\":{d},\"required_fixed_iteration_families\":{d},\"fixed_iteration_families\":{d},\"required_exhausted_attempt_families\":{d},\"exhausted_attempt_families\":{d},\"required_same_worker_fencing_proofs\":{d},\"same_worker_fencing_proofs\":{d}}}\n", + .{ + required.direct_publish_cleanup_families, + summary.direct_publish_cleanup_families, + required.direct_publish_failure_families, + summary.direct_publish_failure_families, + required.direct_active_public_read_families, + summary.direct_active_public_read_families, + required.direct_page_reclaim_phase_proofs, + summary.direct_page_reclaim_phase_proofs, + required.direct_reclaimed_attempt_completion_phase_proofs, + summary.direct_reclaimed_attempt_completion_phase_proofs, + required.direct_stale_attempt_rejection_phase_proofs, + summary.direct_stale_attempt_rejection_phase_proofs, + required.fixed_iteration_families, + summary.fixed_iteration_families, + required.exhausted_attempt_families, + summary.exhausted_attempt_families, + required.same_worker_fencing_proofs, + summary.same_worker_fencing_proofs, + }, + ); +} + +fn seedDegreeDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + return seedDegreeDbWithSources(alloc, db_path, 8); +} + +fn seedDegreeDbWithSources(alloc: std.mem.Allocator, db_path: []const u8, source_count: usize) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..source_count) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedDegreeSearchDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\",\"body\":\"hub graph\"}" }}, + .sync_level = .write, + }); + for (0..8) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"body\":\"oldsource graph {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{ i, i }, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.batch(.{ + .writes = &.{.{ .key = "doc:side", .value = "{\"title\":\"side\",\"body\":\"oldsource side graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:0\",\"weight\":1.0}]}}}" }}, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedPageRankDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + return seedPageRankDbWithMaxIterations(alloc, db_path, 1); +} + +fn seedPageRankDbWithSources(alloc: std.mem.Allocator, db_path: []const u8, source_count: usize) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + for (0..source_count) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:pr:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"pagerank source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedPageRankDbWithMaxIterations( + alloc: std.mem.Allocator, + db_path: []const u8, + max_iterations: u32, +) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const config_json = try std.fmt.allocPrint( + alloc, + "{{\"metrics\":{{\"pagerank\":{{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":{d},\"tolerance\":0.000001,\"edge_filter\":{{\"types\":[\"cites\"]}}}}}}}}", + .{max_iterations}, + ); + defer alloc.free(config_json); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = config_json, + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedPageRankSearchDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"body\":\"alpha graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"body\":\"beta graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"body\":\"gamma graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\",\"body\":\"delta graph\"}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedEigenvectorDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + return seedEigenvectorDbWithMaxIterations(alloc, db_path, 2); +} + +fn seedEigenvectorDbWithSources(alloc: std.mem.Allocator, db_path: []const u8, source_count: usize) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:hub\",\"weight\":1.0}]}}}" }}, + .sync_level = .write, + }); + for (0..source_count) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:ev:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"eigenvector source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedEigenvectorDbWithMaxIterations( + alloc: std.mem.Allocator, + db_path: []const u8, + max_iterations: u32, +) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const config_json = try std.fmt.allocPrint( + alloc, + "{{\"metrics\":{{\"eigenvector\":{{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":{d},\"tolerance\":0.000001,\"edge_filter\":{{\"types\":[\"cites\"]}}}}}}}}", + .{max_iterations}, + ); + defer alloc.free(config_json); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = config_json, + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0},{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:a\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedEigenvectorSearchDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"body\":\"alpha graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0},{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"body\":\"beta graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"body\":\"gamma graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:a\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\",\"body\":\"delta graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedHitsDbWithActiveBuild(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + return seedHitsDbWithActiveBuildMaxIterations(alloc, db_path, 1, 0.000001); +} + +fn seedHitsDbWithSources(alloc: std.mem.Allocator, db_path: []const u8, source_count: usize) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }}, + .sync_level = .write, + }); + for (0..source_count) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:hits:{d}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"hits source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:authority\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn seedHitsDbWithActiveBuildMaxIterations( + alloc: std.mem.Allocator, + db_path: []const u8, + max_iterations: u32, + tolerance: f64, +) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + const config_json = try std.fmt.allocPrint( + alloc, + "{{\"metrics\":{{\"hits_authority\":{{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":{d},\"tolerance\":{d},\"edge_filter\":{{\"types\":[\"cites\"]}}}},\"hits_hub\":{{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":{d},\"tolerance\":{d},\"edge_filter\":{{\"types\":[\"cites\"]}}}}}}}}", + .{ max_iterations, tolerance, max_iterations, tolerance }, + ); + defer alloc.free(config_json); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = config_json, + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"body\":\"hub a graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"body\":\"hub b graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ + .key = "doc:authority", + .value = if (max_iterations > 1) + "{\"title\":\"authority\",\"body\":\"authority graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" + else + "{\"title\":\"authority\",\"body\":\"authority graph\"}", + }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer started.deinit(alloc); + if (started.state != antfly.graph.GraphIndex.GraphMetricState.building or started.building_generation != target_generation) { + return error.GraphMetricBuildNotStarted; + } + return target_generation; +} + +fn seedHitsBackgroundDb(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"body\":\"hub a graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"body\":\"hub b graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\",\"body\":\"authority graph\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn addPageRankDirtyEdge(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:e", .value = "{\"title\":\"epsilon\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn addDegreeDirtyEdge(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:new", + .value = "{\"title\":\"new source\",\"body\":\"newsource graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:hub\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn addEigenvectorDirtyEdge(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:e", .value = "{\"title\":\"epsilon\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn addHitsDirtyEdge(alloc: std.mem.Allocator, db_path: []const u8) !u64 { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-c", .value = "{\"title\":\"hub c\",\"body\":\"hub c graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + return graph_entry.index.edge_generation; +} + +fn runClaimDegreePageHoldMode( + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, +) !void { + if (argv.len != 7) { + std.debug.print("usage: graph_metric_process_harness claim-degree-page-hold \n", .{}); + std.process.exit(2); + } + const db_path = argv[2]; + const worker_id = argv[3]; + const now_ms = try std.fmt.parseInt(u64, argv[4], 10); + const ready_file = argv[5]; + const hold_ms = try std.fmt.parseInt(u64, argv[6], 10); + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree) { + return error.GraphMetricUnexpectedPhase; + } + const page = try graph_entry.index.claimNextGraphMetricBuildPageAt( + "degree", + status.build_job_id, + .scan_edges_and_out_degree, + 0, + worker_id, + now_ms, + ) orelse return error.GraphMetricExpectedPageClaim; + _ = try graph_entry.index.updateGraphMetricBuildPageProgressForAttempt( + "degree", + status.build_job_id, + .scan_edges_and_out_degree, + 0, + page.page_id, + worker_id, + page.attempt, + "process-dead-cursor", + if (page.total_units > 0) 1 else 0, + page.total_units, + ); + + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = ready_file, + .data = "ready\n", + }); + platform.time.sleepNs(hold_ms * std.time.ns_per_ms); +} + +fn runClaimMetricPageHoldMode( + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, +) !void { + if (argv.len != 9) { + std.debug.print("usage: graph_metric_process_harness claim-metric-page-hold \n", .{}); + std.process.exit(2); + } + const db_path = argv[2]; + const metric_name = argv[3]; + const phase = try parseBuildPhase(argv[4]); + const worker_id = argv[5]; + const now_ms = try std.fmt.parseInt(u64, argv[6], 10); + const ready_file = argv[7]; + const hold_ms = try std.fmt.parseInt(u64, argv[8], 10); + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase != phase) return error.GraphMetricUnexpectedPhase; + const iteration: u32 = if (phase == .cleanup_old_generations) 0 else status.build_iteration; + const page = try graph_entry.index.claimNextGraphMetricBuildPageAt( + metric_name, + status.build_job_id, + phase, + iteration, + worker_id, + now_ms, + ) orelse return error.GraphMetricExpectedPageClaim; + _ = try graph_entry.index.updateGraphMetricBuildPageProgressForAttempt( + metric_name, + status.build_job_id, + phase, + iteration, + page.page_id, + worker_id, + page.attempt, + "process-dead-cursor", + if (page.total_units > 0) 1 else 0, + page.total_units, + ); + + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = ready_file, + .data = "ready\n", + }); + platform.time.sleepNs(hold_ms * std.time.ns_per_ms); +} + +fn parseBuildPhase(raw: []const u8) !antfly.graph.GraphIndex.GraphMetricBuildPhase { + if (std.mem.eql(u8, raw, "scan_edges_and_out_degree")) return .scan_edges_and_out_degree; + if (std.mem.eql(u8, raw, "initialize_ranks")) return .initialize_ranks; + if (std.mem.eql(u8, raw, "iterate_contributions")) return .iterate_contributions; + if (std.mem.eql(u8, raw, "reduce_ranks")) return .reduce_ranks; + if (std.mem.eql(u8, raw, "hits_hub_contributions")) return .hits_hub_contributions; + if (std.mem.eql(u8, raw, "hits_hub_reduce_ranks")) return .hits_hub_reduce_ranks; + if (std.mem.eql(u8, raw, "check_convergence")) return .check_convergence; + if (std.mem.eql(u8, raw, "cleanup_old_generations")) return .cleanup_old_generations; + return error.InvalidArguments; +} + +fn prepareDegreeScanBuild( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "degree", target_generation); + defer started.deinit(alloc); + const prepare_worker = try db.runGraphMetricPlannedWorkerPageStepAt("graph_idx", "degree", "process-prepare-worker", 1000); + if (!prepare_worker.claimed_page or !prepare_worker.completed_page) return error.GraphMetricExpectedPageClaim; + const prepare_coordinator = try db.runGraphMetricPlannedCoordinatorStepAt("graph_idx", "degree", 1001); + if (!prepare_coordinator.advanced_phase) return error.GraphMetricUnexpectedPhase; +} + +fn prepareMetricScanBuild( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", metric_name, target_generation); + defer started.deinit(alloc); + const prepare_worker = try db.runGraphMetricPlannedWorkerPageStepAt("graph_idx", metric_name, "process-prepare-worker", 1000); + if (!prepare_worker.claimed_page or !prepare_worker.completed_page) return error.GraphMetricExpectedPageClaim; + const prepare_coordinator = try db.runGraphMetricPlannedCoordinatorStepAt("graph_idx", metric_name, 1001); + if (!prepare_coordinator.advanced_phase) return error.GraphMetricUnexpectedPhase; +} + +fn prepareMetricBuildToPhase( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, + target_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, +) !void { + return prepareMetricBuildToPhaseAndIteration(alloc, db_path, metric_name, target_generation, target_phase, 0); +} + +fn prepareMetricBuildToPhaseAndIteration( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, + target_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + target_iteration: u32, +) !void { + try prepareMetricScanBuild(alloc, db_path, metric_name, target_generation); + if (target_phase == .scan_edges_and_out_degree and target_iteration == 0) return; + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var now_ms: u64 = 1100; + for (0..128) |_| { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase == target_phase and (status.build_iteration == target_iteration or target_phase == .publish_generation or target_phase == .cleanup_old_generations)) { + if (target_phase == .publish_generation) { + // Iterative metrics materialize an unpublished score epoch on + // bounded worker pages before the coordinator atomically flips + // the published pointer. Drain those pages while deliberately + // leaving the coordinator transition for the process proof. + var publish_drained = false; + for (0..128) |_| { + const publish_worker = try db.runGraphMetricPlannedWorkerPageStepAt( + "graph_idx", + metric_name, + "process-phase-prep-publish-worker", + now_ms, + ); + now_ms += 1; + if (publish_worker.failed_build) return error.GraphMetricBuildFailed; + if (!publish_worker.claimed_page) { + publish_drained = true; + break; + } + } + if (!publish_drained) return error.GraphMetricTargetPhaseNotReached; + } + return; + } + if (status.phase == .publish_generation or status.phase == .cleanup_old_generations or status.phase == .complete) { + std.debug.print("metric {s} advanced past target phase {}, got {}\n", .{ metric_name, target_phase, status.phase }); + return error.GraphMetricUnexpectedPhase; + } + + const worker_step = try db.runGraphMetricPlannedWorkerPageStepAt( + "graph_idx", + metric_name, + "process-phase-prep-worker", + now_ms, + ); + now_ms += 1; + if (worker_step.failed_build) return error.GraphMetricBuildFailed; + + const coordinator_step = try db.runGraphMetricPlannedCoordinatorStepAt("graph_idx", metric_name, now_ms); + now_ms += 1; + if (coordinator_step.failed_build) return error.GraphMetricBuildFailed; + } + return error.GraphMetricTargetPhaseNotReached; +} + +fn prepareDegreePublishReadyBuild( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "degree", target_generation); + defer started.deinit(alloc); + + var now_ms: u64 = 20_000; + for (0..128) |_| { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + switch (status.phase) { + .publish_generation => return, + .cleanup_old_generations, .complete => return error.GraphMetricUnexpectedPhase, + else => {}, + } + + const worker_step = try db.runGraphMetricPlannedWorkerPageStepAt("graph_idx", "degree", "process-publish-prep-worker", now_ms); + now_ms += 1; + if (worker_step.phase == .publish_generation) return; + if (worker_step.completed_page or !worker_step.claimed_page) { + const coordinator_step = try db.runGraphMetricPlannedCoordinatorStepAt("graph_idx", "degree", now_ms); + now_ms += 1; + if (coordinator_step.phase == .publish_generation) return; + } + } + return error.GraphMetricPublishReadyNotReached; +} + +fn assertDegreePhase( + alloc: std.mem.Allocator, + db_path: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_generation: u64, +) !void { + return assertMetricPhase(alloc, db_path, "degree", expected_phase, expected_generation); +} + +fn assertMetricPhase( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase != expected_phase) { + std.debug.print("expected {s} phase {}, got {}\n", .{ metric_name, expected_phase, status.phase }); + return error.GraphMetricUnexpectedPhase; + } + if (expected_generation != 0 and status.published_generation != expected_generation) { + std.debug.print( + "expected published generation {d}, got {d}\n", + .{ expected_generation, status.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } +} + +fn assertHitsAfterPairedPublish( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state != antfly.graph.GraphIndex.GraphMetricState.building or + authority.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations) + { + std.debug.print("expected HITS authority to be cleaning after paired publish, got {}/{}\n", .{ authority.state, authority.phase }); + return error.GraphMetricProcessProofFailed; + } + if (hub.state != antfly.graph.GraphIndex.GraphMetricState.building or + hub.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations) + { + std.debug.print("expected HITS hub to share paired cleanup lifecycle, got {}/{}\n", .{ hub.state, hub.phase }); + return error.GraphMetricProcessProofFailed; + } + if (authority.published_generation != target_generation or hub.published_generation != target_generation) { + std.debug.print( + "expected paired HITS published generation {d}, got authority {d} hub {d}\n", + .{ target_generation, authority.published_generation, hub.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } + if (authority.recent_events.len != 1 or hub.recent_events.len != 1 or + authority.recent_events[0].kind != antfly.graph.GraphIndex.GraphMetricEventKind.publish or + hub.recent_events[0].kind != antfly.graph.GraphIndex.GraphMetricEventKind.publish) + { + std.debug.print("expected paired HITS publish events after process publish\n", .{}); + return error.GraphMetricProcessProofFailed; + } +} + +fn assertMetricSinglePublishEvent( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.published_generation != target_generation) { + std.debug.print("expected {s} published generation {d}, got {d}\n", .{ + metric_name, + target_generation, + status.published_generation, + }); + return error.GraphMetricProcessProofFailed; + } + if (status.recent_events.len != 1 or status.recent_events[0].kind != antfly.graph.GraphIndex.GraphMetricEventKind.publish) { + std.debug.print("expected one {s} publish event after duplicate coordinator process\n", .{metric_name}); + return error.GraphMetricProcessProofFailed; + } +} + +fn assertHitsSinglePublishEventPair( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, +) !void { + try assertMetricSinglePublishEvent(alloc, db_path, "hits_authority", target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "hits_hub", target_generation); +} + +fn assertMetricRecentEventKindCount( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + kind: antfly.graph.GraphIndex.GraphMetricEventKind, + expected_count: usize, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + + var actual_count: usize = 0; + for (status.recent_events) |event| { + if (event.kind == kind) actual_count += 1; + } + if (actual_count != expected_count) { + std.debug.print("expected {s} recent event kind {} count {d}, got {d}\n", .{ + metric_name, + kind, + expected_count, + actual_count, + }); + return error.GraphMetricProcessProofFailed; + } +} + +fn assertHitsRecentEventKindCountPair( + alloc: std.mem.Allocator, + db_path: []const u8, + kind: antfly.graph.GraphIndex.GraphMetricEventKind, + expected_count: usize, +) !void { + try assertMetricRecentEventKindCount(alloc, db_path, "hits_authority", kind, expected_count); + try assertMetricRecentEventKindCount(alloc, db_path, "hits_hub", kind, expected_count); +} + +fn assertDuplicateCoordinatorDidNotMutate(summary: RoleRunSummary, label: []const u8) !void { + if (summary.result.published != 0 or summary.result.failed_builds != 0 or summary.result.phases_advanced != 0) { + std.debug.print("expected duplicate coordinator process not to publish, fail, or advance {s}\n", .{label}); + return error.GraphMetricProcessProofFailed; + } +} + +fn verifyPublishAndCleanupRestart( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareDegreePublishReadyBuild(alloc, db_path, target_generation); + try assertDegreePhase(alloc, db_path, .publish_generation, 0); + + const publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "publish-proof-coordinator", + "5000", + "30000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected coordinator process to publish degree build after restart boundary\n", .{}); + return error.GraphMetricPublishRestartProofFailed; + } + try assertDegreePhase(alloc, db_path, .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "degree", target_generation); + + const duplicate_publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "publish-proof-duplicate-coordinator", + "5000", + "30000", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "degree"); + try assertDegreePhase(alloc, db_path, .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "degree", target_generation); + + const first_cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "cleanup-proof-worker-owner-a", + "cleanup-proof-worker-a", + "5000", + "30001", + ); + if (first_cleanup.result.pages_claimed != 1 or first_cleanup.result.pages_completed != 1) { + std.debug.print("expected first cleanup worker process to complete one cleanup page\n", .{}); + return error.GraphMetricCleanupRestartProofFailed; + } + try assertDegreePhase(alloc, db_path, .cleanup_old_generations, target_generation); + + const final_cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "cleanup-proof-worker-owner-b", + "cleanup-proof-worker-b", + "5000", + "30002", + ); + if (final_cleanup.result.pages_claimed == 0 or final_cleanup.result.pages_completed == 0) { + std.debug.print("expected second cleanup worker process to resume cleanup after restart boundary\n", .{}); + return error.GraphMetricCleanupRestartProofFailed; + } + try verifyDegreeFresh(alloc, db_path, target_generation); +} + +fn verifyPageRankPublishAndCleanupRestart( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", target_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "pagerank", .publish_generation, 0); + + const publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-publish-proof-coordinator", + "5000", + "40000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected coordinator process to publish PageRank build after restart boundary\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertMetricPhase(alloc, db_path, "pagerank", .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "pagerank", target_generation); + + const duplicate_publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-publish-proof-duplicate-coordinator", + "5000", + "40000", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "pagerank"); + try assertMetricPhase(alloc, db_path, "pagerank", .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "pagerank", target_generation); + + const first_cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-proof-worker-owner-a", + "pagerank-cleanup-proof-worker-a", + "5000", + "40001", + ); + if (first_cleanup.result.pages_claimed != 1 or first_cleanup.result.pages_completed != 1) { + std.debug.print("expected first PageRank cleanup worker process to complete one cleanup page\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertMetricPhase(alloc, db_path, "pagerank", .cleanup_old_generations, target_generation); + + const second_cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-proof-worker-owner-b", + "pagerank-cleanup-proof-worker-b", + "5000", + "40002", + ); + if (second_cleanup.result.pages_claimed != 1 or second_cleanup.result.pages_completed != 1) { + std.debug.print("expected second PageRank cleanup worker process to complete one cleanup page\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertMetricPhase(alloc, db_path, "pagerank", .cleanup_old_generations, target_generation); + + const final_cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-proof-worker-owner-c", + "pagerank-cleanup-proof-worker-c", + "5000", + "40003", + ); + if (final_cleanup.result.pages_claimed != 1 or final_cleanup.result.pages_completed != 1) { + std.debug.print("expected final PageRank cleanup worker process to complete cleanup\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try verifyPageRankFixedIterationMetadata(alloc, db_path, target_generation, 1); +} + +fn verifyEigenvectorPublishAndCleanupRestart( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "eigenvector", target_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "eigenvector", .publish_generation, 0); + + const publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-publish-proof-coordinator", + "5000", + "70000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected coordinator process to publish eigenvector build after restart boundary\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertMetricPhase(alloc, db_path, "eigenvector", .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "eigenvector", target_generation); + + const duplicate_publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-publish-proof-duplicate-coordinator", + "5000", + "70000", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "eigenvector"); + try assertMetricPhase(alloc, db_path, "eigenvector", .cleanup_old_generations, target_generation); + try assertMetricSinglePublishEvent(alloc, db_path, "eigenvector", target_generation); + + const cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-cleanup-proof-worker-owner", + "eigenvector-cleanup-proof-worker", + "5000", + "70001", + ); + if (cleanup.result.pages_claimed != 1 or cleanup.result.pages_completed != 1) { + std.debug.print("expected eigenvector cleanup worker process to complete cleanup after restart boundary\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try verifyMetricFresh(alloc, db_path, "eigenvector", target_generation); +} + +fn verifyHitsPublishAndCleanupRestart( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "hits_authority", target_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "hits_authority", .publish_generation, 0); + + const publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-publish-proof-coordinator", + "5000", + "72000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected coordinator process to publish HITS pair after restart boundary\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertHitsAfterPairedPublish(alloc, db_path, target_generation); + try assertHitsSinglePublishEventPair(alloc, db_path, target_generation); + + const duplicate_publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-publish-proof-duplicate-coordinator", + "5000", + "72000", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "hits"); + try assertHitsAfterPairedPublish(alloc, db_path, target_generation); + try assertHitsSinglePublishEventPair(alloc, db_path, target_generation); + + var cleanup_progressed = false; + for (0..12) |i| { + const owner_id = try std.fmt.allocPrint(alloc, "hits-cleanup-proof-worker-owner-{d}", .{i}); + defer alloc.free(owner_id); + const worker_id = try std.fmt.allocPrint(alloc, "hits-cleanup-proof-worker-{d}", .{i}); + defer alloc.free(worker_id); + const now_ms = try std.fmt.allocPrint(alloc, "{d}", .{72001 + i}); + defer alloc.free(now_ms); + const cleanup = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + owner_id, + worker_id, + "5000", + now_ms, + ); + cleanup_progressed = cleanup_progressed or cleanup.durable_progressed or cleanup.result.pages_claimed != 0 or cleanup.result.pages_completed != 0 or cleanup.result.published != 0; + if (cleanup.result.published != 0) { + break; + } + } + verifyHitsFresh(alloc, db_path, target_generation) catch |err| { + if (!cleanup_progressed) { + std.debug.print("expected HITS cleanup worker role processes to advance cleanup\n", .{}); + } else { + std.debug.print("expected HITS cleanup to finish through worker role processes\n", .{}); + } + return err; + }; +} + +fn verifyPageRankCleanupPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", target_generation, .publish_generation); + const publish = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-page-proof-publish-coordinator", + "5000", + "45000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected coordinator process to publish PageRank before cleanup page reclaim proof\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertMetricPhase(alloc, db_path, "pagerank", .cleanup_old_generations, target_generation); + + const ready_file = ".zig-cache/tmp/graph-metric-process-pagerank-cleanup-ready"; + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillMetricPageOwnerAfterReady( + io, + harness_exe, + db_path, + "pagerank", + "cleanup_old_generations", + "process-pagerank-cleanup-dead-worker", + "45001", + ready_file, + ); + + const dead_page = try readLeasedMetricPage( + alloc, + db_path, + "pagerank", + .cleanup_old_generations, + "process-pagerank-cleanup-dead-worker", + "process-dead-cursor", + ); + const before_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms - 1}); + defer alloc.free(before_expiry_text); + _ = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-page-proof-early-owner", + "process-pagerank-cleanup-reclaim-worker", + "5000", + before_expiry_text, + ); + _ = try readLeasedMetricPage( + alloc, + db_path, + "pagerank", + .cleanup_old_generations, + "process-pagerank-cleanup-dead-worker", + "process-dead-cursor", + ); + + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const reclaim_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-cleanup-page-proof-reclaim-owner", + "process-pagerank-cleanup-reclaim-worker", + "5000", + after_expiry_text, + ); + if (reclaim_worker.result.pages_claimed == 0 or reclaim_worker.result.pages_completed == 0) { + std.debug.print("expected PageRank replacement worker to reclaim and complete expired cleanup page lease\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try expectStaleMetricPageAttemptRejected(alloc, db_path, "pagerank", .cleanup_old_generations, dead_page, "process-pagerank-cleanup-dead-worker"); + + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", target_generation); +} + +fn verifyPageRankPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", initial_generation); + + const rebuild_generation = try addPageRankDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "pagerank", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "pagerank"); + + const failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-publish-failure-proof-coordinator", + "5000", + "50000", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected coordinator process to fail PageRank publish verification without publishing\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try verifyMetricFailedPreservesPublished(alloc, db_path, "pagerank", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "pagerank", .failed, 1); + + const duplicate_failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-publish-failure-proof-duplicate-coordinator", + "5000", + "50001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "pagerank publish failure"); + try verifyMetricFailedPreservesPublished(alloc, db_path, "pagerank", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "pagerank", .failed, 1); +} + +fn verifyPageRankServiceTargetedPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", initial_generation); + + const rebuild_generation = try addPageRankDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "pagerank", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "pagerank"); + + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-failure-coordinator", + "service-pagerank-publish-failure-coordinator-a", + "5000", + "86100", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected service coordinator process to fail PageRank publish verification without publishing\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + + const duplicate_failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-failure-coordinator", + "service-pagerank-publish-failure-coordinator-b", + "5000", + "86101", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "service pagerank publish failure"); + } + try verifyMetricFailedPreservesPublished(alloc, db_path, "pagerank", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "pagerank", .failed, 1); +} + +fn verifyEigenvectorPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "eigenvector", initial_generation); + + const rebuild_generation = try addEigenvectorDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "eigenvector", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "eigenvector", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "eigenvector"); + + const failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-publish-failure-proof-coordinator", + "5000", + "71000", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected coordinator process to fail eigenvector publish verification without publishing\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try verifyMetricFailedPreservesPublished(alloc, db_path, "eigenvector", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "eigenvector", .failed, 1); + + const duplicate_failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-publish-failure-proof-duplicate-coordinator", + "5000", + "71001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "eigenvector publish failure"); + try verifyMetricFailedPreservesPublished(alloc, db_path, "eigenvector", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "eigenvector", .failed, 1); +} + +fn verifyEigenvectorServiceTargetedPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "eigenvector", initial_generation); + + const rebuild_generation = try addEigenvectorDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "eigenvector", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "eigenvector", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "eigenvector"); + + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-failure-coordinator", + "service-eigenvector-publish-failure-coordinator-a", + "5000", + "86200", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected service coordinator process to fail eigenvector publish verification without publishing\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + const duplicate_failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-failure-coordinator", + "service-eigenvector-publish-failure-coordinator-b", + "5000", + "86201", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "service eigenvector publish failure"); + } + try verifyMetricFailedPreservesPublished(alloc, db_path, "eigenvector", initial_generation, "InvalidGraphMetricBuildManifest"); + try assertMetricRecentEventKindCount(alloc, db_path, "eigenvector", .failed, 1); +} + +fn verifyPageRankExhaustedAttemptProcess( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", initial_generation); + + const rebuild_generation = try addPageRankDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try verifyMetricExhaustedAttemptProcess( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "pagerank", + initial_generation, + rebuild_generation, + .iterate_contributions, + 1, + "iterate_contributions", + "PageRank", + "pagerank", + 70_000, + ); +} + +fn verifyEigenvectorExhaustedAttemptProcess( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "eigenvector", initial_generation); + + const rebuild_generation = try addEigenvectorDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try verifyMetricExhaustedAttemptProcess( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "eigenvector", + initial_generation, + rebuild_generation, + .iterate_contributions, + 1, + "iterate_contributions", + "eigenvector", + "eigenvector", + 72_000, + ); +} + +fn verifyMetricExhaustedAttemptProcess( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + metric_name: []const u8, + initial_generation: u64, + rebuild_generation: u64, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + iteration: u32, + phase_arg: []const u8, + label: []const u8, + id_prefix: []const u8, + first_now_ms: u64, +) !void { + try prepareMetricBuildToPhaseAndIteration( + alloc, + db_path, + metric_name, + rebuild_generation, + phase, + iteration, + ); + try assertMetricPhase(alloc, db_path, metric_name, phase, initial_generation); + + var now_ms = first_now_ms; + var exhausted_page_id: u64 = 0; + var last_lease_expires_at_ms: u64 = 0; + for (0..3) |attempt_index| { + const worker_id = try std.fmt.allocPrint(alloc, "process-{s}-exhausted-worker-{d}", .{ id_prefix, attempt_index }); + defer alloc.free(worker_id); + const ready_file = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/graph-metric-process-{s}-exhausted-attempt-{d}-ready", .{ id_prefix, attempt_index }); + defer alloc.free(ready_file); + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + + const now_text = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_text); + try runAndKillMetricPageOwnerAfterReady( + io, + harness_exe, + db_path, + metric_name, + phase_arg, + worker_id, + now_text, + ready_file, + ); + const leased = try readSingleLeasedMetricPage( + alloc, + db_path, + metric_name, + phase, + worker_id, + "process-dead-cursor", + ); + if (leased.iteration != iteration) { + std.debug.print("expected exhausted {s} page at iteration {d}, got {d}\n", .{ label, iteration, leased.iteration }); + return error.GraphMetricProcessProofFailed; + } + if (attempt_index == 0) { + exhausted_page_id = leased.page_id; + } else if (leased.page_id != exhausted_page_id) { + std.debug.print("expected {s} exhausted attempts to reclaim page {d}, got {d}\n", .{ label, exhausted_page_id, leased.page_id }); + return error.GraphMetricProcessProofFailed; + } + const expected_attempt: u64 = @intCast(attempt_index + 1); + if (leased.attempt != expected_attempt) { + std.debug.print("expected {s} page attempt {d}, got {d}\n", .{ label, expected_attempt, leased.attempt }); + return error.GraphMetricProcessProofFailed; + } + last_lease_expires_at_ms = leased.lease_expires_at_ms; + now_ms = leased.lease_expires_at_ms + 1; + } + + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{last_lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const coordinator_owner_id = try std.fmt.allocPrint(alloc, "{s}-exhausted-attempt-proof-coordinator", .{id_prefix}); + defer alloc.free(coordinator_owner_id); + const failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + coordinator_owner_id, + "5000", + after_expiry_text, + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected coordinator process to fail exhausted {s} page attempts without publishing\n", .{label}); + return error.GraphMetricProcessProofFailed; + } + const exhaustion_reason = try std.fmt.allocPrint(alloc, "GraphMetricBuildPageAttemptsExhausted: phase={s}, iteration={d}, page_id={d}, attempt=3, cause=GraphMetricBuildPageLeaseExpired", .{ @tagName(phase), iteration, exhausted_page_id }); + defer alloc.free(exhaustion_reason); + try verifyMetricFailedPreservesPublishedAtPhase( + alloc, + db_path, + metric_name, + initial_generation, + exhaustion_reason, + phase, + iteration, + ); + try assertMetricRecentEventKindCount(alloc, db_path, metric_name, .failed, 1); + + const duplicate_coordinator_owner_id = try std.fmt.allocPrint(alloc, "{s}-exhausted-attempt-proof-duplicate-coordinator", .{id_prefix}); + defer alloc.free(duplicate_coordinator_owner_id); + const duplicate_failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + duplicate_coordinator_owner_id, + "5000", + after_expiry_text, + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, label); + try verifyMetricFailedPreservesPublishedAtPhase( + alloc, + db_path, + metric_name, + initial_generation, + exhaustion_reason, + phase, + iteration, + ); + try assertMetricRecentEventKindCount(alloc, db_path, metric_name, .failed, 1); +} + +fn verifyEigenvectorActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "eigenvector", initial_generation); + + const rebuild_generation = try addEigenvectorDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const started = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-active-read-proof-coordinator", + "5000", + "72000", + ); + if (!started.durable_progressed or !started.stats.has_lease) { + std.debug.print("expected coordinator process to start active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "eigenvector-active-read-proof-worker-owner", + "eigenvector-active-read-proof-worker", + "5000", + "72001", + ); + if (!worker.durable_progressed or worker.result.pages_claimed == 0 or worker.result.pages_completed == 0) { + std.debug.print("expected worker process to advance active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 1) { + std.debug.print("expected one eigenvector graph metric result during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const result = published_result.graph_metric_results[0]; + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected active eigenvector query status building, got {}\n", .{result.status.state}); + return error.GraphMetricProcessProofFailed; + } + if (result.status.published_generation != initial_generation or result.status.building_generation != rebuild_generation) { + std.debug.print( + "expected eigenvector published generation {d} and building generation {d}, got {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + result.status.published_generation, + result.status.building_generation, + }, + ); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected active eigenvector published read to serve prior scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:e")) { + std.debug.print("active eigenvector published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricProcessProofFailed; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "eigenvector", + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len == 0) { + std.debug.print("expected eigenvector graph traversal result during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + for (traversal.nodes) |node| { + if (node.metrics.len != 1 or node.metrics[0].score == null) { + std.debug.print("expected traversal published metric projection to serve prior eigenvector score\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + if (traversal.metric_status.len != 1 or traversal.metric_status[0].state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected traversal metric status building during active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + if (traversal.metric_status[0].published_generation != initial_generation or traversal.metric_status[0].building_generation != rebuild_generation) { + std.debug.print("expected traversal status to report eigenvector published/building generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "eigenvector", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "eigenvector", + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "eigenvector", + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected search rerank status building, got {}\n", .{rerank_status.state}); + return error.GraphMetricProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation or rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected eigenvector rerank status to report published/building generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:e")) { + std.debug.print("active eigenvector search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected eigenvector reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected eigenvector reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +fn verifyHitsPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyHitsFresh(alloc, db_path, initial_generation); + + const rebuild_generation = try addHitsDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "hits_authority", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "hits_authority", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "hits_authority"); + + const failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-publish-failure-proof-coordinator", + "5000", + "73000", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected coordinator process to fail HITS publish verification without publishing\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try verifyHitsFailedPreservesPublished(alloc, db_path, initial_generation, "InvalidGraphMetricBuildManifest"); + try assertHitsRecentEventKindCountPair(alloc, db_path, .failed, 1); + + const duplicate_failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-publish-failure-proof-duplicate-coordinator", + "5000", + "73001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "hits publish failure"); + try verifyHitsFailedPreservesPublished(alloc, db_path, initial_generation, "InvalidGraphMetricBuildManifest"); + try assertHitsRecentEventKindCountPair(alloc, db_path, .failed, 1); +} + +fn verifyHitsServiceTargetedPublishVerifierFailureProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyHitsFresh(alloc, db_path, initial_generation); + + const rebuild_generation = try addHitsDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhase(alloc, db_path, "hits_authority", rebuild_generation, .publish_generation); + try assertMetricPhase(alloc, db_path, "hits_authority", .publish_generation, initial_generation); + try invalidateMetricBuildManifestConfigFingerprintForTest(alloc, db_path, "hits_authority"); + + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-failure-coordinator", + "service-hits-publish-failure-coordinator-a", + "5000", + "86300", + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected service coordinator process to fail HITS publish verification without publishing\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + const duplicate_failed = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-failure-coordinator", + "service-hits-publish-failure-coordinator-b", + "5000", + "86301", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "service hits publish failure"); + } + try verifyHitsFailedPreservesPublished(alloc, db_path, initial_generation, "InvalidGraphMetricBuildManifest"); + try assertHitsRecentEventKindCountPair(alloc, db_path, .failed, 1); +} + +fn verifyHitsExhaustedAttemptProcess( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyHitsFresh(alloc, db_path, initial_generation); + + const rebuild_generation = try addHitsDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + try prepareMetricBuildToPhaseAndIteration( + alloc, + db_path, + "hits_authority", + rebuild_generation, + .hits_hub_reduce_ranks, + 1, + ); + try assertMetricPhase(alloc, db_path, "hits_authority", .hits_hub_reduce_ranks, initial_generation); + + var now_ms: u64 = 75_000; + var exhausted_page_id: u64 = 0; + var last_lease_expires_at_ms: u64 = 0; + const worker_ids = [_][]const u8{ + "process-hits-exhausted-worker-a", + "process-hits-exhausted-worker-b", + "process-hits-exhausted-worker-c", + }; + const ready_files = [_][]const u8{ + ".zig-cache/tmp/graph-metric-process-hits-exhausted-attempt-a-ready", + ".zig-cache/tmp/graph-metric-process-hits-exhausted-attempt-b-ready", + ".zig-cache/tmp/graph-metric-process-hits-exhausted-attempt-c-ready", + }; + for (worker_ids, ready_files, 0..) |worker_id, ready_file, attempt_index| { + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + + const now_text = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_text); + try runAndKillMetricPageOwnerAfterReady( + io, + harness_exe, + db_path, + "hits_authority", + "hits_hub_reduce_ranks", + worker_id, + now_text, + ready_file, + ); + const leased = try readLeasedMetricPage( + alloc, + db_path, + "hits_authority", + .hits_hub_reduce_ranks, + worker_id, + "process-dead-cursor", + ); + if (leased.iteration != 1) { + std.debug.print("expected exhausted HITS page at iteration 1, got {d}\n", .{leased.iteration}); + return error.GraphMetricProcessProofFailed; + } + if (attempt_index == 0) { + exhausted_page_id = leased.page_id; + } else if (leased.page_id != exhausted_page_id) { + std.debug.print("expected HITS exhausted attempts to reclaim page {d}, got {d}\n", .{ exhausted_page_id, leased.page_id }); + return error.GraphMetricProcessProofFailed; + } + const expected_attempt = @as(u32, @intCast(attempt_index + 1)); + if (leased.attempt != expected_attempt) { + std.debug.print("expected HITS page attempt {d}, got {d}\n", .{ expected_attempt, leased.attempt }); + return error.GraphMetricProcessProofFailed; + } + last_lease_expires_at_ms = leased.lease_expires_at_ms; + now_ms = leased.lease_expires_at_ms + 1; + } + + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{last_lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-exhausted-attempt-proof-coordinator", + "5000", + after_expiry_text, + ); + if (failed.result.failed_builds != 1 or failed.result.published != 0 or failed.result.phases_advanced != 0) { + std.debug.print("expected coordinator process to fail exhausted HITS page attempts without publishing\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const exhaustion_reason = try std.fmt.allocPrint(alloc, "GraphMetricBuildPageAttemptsExhausted: phase=hits_hub_reduce_ranks, iteration=1, page_id={d}, attempt=3, cause=GraphMetricBuildPageLeaseExpired", .{exhausted_page_id}); + defer alloc.free(exhaustion_reason); + try verifyHitsFailedPreservesPublishedAtPhase( + alloc, + db_path, + initial_generation, + exhaustion_reason, + .hits_hub_reduce_ranks, + 1, + ); + try assertHitsRecentEventKindCountPair(alloc, db_path, .failed, 1); + + const duplicate_failed = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-exhausted-attempt-proof-duplicate-coordinator", + "5000", + after_expiry_text, + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_failed, "hits exhausted attempt failure"); + try verifyHitsFailedPreservesPublishedAtPhase( + alloc, + db_path, + initial_generation, + exhaustion_reason, + .hits_hub_reduce_ranks, + 1, + ); + try assertHitsRecentEventKindCountPair(alloc, db_path, .failed, 1); +} + +fn verifyHitsActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyHitsFresh(alloc, db_path, initial_generation); + + const rebuild_generation = try addHitsDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const started = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-active-read-proof-coordinator", + "5000", + "74000", + ); + if (!started.durable_progressed or !started.stats.has_lease) { + std.debug.print("expected coordinator process to start active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "hits-active-read-proof-worker-owner", + "hits-active-read-proof-worker", + "5000", + "74001", + ); + if (!worker.durable_progressed or worker.result.pages_claimed == 0 or worker.result.pages_completed == 0) { + std.debug.print("expected worker process to advance active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .published, + }, + }, + }, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 2) { + std.debug.print("expected two HITS graph metric results during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (published_result.graph_metric_results) |result| { + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building and + result.status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected active HITS query status building or stale, got {}\n", .{result.status.state}); + return error.GraphMetricProcessProofFailed; + } + if (result.status.published_generation != initial_generation) { + std.debug.print( + "expected HITS published generation {d}, got {d}\n", + .{ initial_generation, result.status.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } + if (result.status.state == antfly.graph.GraphIndex.GraphMetricState.building and result.status.building_generation != rebuild_generation) { + std.debug.print( + "expected HITS building generation {d}, got {d}\n", + .{ + rebuild_generation, + result.status.building_generation, + }, + ); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected active HITS published read to serve prior scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:hub-c")) { + std.debug.print("active HITS published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricProcessProofFailed; + } + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 1, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 1, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .published }, + .{ .name = "hits_hub", .freshness = .published }, + }; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:hub-a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected one HITS graph traversal result during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!std.mem.eql(u8, traversal.nodes[0].key, "doc:authority")) { + std.debug.print("expected HITS traversal to return doc:authority, got {s}\n", .{traversal.nodes[0].key}); + return error.GraphMetricProcessProofFailed; + } + if (traversal.nodes[0].metrics.len != 2) { + std.debug.print("expected HITS traversal to project authority and hub scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.nodes[0].metrics) |metric| { + if (metric.score == null) { + std.debug.print("expected HITS traversal metric {s} to serve a prior published score\n", .{metric.name}); + return error.GraphMetricProcessProofFailed; + } + } + if (traversal.metric_status.len != 2) { + std.debug.print("expected two HITS traversal metric statuses during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.metric_status) |status| { + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building and + status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected HITS traversal status building or stale, got {}\n", .{status.state}); + return error.GraphMetricProcessProofFailed; + } + if (status.published_generation != initial_generation) { + std.debug.print("expected HITS traversal published generation {d}, got {d}\n", .{ initial_generation, status.published_generation }); + return error.GraphMetricGenerationMismatch; + } + if (status.state == antfly.graph.GraphIndex.GraphMetricState.building and status.building_generation != rebuild_generation) { + std.debug.print("expected HITS traversal building generation {d}, got {d}\n", .{ rebuild_generation, status.building_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 3, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building and + rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected HITS search rerank status building or stale, got {}\n", .{rerank_status.state}); + return error.GraphMetricProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation) { + std.debug.print("expected HITS rerank published generation {d}, got {d}\n", .{ initial_generation, rerank_status.published_generation }); + return error.GraphMetricGenerationMismatch; + } + if (rerank_status.state == antfly.graph.GraphIndex.GraphMetricState.building and rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected HITS rerank building generation {d}, got {d}\n", .{ rebuild_generation, rerank_status.building_generation }); + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:hub-c")) { + std.debug.print("active HITS search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected HITS reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected HITS reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 3, + .include_stored = false, + })); +} + +fn verifyPageRankActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", initial_generation); + + const rebuild_generation = try addPageRankDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const started = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-active-read-proof-coordinator", + "5000", + "55000", + ); + if (!started.durable_progressed or !started.stats.has_lease) { + std.debug.print("expected coordinator process to start active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 1) { + std.debug.print("expected one PageRank graph metric result during active rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const result = published_result.graph_metric_results[0]; + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected active PageRank query status building, got {}\n", .{result.status.state}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (result.status.published_generation != initial_generation or result.status.building_generation != rebuild_generation) { + std.debug.print( + "expected published generation {d} and building generation {d}, got {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + result.status.published_generation, + result.status.building_generation, + }, + ); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected active PageRank published read to serve prior scores\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:e")) { + std.debug.print("active PageRank published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected one PageRank graph traversal result during active rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!std.mem.eql(u8, traversal.nodes[0].key, "doc:b")) { + std.debug.print("expected traversal to return doc:b, got {s}\n", .{traversal.nodes[0].key}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.nodes[0].metrics.len != 1 or traversal.nodes[0].metrics[0].score == null) { + std.debug.print("expected traversal published metric projection to serve prior PageRank score\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.metric_status.len != 1 or traversal.metric_status[0].state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected traversal metric status building during active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.metric_status[0].published_generation != initial_generation or traversal.metric_status[0].building_generation != rebuild_generation) { + std.debug.print("expected traversal status to report published/building generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "pagerank", + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected search rerank status building, got {}\n", .{rerank_status.state}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation or rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected rerank status to report published/building generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:e")) { + std.debug.print("active PageRank search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricPageRankProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricPageRankProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +fn verifyPageRankSameWorkerReplacementAttemptFence( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", target_generation, .scan_edges_and_out_degree); + + const worker_id = "process-pagerank-same-worker"; + const ready_file = ".zig-cache/tmp/graph-metric-process-pagerank-same-worker-ready"; + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillMetricPageOwnerAfterReady( + io, + harness_exe, + db_path, + "pagerank", + "scan_edges_and_out_degree", + worker_id, + "60000", + ready_file, + ); + + const dead_page = try readSingleLeasedMetricPage( + alloc, + db_path, + "pagerank", + .scan_edges_and_out_degree, + worker_id, + "process-dead-cursor", + ); + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const reclaim_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-same-worker-proof-replacement-owner", + worker_id, + "5000", + after_expiry_text, + ); + if (reclaim_worker.result.pages_claimed == 0 or reclaim_worker.result.pages_completed == 0) { + std.debug.print("expected same worker id replacement process to reclaim and complete expired PageRank page lease\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try expectStaleMetricPageAttemptRejected(alloc, db_path, "pagerank", .scan_edges_and_out_degree, dead_page, worker_id); + + _ = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "pagerank-same-worker-proof-coordinator", + "5000", + after_expiry_text, + ); + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, "pagerank", target_generation); +} + +fn verifyPageRankScanPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyPageRankPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .scan_edges_and_out_degree, + "scan_edges_and_out_degree", + "scan", + ".zig-cache/tmp/graph-metric-process-pagerank-scan-ready", + "process-pagerank-scan-dead-worker", + "process-pagerank-scan-reclaim-worker", + "pagerank-scan-proof-early-owner", + "pagerank-scan-proof-reclaim-owner", + "pagerank-scan-proof-coordinator", + ); +} + +fn verifyPageRankInitializePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyPageRankPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .initialize_ranks, + "initialize_ranks", + "initialize", + ".zig-cache/tmp/graph-metric-process-pagerank-initialize-ready", + "process-pagerank-initialize-dead-worker", + "process-pagerank-initialize-reclaim-worker", + "pagerank-initialize-proof-early-owner", + "pagerank-initialize-proof-reclaim-owner", + "pagerank-initialize-proof-coordinator", + ); +} + +fn verifyPageRankContributionPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyPageRankPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .iterate_contributions, + "iterate_contributions", + "contribution", + ".zig-cache/tmp/graph-metric-process-pagerank-contribution-ready", + "process-pagerank-contribution-dead-worker", + "process-pagerank-contribution-reclaim-worker", + "pagerank-contribution-proof-early-owner", + "pagerank-contribution-proof-reclaim-owner", + "pagerank-contribution-proof-coordinator", + ); +} + +fn verifyPageRankReducePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyPageRankPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .reduce_ranks, + "reduce_ranks", + "reduce", + ".zig-cache/tmp/graph-metric-process-pagerank-reduce-ready", + "process-pagerank-reduce-dead-worker", + "process-pagerank-reduce-reclaim-worker", + "pagerank-reduce-proof-early-owner", + "pagerank-reduce-proof-reclaim-owner", + "pagerank-reduce-proof-coordinator", + ); +} + +fn verifyPageRankConvergencePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyPageRankPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .check_convergence, + "check_convergence", + "convergence", + ".zig-cache/tmp/graph-metric-process-pagerank-convergence-ready", + "process-pagerank-convergence-dead-worker", + "process-pagerank-convergence-reclaim-worker", + "pagerank-convergence-proof-early-owner", + "pagerank-convergence-proof-reclaim-owner", + "pagerank-convergence-proof-coordinator", + ); +} + +fn verifyPageRankPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + phase_arg: []const u8, + phase_label: []const u8, + ready_file: []const u8, + dead_worker_id: []const u8, + reclaim_worker_id: []const u8, + early_owner_id: []const u8, + reclaim_owner_id: []const u8, + coordinator_owner_id: []const u8, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "pagerank", + target_generation, + phase, + 0, + phase_arg, + phase_label, + ready_file, + dead_worker_id, + reclaim_worker_id, + early_owner_id, + reclaim_owner_id, + coordinator_owner_id, + ); +} + +fn verifyEigenvectorScanPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyEigenvectorPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .scan_edges_and_out_degree, + "scan_edges_and_out_degree", + "scan", + ".zig-cache/tmp/graph-metric-process-eigenvector-scan-ready", + "process-eigenvector-scan-dead-worker", + "process-eigenvector-scan-reclaim-worker", + "eigenvector-scan-proof-early-owner", + "eigenvector-scan-proof-reclaim-owner", + "eigenvector-scan-proof-coordinator", + ); +} + +fn verifyEigenvectorInitializePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyEigenvectorPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .initialize_ranks, + "initialize_ranks", + "initialize", + ".zig-cache/tmp/graph-metric-process-eigenvector-initialize-ready", + "process-eigenvector-initialize-dead-worker", + "process-eigenvector-initialize-reclaim-worker", + "eigenvector-initialize-proof-early-owner", + "eigenvector-initialize-proof-reclaim-owner", + "eigenvector-initialize-proof-coordinator", + ); +} + +fn verifyEigenvectorContributionPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyEigenvectorPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .iterate_contributions, + "iterate_contributions", + "contribution", + ".zig-cache/tmp/graph-metric-process-eigenvector-contribution-ready", + "process-eigenvector-contribution-dead-worker", + "process-eigenvector-contribution-reclaim-worker", + "eigenvector-contribution-proof-early-owner", + "eigenvector-contribution-proof-reclaim-owner", + "eigenvector-contribution-proof-coordinator", + ); +} + +fn verifyEigenvectorReducePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyEigenvectorPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .reduce_ranks, + "reduce_ranks", + "reduce", + ".zig-cache/tmp/graph-metric-process-eigenvector-reduce-ready", + "process-eigenvector-reduce-dead-worker", + "process-eigenvector-reduce-reclaim-worker", + "eigenvector-reduce-proof-early-owner", + "eigenvector-reduce-proof-reclaim-owner", + "eigenvector-reduce-proof-coordinator", + ); +} + +fn verifyEigenvectorConvergencePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyEigenvectorPageLeaseReclaim( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + target_generation, + .check_convergence, + "check_convergence", + "convergence", + ".zig-cache/tmp/graph-metric-process-eigenvector-convergence-ready", + "process-eigenvector-convergence-dead-worker", + "process-eigenvector-convergence-reclaim-worker", + "eigenvector-convergence-proof-early-owner", + "eigenvector-convergence-proof-reclaim-owner", + "eigenvector-convergence-proof-coordinator", + ); +} + +fn verifyEigenvectorPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + phase_arg: []const u8, + phase_label: []const u8, + ready_file: []const u8, + dead_worker_id: []const u8, + reclaim_worker_id: []const u8, + early_owner_id: []const u8, + reclaim_owner_id: []const u8, + coordinator_owner_id: []const u8, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "eigenvector", + target_generation, + phase, + 0, + phase_arg, + phase_label, + ready_file, + dead_worker_id, + reclaim_worker_id, + early_owner_id, + reclaim_owner_id, + coordinator_owner_id, + ); +} + +fn verifyHitsAuthorityContributionPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "hits_authority", + target_generation, + .iterate_contributions, + 0, + "iterate_contributions", + "HITS authority contribution", + ".zig-cache/tmp/graph-metric-process-hits-authority-contribution-ready", + "process-hits-authority-contribution-dead-worker", + "process-hits-authority-contribution-reclaim-worker", + "hits-authority-contribution-proof-early-owner", + "hits-authority-contribution-proof-reclaim-owner", + "hits-authority-contribution-proof-coordinator", + ); + try verifyHitsFresh(alloc, db_path, target_generation); +} + +fn verifyHitsAuthorityReducePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "hits_authority", + target_generation, + .reduce_ranks, + 0, + "reduce_ranks", + "HITS authority reduce", + ".zig-cache/tmp/graph-metric-process-hits-authority-reduce-ready", + "process-hits-authority-reduce-dead-worker", + "process-hits-authority-reduce-reclaim-worker", + "hits-authority-reduce-proof-early-owner", + "hits-authority-reduce-proof-reclaim-owner", + "hits-authority-reduce-proof-coordinator", + ); + try verifyHitsFresh(alloc, db_path, target_generation); +} + +fn verifyHitsConvergencePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "hits_authority", + target_generation, + .check_convergence, + 0, + "check_convergence", + "HITS convergence", + ".zig-cache/tmp/graph-metric-process-hits-convergence-ready", + "process-hits-convergence-dead-worker", + "process-hits-convergence-reclaim-worker", + "hits-convergence-proof-early-owner", + "hits-convergence-proof-reclaim-owner", + "hits-convergence-proof-coordinator", + ); + try verifyHitsFresh(alloc, db_path, target_generation); +} + +fn verifyHitsHubContributionPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "hits_authority", + target_generation, + .hits_hub_contributions, + 0, + "hits_hub_contributions", + "HITS hub contribution", + ".zig-cache/tmp/graph-metric-process-hits-hub-contribution-ready", + "process-hits-hub-contribution-dead-worker", + "process-hits-hub-contribution-reclaim-worker", + "hits-hub-contribution-proof-early-owner", + "hits-hub-contribution-proof-reclaim-owner", + "hits-hub-contribution-proof-coordinator", + ); + try verifyHitsFresh(alloc, db_path, target_generation); +} + +fn verifyHitsHubReducePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "hits_authority", + target_generation, + .hits_hub_reduce_ranks, + 0, + "hits_hub_reduce_ranks", + "HITS hub reduce", + ".zig-cache/tmp/graph-metric-process-hits-hub-reduce-ready", + "process-hits-hub-reduce-dead-worker", + "process-hits-hub-reduce-reclaim-worker", + "hits-hub-reduce-proof-early-owner", + "hits-hub-reduce-proof-reclaim-owner", + "hits-hub-reduce-proof-coordinator", + ); + try verifyHitsFresh(alloc, db_path, target_generation); +} + +fn verifyPageRankLaterContributionPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "pagerank", + target_generation, + .iterate_contributions, + 1, + "iterate_contributions", + "later contribution", + ".zig-cache/tmp/graph-metric-process-pagerank-later-contribution-ready", + "process-pagerank-later-contribution-dead-worker", + "process-pagerank-later-contribution-reclaim-worker", + "pagerank-later-contribution-proof-early-owner", + "pagerank-later-contribution-proof-reclaim-owner", + "pagerank-later-contribution-proof-coordinator", + ); +} + +fn verifyPageRankLaterReducePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "pagerank", + target_generation, + .reduce_ranks, + 1, + "reduce_ranks", + "later reduce", + ".zig-cache/tmp/graph-metric-process-pagerank-later-reduce-ready", + "process-pagerank-later-reduce-dead-worker", + "process-pagerank-later-reduce-reclaim-worker", + "pagerank-later-reduce-proof-early-owner", + "pagerank-later-reduce-proof-reclaim-owner", + "pagerank-later-reduce-proof-coordinator", + ); +} + +fn verifyPageRankLaterConvergencePageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try verifyMetricPageLeaseReclaimAtIteration( + alloc, + io, + harness_exe, + antfly_exe, + db_path, + "pagerank", + target_generation, + .check_convergence, + 1, + "check_convergence", + "later convergence", + ".zig-cache/tmp/graph-metric-process-pagerank-later-convergence-ready", + "process-pagerank-later-convergence-dead-worker", + "process-pagerank-later-convergence-reclaim-worker", + "pagerank-later-convergence-proof-early-owner", + "pagerank-later-convergence-proof-reclaim-owner", + "pagerank-later-convergence-proof-coordinator", + ); +} + +fn verifyMetricPageLeaseReclaimAtIteration( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + iteration: u32, + phase_arg: []const u8, + phase_label: []const u8, + ready_file: []const u8, + dead_worker_id: []const u8, + reclaim_worker_id: []const u8, + early_owner_id: []const u8, + reclaim_owner_id: []const u8, + coordinator_owner_id: []const u8, +) !void { + try prepareMetricBuildToPhaseAndIteration( + alloc, + db_path, + metric_name, + target_generation, + phase, + iteration, + ); + + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillMetricPageOwnerAfterReady( + io, + harness_exe, + db_path, + metric_name, + phase_arg, + dead_worker_id, + "10000", + ready_file, + ); + + const dead_page = try readSingleLeasedMetricPage( + alloc, + db_path, + metric_name, + phase, + dead_worker_id, + "process-dead-cursor", + ); + const before_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms - 1}); + defer alloc.free(before_expiry_text); + const early_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + early_owner_id, + reclaim_worker_id, + "5000", + before_expiry_text, + ); + if (early_worker.result.pages_claimed != 0 or early_worker.result.pages_completed != 0) { + std.debug.print("expected {s} replacement worker to be fenced before {s} page lease expiry\n", .{ metric_name, phase_label }); + return error.GraphMetricProcessProofFailed; + } + _ = try readSingleLeasedMetricPage( + alloc, + db_path, + metric_name, + phase, + dead_worker_id, + "process-dead-cursor", + ); + + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const reclaim_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + reclaim_owner_id, + reclaim_worker_id, + "5000", + after_expiry_text, + ); + if (reclaim_worker.result.pages_claimed == 0 or reclaim_worker.result.pages_completed == 0) { + std.debug.print("expected {s} replacement worker to reclaim and complete expired {s} page lease\n", .{ metric_name, phase_label }); + return error.GraphMetricProcessProofFailed; + } + try expectReclaimedMetricPageCompleted( + alloc, + db_path, + metric_name, + phase, + dead_page, + reclaim_worker_id, + ); + try expectStaleMetricPageAttemptRejected(alloc, db_path, metric_name, phase, dead_page, dead_worker_id); + + _ = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + coordinator_owner_id, + "5000", + after_expiry_text, + ); + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyMetricFresh(alloc, db_path, metric_name, target_generation); +} + +fn verifyWorkerPageLeaseReclaim( + alloc: std.mem.Allocator, + io: std.Io, + harness_exe: []const u8, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareDegreeScanBuild(alloc, db_path, target_generation); + + const ready_file = ".zig-cache/tmp/graph-metric-process-worker-page-ready"; + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillDegreePageOwnerAfterReady( + io, + harness_exe, + db_path, + "process-dead-worker", + "10000", + ready_file, + ); + + const dead_page = try readSingleLeasedDegreePage(alloc, db_path, "process-dead-worker", "process-dead-cursor"); + const before_expiry_ms = dead_page.lease_expires_at_ms - 1; + const before_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{before_expiry_ms}); + defer alloc.free(before_expiry_text); + const early_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "worker-page-proof-early-owner", + "process-reclaim-worker", + "5000", + before_expiry_text, + ); + if (early_worker.result.pages_claimed != 0 or early_worker.result.pages_completed != 0) { + std.debug.print("expected replacement worker to be fenced before page lease expiry\n", .{}); + return error.GraphMetricWorkerPageProofFailed; + } + _ = try readSingleLeasedDegreePage(alloc, db_path, "process-dead-worker", "process-dead-cursor"); + + const after_expiry_text = try std.fmt.allocPrint(alloc, "{d}", .{dead_page.lease_expires_at_ms + 1}); + defer alloc.free(after_expiry_text); + const reclaim_worker = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "worker-page-proof-reclaim-owner", + "process-reclaim-worker", + "5000", + after_expiry_text, + ); + if (reclaim_worker.result.pages_claimed == 0 or reclaim_worker.result.pages_completed == 0) { + std.debug.print("expected replacement worker to reclaim and complete expired page lease\n", .{}); + return error.GraphMetricWorkerPageProofFailed; + } + try expectReclaimedMetricPageCompleted( + alloc, + db_path, + "degree", + .scan_edges_and_out_degree, + dead_page, + "process-reclaim-worker", + ); + try expectStaleDegreePageAttemptRejected(alloc, db_path, dead_page); + + _ = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "worker-page-proof-coordinator", + "5000", + after_expiry_text, + ); + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyDegreeFresh(alloc, db_path, target_generation); +} + +fn verifyWorkerRuntimeSameWorkerLeaseFencing( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareDegreeScanBuild(alloc, db_path, target_generation); + + const ready_file = ".zig-cache/tmp/graph-metric-process-worker-runtime-ready"; + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillWorkerRoleAfterReady( + io, + antfly_exe, + db_path, + "worker-runtime-proof-owner-a", + "worker-runtime-proof-worker", + "5000", + "10000", + ready_file, + ); + + const duplicate = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "worker-runtime-proof-owner-b", + "worker-runtime-proof-worker", + "5000", + "14999", + ); + if (duplicate.durable_progressed or duplicate.stats.has_lease or duplicate.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate same-worker runtime owner to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const replacement = try runWorkerRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "worker-runtime-proof-owner-b", + "worker-runtime-proof-worker", + "5000", + "15001", + ); + if (!replacement.stats.has_lease or replacement.stats.acquisition_count == 0 or replacement.stats.takeover_count == 0) { + std.debug.print("expected same-worker replacement runtime owner to acquire expired worker lease\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + if (!replacement.durable_progressed or replacement.result.pages_claimed == 0 or replacement.result.pages_completed == 0) { + std.debug.print("expected same-worker replacement runtime owner to advance durable page work after takeover\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyDegreeFresh(alloc, db_path, target_generation); +} + +fn runSupervisorProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, +) !void { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "supervise", + "--db-path", + db_path, + "--executable", + antfly_exe, + "--coordinator-owner-id", + "process-degree-coordinator", + "--worker-pool-owner-id", + "process-degree-worker-pool", + "--worker-ids", + "process-worker-a,process-worker-b", + "--ticks", + "8", + "--max-idle-ticks", + "2", + "--supervisor-rounds", + "80", + "--supervisor-idle-rounds", + "1", + "--tick-ms", + "0", + "--max-restarts", + "0", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + }; + const result = try std.process.run(alloc, io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .reserve_amount = 512, + }); + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + switch (result.term) { + .exited => |code| if (code != 0) { + std.debug.print( + "graph metric supervisor exited with code {d}\nstdout:\n{s}\nstderr:\n{s}\n", + .{ code, result.stdout, result.stderr }, + ); + return error.SupervisorProcessFailed; + }, + else => { + std.debug.print( + "graph metric supervisor terminated unexpectedly\nstdout:\n{s}\nstderr:\n{s}\n", + .{ result.stdout, result.stderr }, + ); + return error.SupervisorProcessFailed; + }, + } + + try verifyProcessSummaryJsonNoRawOperationalFields(alloc, result.stdout); + var parsed = try std.json.parseFromSlice(SupervisorSummary, alloc, result.stdout, .{ + .ignore_unknown_fields = true, + .allocate = .alloc_always, + }); + defer parsed.deinit(); + if (!parsed.value.succeeded or !std.mem.eql(u8, parsed.value.exit_reason, "idle")) { + std.debug.print("unexpected supervisor summary:\n{s}\n", .{result.stdout}); + return error.SupervisorProcessFailed; + } + if (parsed.value.rounds_executed == 0) return error.SupervisorProcessFailed; + try verifyProcessSupervisorTelemetry( + parsed.value, + "process-degree-coordinator", + "process-degree-worker-pool", + ); +} + +fn runLaunchProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + label: []const u8, +) !void { + const summary_dir = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/graph-metric-process-launch-{s}-summaries", .{label}); + defer alloc.free(summary_dir); + const coordinator_owner_id = try std.fmt.allocPrint(alloc, "process-launch-{s}-coordinator", .{label}); + defer alloc.free(coordinator_owner_id); + const worker_pool_owner_id = try std.fmt.allocPrint(alloc, "process-launch-{s}-worker-pool", .{label}); + defer alloc.free(worker_pool_owner_id); + const worker_ids = try std.fmt.allocPrint(alloc, "process-launch-{s}-worker-a,process-launch-{s}-worker-b", .{ label, label }); + defer alloc.free(worker_ids); + + std.Io.Dir.cwd().deleteTree(io, summary_dir) catch {}; + defer std.Io.Dir.cwd().deleteTree(io, summary_dir) catch {}; + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "launch", + "--db-path", + db_path, + "--executable", + antfly_exe, + "--coordinator-owner-id", + coordinator_owner_id, + "--worker-pool-owner-id", + worker_pool_owner_id, + "--worker-ids", + worker_ids, + "--ticks", + "16", + "--max-idle-ticks", + "4", + "--supervisor-rounds", + "20", + "--supervisor-idle-rounds", + "1", + "--tick-ms", + "0", + "--max-restarts", + "0", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + "--summary-dir", + summary_dir, + }; + const result = try std.process.run(alloc, io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .reserve_amount = 512, + }); + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + switch (result.term) { + .exited => |code| if (code != 0) { + std.debug.print( + "graph metric launcher exited with code {d}\nstdout:\n{s}\nstderr:\n{s}\n", + .{ code, result.stdout, result.stderr }, + ); + return error.SupervisorProcessFailed; + }, + else => { + std.debug.print( + "graph metric launcher terminated unexpectedly\nstdout:\n{s}\nstderr:\n{s}\n", + .{ result.stdout, result.stderr }, + ); + return error.SupervisorProcessFailed; + }, + } + + try verifyProcessSummaryJsonNoRawOperationalFields(alloc, result.stdout); + var parsed = try std.json.parseFromSlice(SupervisorSummary, alloc, result.stdout, .{ + .ignore_unknown_fields = true, + .allocate = .alloc_always, + }); + defer parsed.deinit(); + if (!parsed.value.succeeded or !std.mem.eql(u8, parsed.value.exit_reason, "idle")) { + std.debug.print("unexpected launcher summary:\n{s}\nstderr:\n{s}\n", .{ result.stdout, result.stderr }); + return error.SupervisorProcessFailed; + } + if (parsed.value.rounds_executed == 0) return error.SupervisorProcessFailed; + try verifyProcessSupervisorTelemetry(parsed.value, coordinator_owner_id, worker_pool_owner_id); +} + +fn verifyProcessSummaryJsonNoRawOperationalFields(alloc: std.mem.Allocator, stdout: []const u8) !void { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, stdout, .{}); + defer parsed.deinit(); + try verifyJsonNoRawOperationalFields(parsed.value, error.SupervisorProcessFailed); +} + +fn verifyProcessSupervisorTelemetry( + summary: SupervisorSummary, + coordinator_owner_id: []const u8, + worker_pool_owner_id: []const u8, +) !void { + const coordinator = summary.coordinator.telemetry orelse return error.SupervisorProcessFailed; + try verifyChildTelemetry( + coordinator, + .coordinator, + coordinator_owner_id, + 0, + false, + ); + + const worker_pool = summary.worker_pool.telemetry orelse return error.SupervisorProcessFailed; + try verifyChildTelemetry( + worker_pool, + .worker_pool, + worker_pool_owner_id, + 2, + true, + ); +} + +fn verifyChildTelemetry( + telemetry: ChildRuntimeTelemetry, + role: RuntimeRole, + logical_owner_id: []const u8, + worker_count: usize, + expect_worker_hash: bool, +) !void { + if (telemetry.role != role) return error.SupervisorProcessFailed; + const logical_owner_hash = std.hash.Wyhash.hash(0, logical_owner_id); + if (telemetry.runtime_id_hash != logical_owner_hash) return error.SupervisorProcessFailed; + // The command appends a process-incarnation fence to the logical owner ID. + // The runtime identity remains stable for observability, while the lease + // owner must be non-zero and distinct across process restarts. + if (telemetry.owner_id_hash == 0 or telemetry.owner_id_hash == logical_owner_hash) return error.SupervisorProcessFailed; + if (telemetry.lease_key_hash == 0) return error.SupervisorProcessFailed; + if (expect_worker_hash) { + if (telemetry.worker_id_hash == 0) return error.SupervisorProcessFailed; + } else if (telemetry.worker_id_hash != 0) { + return error.SupervisorProcessFailed; + } + if (telemetry.worker_count != worker_count) return error.SupervisorProcessFailed; + if (!telemetry.lease_owned) return error.SupervisorProcessFailed; + if (!telemetry.has_lease) return error.SupervisorProcessFailed; + if (telemetry.acquisition_count == 0) return error.SupervisorProcessFailed; + if (telemetry.ticks_started == 0) return error.SupervisorProcessFailed; + if (telemetry.ticks_completed == 0) return error.SupervisorProcessFailed; + if (telemetry.error_ticks != 0) return error.SupervisorProcessFailed; + if (telemetry.has_last_error) return error.SupervisorProcessFailed; +} + +fn verifyCoordinatorLeaseExpiryTakeover( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, +) !void { + const ready_file = ".zig-cache/tmp/graph-metric-process-lease-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + try runAndKillCoordinatorAfterReady( + io, + antfly_exe, + db_path, + "lease-proof-coordinator-a", + "5000", + ready_file, + ); + defer std.Io.Dir.cwd().deleteFile(io, ready_file) catch {}; + + const worker_pool = try runWorkerPoolRoleProcess( + alloc, + io, + antfly_exe, + db_path, + "lease-proof-worker-pool", + "5000", + ); + if (!worker_pool.durable_progressed or !worker_pool.stats.has_lease) { + std.debug.print("expected worker pool to acquire independent lease and complete work\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_b_blocked = try runCoordinatorRoleProcess( + alloc, + io, + antfly_exe, + db_path, + "lease-proof-coordinator-b", + "5000", + ); + if (coordinator_b_blocked.durable_progressed or coordinator_b_blocked.stats.has_lease or coordinator_b_blocked.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate coordinator to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + platform.time.sleepNs(5100 * std.time.ns_per_ms); + + const coordinator_b_takeover = try runCoordinatorRoleProcess( + alloc, + io, + antfly_exe, + db_path, + "lease-proof-coordinator-b", + "5000", + ); + if (!coordinator_b_takeover.stats.has_lease or coordinator_b_takeover.stats.acquisition_count == 0 or coordinator_b_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement coordinator to acquire expired lease\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + if (!coordinator_b_takeover.durable_progressed) { + std.debug.print("expected replacement coordinator to advance durable work after takeover\n", .{}); + return error.GraphMetricLeaseProofFailed; + } +} + +fn verifyServiceTargetedMetricOwnerRestartProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const coordinator_ready_file = ".zig-cache/tmp/graph-metric-process-service-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + try runAndKillServiceCoordinatorAfterReady( + io, + antfly_exe, + base_uri, + "service-process-coordinator", + "service-process-coordinator-a", + "200", + "1000", + coordinator_ready_file, + ); + + const coordinator_b_fenced = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-coordinator", + "service-process-coordinator-b", + "200", + "1100", + ); + if (coordinator_b_fenced.durable_progressed or coordinator_b_fenced.stats.has_lease or coordinator_b_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service coordinator process to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_b_takeover = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-coordinator", + "service-process-coordinator-b", + "200", + "1301", + ); + if (coordinator_b_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement service coordinator process to acquire expired lease\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const worker_ready_file = ".zig-cache/tmp/graph-metric-process-service-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReady( + io, + antfly_exe, + base_uri, + "service-process-worker-pool", + "service-process-worker-pool-a", + "service-process-worker-a,service-process-worker-b", + "200", + "2000", + worker_ready_file, + ); + + const worker_pool_b_fenced = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-worker-pool", + "service-process-worker-pool-b", + "service-process-worker-a,service-process-worker-b", + "200", + "2100", + ); + if (worker_pool_b_fenced.durable_progressed or worker_pool_b_fenced.stats.has_lease or worker_pool_b_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service worker-pool process to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const worker_pool_b_takeover = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-worker-pool", + "service-process-worker-pool-b", + "service-process-worker-a,service-process-worker-b", + "200", + "2301", + ); + if (worker_pool_b_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement service worker-pool process to acquire expired lease\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + var now_ms: u64 = 2400; + var idle_rounds: usize = 0; + for (0..80) |_| { + const now_coordinator = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_coordinator); + _ = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-coordinator", + "service-process-coordinator-b", + "200", + now_coordinator, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + const worker_summary = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-process-worker-pool", + "service-process-worker-pool-b", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + now_ms += 1; + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + if (status.published_generation != target_generation) return error.GraphMetricGenerationMismatch; + return; + } + if (worker_summary.durable_progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= 8) break; + } + } + + return error.GraphMetricBuildNotComplete; +} + +fn verifyDegreeServiceTargetedPublishAndCleanupRestartProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareDegreePublishReadyBuild(alloc, db_path, target_generation); + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-publish-cleanup-coordinator", + "service-degree-publish-cleanup-coordinator-a", + "5000", + "75000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish degree before cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "degree", .cleanup_old_generations, target_generation); + + const duplicate_publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-publish-cleanup-coordinator", + "service-degree-publish-cleanup-coordinator-b", + "5000", + "75001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "service degree cleanup"); + try assertOpenDbMetricPhase(alloc, &db, "degree", .cleanup_old_generations, target_generation); + + const cleanup_ready_file = ".zig-cache/tmp/graph-metric-process-service-degree-publish-cleanup-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io, + antfly_exe, + base_uri, + "service-degree-publish-cleanup-worker-pool", + "service-degree-publish-cleanup-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + "75002", + "1", + cleanup_ready_file, + ); + try assertOpenDbMetricPhase(alloc, &db, "degree", .cleanup_old_generations, target_generation); + + const fenced_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-publish-cleanup-worker-pool", + "service-degree-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "75100", + "1", + ); + if (fenced_cleanup.durable_progressed or fenced_cleanup.stats.has_lease or fenced_cleanup.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service degree cleanup worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "degree", .cleanup_old_generations, target_generation); + + const takeover_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-publish-cleanup-worker-pool", + "service-degree-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "75203", + "1", + ); + if (takeover_cleanup.stats.takeover_count == 0 or takeover_cleanup.result.pages_claimed != 1 or takeover_cleanup.result.pages_completed != 1) { + std.debug.print("expected replacement service worker-pool process to finish degree cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyDegreeFresh(alloc, db_path, target_generation); +} + +fn verifyDegreeServiceTargetedMultiPageWorkerPoolProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const coordinator_runtime = "service-degree-multipage-coordinator"; + const worker_pool_runtime = "service-degree-multipage-worker-pool"; + const coordinator_ready_file = ".zig-cache/tmp/graph-metric-process-service-degree-multipage-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + try runAndKillServiceCoordinatorAfterReady( + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-a", + "200", + "81000", + coordinator_ready_file, + ); + try assertOpenDbMetricActivePhase(alloc, &db, "degree", .prepare_generation, target_generation); + + const coordinator_fenced = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-b", + "200", + "81100", + ); + if (coordinator_fenced.durable_progressed or coordinator_fenced.stats.has_lease or coordinator_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service degree coordinator to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_takeover = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-b", + "200", + "81201", + ); + if (coordinator_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service degree coordinator to acquire expired lease\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricActivePhase(alloc, &db, "degree", .prepare_generation, target_generation); + + const prepare = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-degree-multipage-worker-pool-a", + "service-process-worker-a,service-process-worker-b", + "5000", + "81202", + "4", + ); + if (prepare.result.pages_completed != 1 or prepare.stats.worker_count != 2) { + std.debug.print("expected service worker-pool to complete the single degree prepare page with two configured workers\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + const to_scan = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-b", + "5000", + "81203", + ); + if (to_scan.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to advance degree build to scan phase\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricActivePhase(alloc, &db, "degree", .scan_edges_and_out_degree, target_generation); + + const worker_ready_file = ".zig-cache/tmp/graph-metric-process-service-degree-multipage-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReady( + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-degree-multipage-worker-pool-b", + "service-process-worker-a,service-process-worker-b", + "200", + "81204", + worker_ready_file, + ); + + const worker_pool_fenced = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-degree-multipage-worker-pool-c", + "service-process-worker-a,service-process-worker-b", + "200", + "81300", + "4", + ); + if (worker_pool_fenced.durable_progressed or worker_pool_fenced.stats.has_lease or worker_pool_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service degree worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const scan = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-degree-multipage-worker-pool-c", + "service-process-worker-a,service-process-worker-b", + "200", + "81405", + "4", + ); + if (scan.stats.takeover_count == 0 or scan.result.pages_completed == 0 or scan.stats.worker_count != 2) { + std.debug.print("expected replacement service worker-pool to take over and complete remaining degree scan pages, got {d}\n", .{scan.result.pages_completed}); + return error.GraphMetricProcessProofFailed; + } + + const to_reduce = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-c", + "5000", + "81406", + ); + if (to_reduce.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to advance degree build to reduce phase\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricActivePhase(alloc, &db, "degree", .reduce_ranks, target_generation); + + const reduce = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-degree-multipage-worker-pool-c", + "service-process-worker-a,service-process-worker-b", + "5000", + "81407", + "4", + ); + if (reduce.result.pages_completed < 2 or reduce.stats.worker_count != 2) { + std.debug.print("expected service worker-pool to complete multiple degree reduce pages, got {d}\n", .{reduce.result.pages_completed}); + return error.GraphMetricProcessProofFailed; + } + + const to_publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-d", + "5000", + "81408", + ); + if (to_publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to advance degree build to publish phase\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricActivePhase(alloc, &db, "degree", .publish_generation, target_generation); + + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-degree-multipage-coordinator-e", + "5000", + "81409", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish multi-page degree build\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "degree", .cleanup_old_generations, target_generation); + + var cleanup_now_ms: u64 = 81410; + var cleanup_progressed = false; + var cleanup_fresh = false; + for (0..8) |i| { + var status = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + cleanup_fresh = true; + break; + } + + const owner_id = try std.fmt.allocPrint(alloc, "service-degree-multipage-worker-pool-cleanup-{d}", .{i}); + defer alloc.free(owner_id); + const cleanup_now = try std.fmt.allocPrint(alloc, "{d}", .{cleanup_now_ms}); + defer alloc.free(cleanup_now); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + cleanup_now, + "4", + ); + if (cleanup.stats.worker_count != 2) { + std.debug.print("expected service cleanup worker-pool to keep two configured workers\n", .{}); + return error.GraphMetricProcessProofFailed; + } + cleanup_progressed = cleanup_progressed or cleanup.durable_progressed; + cleanup_now_ms += 1; + } + if (!cleanup_fresh and !cleanup_progressed) { + std.debug.print("expected service worker-pool to make cleanup progress for multi-page degree build\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyDegreeFresh(alloc, db_path, target_generation); +} + +fn verifyPageRankServiceTargetedPublishAndCleanupRestartProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "pagerank", target_generation, .publish_generation); + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-coordinator", + "service-pagerank-publish-cleanup-coordinator-a", + "5000", + "76000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish PageRank before cleanup\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "pagerank", .cleanup_old_generations, target_generation); + + const duplicate_publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-coordinator", + "service-pagerank-publish-cleanup-coordinator-b", + "5000", + "76001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "service pagerank cleanup"); + try assertOpenDbMetricPhase(alloc, &db, "pagerank", .cleanup_old_generations, target_generation); + + const cleanup_ready_file = ".zig-cache/tmp/graph-metric-process-service-pagerank-publish-cleanup-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-worker-pool", + "service-pagerank-publish-cleanup-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + "76002", + "1", + cleanup_ready_file, + ); + try assertOpenDbMetricPhase(alloc, &db, "pagerank", .cleanup_old_generations, target_generation); + + const fenced_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-worker-pool", + "service-pagerank-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "76100", + "4", + ); + if (fenced_cleanup.durable_progressed or fenced_cleanup.stats.has_lease or fenced_cleanup.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service PageRank cleanup worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "pagerank", .cleanup_old_generations, target_generation); + + const takeover_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-worker-pool", + "service-pagerank-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "76203", + "4", + ); + if (takeover_cleanup.stats.takeover_count == 0 or takeover_cleanup.result.pages_claimed == 0 or takeover_cleanup.result.pages_completed == 0) { + std.debug.print("expected replacement service PageRank cleanup worker-pool to take over and advance cleanup\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + + var cleanup_progressed = takeover_cleanup.durable_progressed; + for (0..6) |i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-pagerank-publish-cleanup-worker-pool-final-{d}", .{i}); + defer alloc.free(owner_id); + const now_ms = try std.fmt.allocPrint(alloc, "{d}", .{76204 + i}); + defer alloc.free(now_ms); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-publish-cleanup-worker-pool", + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + now_ms, + "4", + ); + cleanup_progressed = cleanup_progressed or cleanup.durable_progressed or cleanup.result.pages_claimed != 0 or cleanup.result.pages_completed != 0; + } + if (!cleanup_progressed) { + std.debug.print("expected service PageRank cleanup worker-pool processes to make cleanup progress\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + try verifyPageRankFixedIterationMetadata(alloc, db_path, target_generation, 1); +} + +fn verifyPageRankServiceTargetedMultiPageWorkerPoolProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const coordinator_runtime = "service-pagerank-multipage-coordinator"; + const worker_pool_runtime = "service-pagerank-multipage-worker-pool"; + var now_ms: u64 = 82000; + const coordinator_ready_file = ".zig-cache/tmp/graph-metric-process-service-pagerank-multipage-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + try runAndKillServiceCoordinatorAfterReady( + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-pagerank-multipage-coordinator-start", + "200", + "82000", + coordinator_ready_file, + ); + try assertOpenDbMetricActivePhase(alloc, &db, "pagerank", .prepare_generation, target_generation); + + const coordinator_fenced = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-pagerank-multipage-coordinator-replacement", + "200", + "82100", + ); + if (coordinator_fenced.durable_progressed or coordinator_fenced.stats.has_lease or coordinator_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service PageRank coordinator to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_takeover = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-pagerank-multipage-coordinator-replacement", + "200", + "82201", + ); + if (coordinator_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service PageRank coordinator to acquire expired lease\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + now_ms = 82202; + try assertOpenDbMetricActivePhase(alloc, &db, "pagerank", .prepare_generation, target_generation); + + const phases = [_]antfly.graph.GraphIndex.GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + }; + for (phases, 0..) |phase, i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-pagerank-multipage-worker-pool-{d}", .{i}); + defer alloc.free(owner_id); + const worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(worker_now); + const worker = if (phase == .scan_edges_and_out_degree) blk: { + const worker_ready_file = ".zig-cache/tmp/graph-metric-process-service-pagerank-multipage-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReady( + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-pagerank-multipage-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + worker_now, + worker_ready_file, + ); + now_ms += 1; + + const fenced_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(fenced_now); + const worker_pool_fenced = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-pagerank-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + fenced_now, + "4", + ); + now_ms += 1; + if (worker_pool_fenced.durable_progressed or worker_pool_fenced.stats.has_lease or worker_pool_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service PageRank worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const takeover_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms + 205}); + defer alloc.free(takeover_now); + const replacement = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-pagerank-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + takeover_now, + "4", + ); + now_ms += 206; + if (replacement.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service PageRank worker-pool to acquire expired lease\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + break :blk replacement; + } else blk: { + const summary = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + worker_now, + "4", + ); + now_ms += 1; + break :blk summary; + }; + const expected_min_pages: usize = if (phase == .prepare_generation or phase == .scan_edges_and_out_degree) 1 else 2; + if (worker.result.pages_completed < expected_min_pages or worker.stats.worker_count != 2) { + std.debug.print("expected service worker-pool to complete at least {d} PageRank pages for phase {}, got {d}\n", .{ + expected_min_pages, + phase, + worker.result.pages_completed, + }); + return error.GraphMetricPageRankProcessProofFailed; + } + + const coordinator_owner = try std.fmt.allocPrint(alloc, "service-pagerank-multipage-coordinator-{d}", .{i}); + defer alloc.free(coordinator_owner); + const coordinator_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(coordinator_now); + var coordinator = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + coordinator_owner, + "5000", + coordinator_now, + ); + now_ms += 1; + if (coordinator.result.phases_advanced == 0) { + coordinator = try drainServicePhaseRemainder(alloc, io, antfly_exe, base_uri, coordinator_runtime, worker_pool_runtime, &now_ms); + } + const next_phase = switch (phase) { + .prepare_generation => antfly.graph.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, + .scan_edges_and_out_degree => .initialize_ranks, + .initialize_ranks => .iterate_contributions, + .iterate_contributions => .reduce_ranks, + .reduce_ranks => .check_convergence, + .check_convergence => .publish_generation, + else => unreachable, + }; + try assertOpenDbMetricActivePhase(alloc, &db, "pagerank", next_phase, target_generation); + } + + const publish_worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_worker_now); + const publish_worker = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-pagerank-multipage-worker-pool-publish", + "service-process-worker-a,service-process-worker-b", + "5000", + publish_worker_now, + "4", + ); + now_ms += 1; + if (publish_worker.result.pages_completed == 0) { + std.debug.print("expected service worker-pool to materialize PageRank publish pages\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + + const publish_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_now); + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-pagerank-multipage-coordinator-publish", + "5000", + publish_now, + ); + now_ms += 1; + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish multi-page PageRank build\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "pagerank", .cleanup_old_generations, target_generation); + + var cleanup_fresh = false; + for (0..8) |i| { + var status = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + cleanup_fresh = true; + break; + } + + const owner_id = try std.fmt.allocPrint(alloc, "service-pagerank-multipage-worker-pool-cleanup-{d}", .{i}); + defer alloc.free(owner_id); + const cleanup_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(cleanup_now); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + cleanup_now, + "4", + ); + now_ms += 1; + if (cleanup.stats.worker_count != 2) { + std.debug.print("expected service PageRank cleanup worker-pool to keep two configured workers\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + if (!cleanup_fresh) { + var status = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + cleanup_fresh = status.state == antfly.graph.GraphIndex.GraphMetricState.fresh; + } + if (!cleanup_fresh) { + std.debug.print("expected service worker-pool to finish multi-page PageRank cleanup\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + try verifyPageRankFixedIterationMetadata(alloc, db_path, target_generation, 1); +} + +fn verifyEigenvectorServiceTargetedPublishAndCleanupRestartProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "eigenvector", target_generation, .publish_generation); + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-coordinator", + "service-eigenvector-publish-cleanup-coordinator-a", + "5000", + "77000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish eigenvector before cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "eigenvector", .cleanup_old_generations, target_generation); + + const duplicate_publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-coordinator", + "service-eigenvector-publish-cleanup-coordinator-b", + "5000", + "77001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "service eigenvector cleanup"); + try assertOpenDbMetricPhase(alloc, &db, "eigenvector", .cleanup_old_generations, target_generation); + + const cleanup_ready_file = ".zig-cache/tmp/graph-metric-process-service-eigenvector-publish-cleanup-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-worker-pool", + "service-eigenvector-publish-cleanup-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + "77002", + "1", + cleanup_ready_file, + ); + try assertOpenDbMetricPhase(alloc, &db, "eigenvector", .cleanup_old_generations, target_generation); + + const fenced_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-worker-pool", + "service-eigenvector-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "77100", + "4", + ); + if (fenced_cleanup.durable_progressed or fenced_cleanup.stats.has_lease or fenced_cleanup.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service eigenvector cleanup worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "eigenvector", .cleanup_old_generations, target_generation); + + const takeover_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-worker-pool", + "service-eigenvector-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "77203", + "4", + ); + if (takeover_cleanup.stats.takeover_count == 0 or takeover_cleanup.result.pages_claimed == 0 or takeover_cleanup.result.pages_completed == 0) { + std.debug.print("expected replacement service eigenvector cleanup worker-pool to take over and advance cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + var cleanup_progressed = takeover_cleanup.durable_progressed; + for (0..6) |i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-eigenvector-publish-cleanup-worker-pool-final-{d}", .{i}); + defer alloc.free(owner_id); + const now_ms = try std.fmt.allocPrint(alloc, "{d}", .{77204 + i}); + defer alloc.free(now_ms); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-publish-cleanup-worker-pool", + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + now_ms, + "4", + ); + cleanup_progressed = cleanup_progressed or cleanup.durable_progressed or cleanup.result.pages_claimed != 0 or cleanup.result.pages_completed != 0; + } + if (!cleanup_progressed) { + std.debug.print("expected service worker-pool processes to make eigenvector cleanup progress\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyMetricFresh(alloc, db_path, "eigenvector", target_generation); +} + +fn verifyEigenvectorServiceTargetedMultiPageWorkerPoolProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const coordinator_runtime = "service-eigenvector-multipage-coordinator"; + const worker_pool_runtime = "service-eigenvector-multipage-worker-pool"; + var now_ms: u64 = 83000; + const coordinator_ready_file = ".zig-cache/tmp/graph-metric-process-service-eigenvector-multipage-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + try runAndKillServiceCoordinatorAfterReady( + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-eigenvector-multipage-coordinator-start", + "200", + "83000", + coordinator_ready_file, + ); + try assertOpenDbMetricActivePhase(alloc, &db, "eigenvector", .prepare_generation, target_generation); + + const coordinator_fenced = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-eigenvector-multipage-coordinator-replacement", + "200", + "83100", + ); + if (coordinator_fenced.durable_progressed or coordinator_fenced.stats.has_lease or coordinator_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service eigenvector coordinator to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_takeover = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-eigenvector-multipage-coordinator-replacement", + "200", + "83201", + ); + if (coordinator_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service eigenvector coordinator to acquire expired lease\n", .{}); + return error.GraphMetricProcessProofFailed; + } + now_ms = 83202; + try assertOpenDbMetricActivePhase(alloc, &db, "eigenvector", .prepare_generation, target_generation); + + const phases = [_]antfly.graph.GraphIndex.GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + }; + for (phases, 0..) |phase, i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-eigenvector-multipage-worker-pool-{d}", .{i}); + defer alloc.free(owner_id); + const worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(worker_now); + const worker = if (phase == .scan_edges_and_out_degree) blk: { + const worker_ready_file = ".zig-cache/tmp/graph-metric-process-service-eigenvector-multipage-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReady( + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-eigenvector-multipage-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + worker_now, + worker_ready_file, + ); + now_ms += 1; + + const fenced_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(fenced_now); + const worker_pool_fenced = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-eigenvector-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + fenced_now, + "4", + ); + now_ms += 1; + if (worker_pool_fenced.durable_progressed or worker_pool_fenced.stats.has_lease or worker_pool_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service eigenvector worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const takeover_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms + 205}); + defer alloc.free(takeover_now); + const replacement = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-eigenvector-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + takeover_now, + "4", + ); + now_ms += 206; + if (replacement.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service eigenvector worker-pool to acquire expired lease\n", .{}); + return error.GraphMetricProcessProofFailed; + } + break :blk replacement; + } else blk: { + const summary = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + worker_now, + "4", + ); + now_ms += 1; + break :blk summary; + }; + const expected_min_pages: usize = if (phase == .prepare_generation or phase == .scan_edges_and_out_degree) 1 else 2; + if (worker.result.pages_completed < expected_min_pages or worker.stats.worker_count != 2) { + std.debug.print("expected service worker-pool to complete at least {d} eigenvector pages for phase {}, got {d}\n", .{ + expected_min_pages, + phase, + worker.result.pages_completed, + }); + return error.GraphMetricProcessProofFailed; + } + + const coordinator_owner = try std.fmt.allocPrint(alloc, "service-eigenvector-multipage-coordinator-{d}", .{i}); + defer alloc.free(coordinator_owner); + const coordinator_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(coordinator_now); + var coordinator = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + coordinator_owner, + "5000", + coordinator_now, + ); + now_ms += 1; + if (coordinator.result.phases_advanced == 0) { + coordinator = try drainServicePhaseRemainder(alloc, io, antfly_exe, base_uri, coordinator_runtime, worker_pool_runtime, &now_ms); + } + const next_phase = switch (phase) { + .prepare_generation => antfly.graph.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, + .scan_edges_and_out_degree => .initialize_ranks, + .initialize_ranks => .iterate_contributions, + .iterate_contributions => .reduce_ranks, + .reduce_ranks => .check_convergence, + .check_convergence => .publish_generation, + else => unreachable, + }; + try assertOpenDbMetricActivePhase(alloc, &db, "eigenvector", next_phase, target_generation); + } + + const publish_worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_worker_now); + const publish_worker = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-eigenvector-multipage-worker-pool-publish", + "service-process-worker-a,service-process-worker-b", + "5000", + publish_worker_now, + "4", + ); + now_ms += 1; + if (publish_worker.result.pages_completed == 0) { + std.debug.print("expected service worker-pool to materialize eigenvector publish pages\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + const publish_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_now); + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-eigenvector-multipage-coordinator-publish", + "5000", + publish_now, + ); + now_ms += 1; + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish multi-page eigenvector build\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbMetricPhase(alloc, &db, "eigenvector", .cleanup_old_generations, target_generation); + + var cleanup_fresh = false; + for (0..8) |i| { + var status = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + cleanup_fresh = true; + break; + } + + const owner_id = try std.fmt.allocPrint(alloc, "service-eigenvector-multipage-worker-pool-cleanup-{d}", .{i}); + defer alloc.free(owner_id); + const cleanup_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(cleanup_now); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + cleanup_now, + "4", + ); + now_ms += 1; + if (cleanup.stats.worker_count != 2) { + std.debug.print("expected service eigenvector cleanup worker-pool to keep two configured workers\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + if (!cleanup_fresh) { + var status = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + cleanup_fresh = status.state == antfly.graph.GraphIndex.GraphMetricState.fresh; + } + if (!cleanup_fresh) { + std.debug.print("expected service worker-pool to finish multi-page eigenvector cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyFixedIterationMetadata(alloc, db_path, "eigenvector", target_generation, 1); +} + +fn verifyHitsServiceTargetedPublishAndCleanupRestartProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + try prepareMetricBuildToPhase(alloc, db_path, "hits_authority", target_generation, .publish_generation); + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-coordinator", + "service-hits-publish-cleanup-coordinator-a", + "5000", + "78000", + ); + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish HITS pair before cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbHitsAfterPairedPublish(alloc, &db, target_generation); + + const duplicate_publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-coordinator", + "service-hits-publish-cleanup-coordinator-b", + "5000", + "78001", + ); + try assertDuplicateCoordinatorDidNotMutate(duplicate_publish, "service hits cleanup"); + try assertOpenDbHitsAfterPairedPublish(alloc, &db, target_generation); + + const cleanup_ready_file = ".zig-cache/tmp/graph-metric-process-service-hits-publish-cleanup-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, cleanup_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-worker-pool", + "service-hits-publish-cleanup-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + "78002", + "1", + cleanup_ready_file, + ); + try assertOpenDbHitsAfterPairedPublish(alloc, &db, target_generation); + + const fenced_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-worker-pool", + "service-hits-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "78100", + "4", + ); + if (fenced_cleanup.durable_progressed or fenced_cleanup.stats.has_lease or fenced_cleanup.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate service HITS cleanup worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + try assertOpenDbHitsAfterPairedPublish(alloc, &db, target_generation); + + const takeover_cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-worker-pool", + "service-hits-publish-cleanup-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + "78203", + "4", + ); + if (takeover_cleanup.stats.takeover_count == 0 or takeover_cleanup.result.pages_claimed == 0 or takeover_cleanup.result.pages_completed == 0) { + std.debug.print("expected replacement service HITS cleanup worker-pool to take over and advance cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + var cleanup_progressed = takeover_cleanup.durable_progressed; + for (0..12) |i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-hits-publish-cleanup-worker-pool-{d}", .{i}); + defer alloc.free(owner_id); + const now_ms = try std.fmt.allocPrint(alloc, "{d}", .{78204 + i}); + defer alloc.free(now_ms); + const cleanup = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-publish-cleanup-worker-pool", + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + now_ms, + ); + cleanup_progressed = cleanup_progressed or cleanup.durable_progressed or cleanup.result.pages_claimed != 0 or cleanup.result.pages_completed != 0 or cleanup.result.published != 0; + if (cleanup.result.published != 0) break; + } + if (!cleanup_progressed) { + std.debug.print("expected service worker-pool processes to advance HITS cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyHitsFresh(alloc, db_path, target_generation); + try verifyHitsFixedIterationMetadata(alloc, db_path, target_generation, 1); +} + +fn verifyHitsServiceTargetedMultiPageWorkerPoolProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + target_generation: u64, +) !void { + { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + const coordinator_runtime = "service-hits-multipage-coordinator"; + const worker_pool_runtime = "service-hits-multipage-worker-pool"; + var now_ms: u64 = 84000; + const coordinator_ready_file = ".zig-cache/tmp/graph-metric-process-service-hits-multipage-coordinator-ready"; + std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, coordinator_ready_file) catch {}; + try runAndKillServiceCoordinatorAfterReady( + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-hits-multipage-coordinator-start", + "200", + "84000", + coordinator_ready_file, + ); + try assertOpenDbMetricActivePhase(alloc, &db, "hits_authority", .prepare_generation, target_generation); + + const coordinator_fenced = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-hits-multipage-coordinator-replacement", + "200", + "84100", + ); + if (coordinator_fenced.durable_progressed or coordinator_fenced.stats.has_lease or coordinator_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service HITS coordinator to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const coordinator_takeover = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-hits-multipage-coordinator-replacement", + "200", + "84201", + ); + if (coordinator_takeover.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service HITS coordinator to acquire expired lease\n", .{}); + return error.GraphMetricProcessProofFailed; + } + now_ms = 84202; + try assertOpenDbMetricActivePhase(alloc, &db, "hits_authority", .prepare_generation, target_generation); + + const phases = [_]antfly.graph.GraphIndex.GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + }; + for (phases, 0..) |phase, i| { + const owner_id = try std.fmt.allocPrint(alloc, "service-hits-multipage-worker-pool-{d}", .{i}); + defer alloc.free(owner_id); + const worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(worker_now); + const worker = if (phase == .scan_edges_and_out_degree) blk: { + const worker_ready_file = ".zig-cache/tmp/graph-metric-process-service-hits-multipage-worker-pool-ready"; + std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + defer std.Io.Dir.cwd().deleteFile(io, worker_ready_file) catch {}; + try runAndKillServiceWorkerPoolAfterReady( + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-hits-multipage-worker-pool-killed", + "service-process-worker-a,service-process-worker-b", + "200", + worker_now, + worker_ready_file, + ); + now_ms += 1; + + const fenced_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(fenced_now); + const worker_pool_fenced = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-hits-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + fenced_now, + "4", + ); + now_ms += 1; + if (worker_pool_fenced.durable_progressed or worker_pool_fenced.stats.has_lease or worker_pool_fenced.stats.lease_acquire_failures == 0) { + std.debug.print("expected duplicate multi-page service HITS worker-pool to be fenced before lease expiry\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + const takeover_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms + 205}); + defer alloc.free(takeover_now); + const replacement = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-hits-multipage-worker-pool-replacement", + "service-process-worker-a,service-process-worker-b", + "200", + takeover_now, + "4", + ); + now_ms += 206; + if (replacement.stats.takeover_count == 0) { + std.debug.print("expected replacement multi-page service HITS worker-pool to acquire expired lease\n", .{}); + return error.GraphMetricProcessProofFailed; + } + break :blk replacement; + } else blk: { + const summary = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + worker_now, + "4", + ); + now_ms += 1; + break :blk summary; + }; + const expected_min_pages: usize = if (phase == .prepare_generation or phase == .scan_edges_and_out_degree) 1 else 2; + if (worker.result.pages_completed < expected_min_pages or worker.stats.worker_count != 2) { + std.debug.print("expected service worker-pool to complete at least {d} HITS pages for phase {}, got {d}\n", .{ + expected_min_pages, + phase, + worker.result.pages_completed, + }); + return error.GraphMetricProcessProofFailed; + } + + const coordinator_owner = try std.fmt.allocPrint(alloc, "service-hits-multipage-coordinator-{d}", .{i}); + defer alloc.free(coordinator_owner); + const coordinator_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(coordinator_now); + var coordinator = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + coordinator_owner, + "5000", + coordinator_now, + ); + now_ms += 1; + if (coordinator.result.phases_advanced == 0) { + coordinator = try drainServicePhaseRemainder(alloc, io, antfly_exe, base_uri, coordinator_runtime, worker_pool_runtime, &now_ms); + } + const next_phase = switch (phase) { + .prepare_generation => antfly.graph.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, + .scan_edges_and_out_degree => .initialize_ranks, + .initialize_ranks => .iterate_contributions, + .iterate_contributions => .reduce_ranks, + .reduce_ranks => .hits_hub_contributions, + .hits_hub_contributions => .hits_hub_reduce_ranks, + .hits_hub_reduce_ranks => .check_convergence, + .check_convergence => .publish_generation, + else => unreachable, + }; + try assertOpenDbMetricActivePhase(alloc, &db, "hits_authority", next_phase, target_generation); + } + + const publish_worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_worker_now); + const publish_worker = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + "service-hits-multipage-worker-pool-publish", + "service-process-worker-a,service-process-worker-b", + "5000", + publish_worker_now, + "4", + ); + now_ms += 1; + if (publish_worker.result.pages_completed == 0) { + std.debug.print("expected service worker-pool to materialize HITS publish pages\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + const publish_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(publish_now); + const publish = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "service-hits-multipage-coordinator-publish", + "5000", + publish_now, + ); + now_ms += 1; + if (publish.result.published != 1 or publish.result.phases_advanced == 0) { + std.debug.print("expected service coordinator process to publish multi-page HITS pair\n", .{}); + return error.GraphMetricProcessProofFailed; + } + try assertOpenDbHitsAfterPairedPublish(alloc, &db, target_generation); + + var cleanup_fresh = false; + for (0..12) |i| { + var authority = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state == antfly.graph.GraphIndex.GraphMetricState.fresh and hub.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + cleanup_fresh = true; + break; + } + + const owner_id = try std.fmt.allocPrint(alloc, "service-hits-multipage-worker-pool-cleanup-{d}", .{i}); + defer alloc.free(owner_id); + const cleanup_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(cleanup_now); + const cleanup = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_pool_runtime, + owner_id, + "service-process-worker-a,service-process-worker-b", + "5000", + cleanup_now, + "4", + ); + now_ms += 1; + if (cleanup.stats.worker_count != 2) { + std.debug.print("expected service HITS cleanup worker-pool to keep two configured workers\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + if (!cleanup_fresh) { + var authority = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try (db.core.graphIndex("graph_idx") orelse return error.IndexNotFound).index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + cleanup_fresh = authority.state == antfly.graph.GraphIndex.GraphMetricState.fresh and hub.state == antfly.graph.GraphIndex.GraphMetricState.fresh; + } + if (!cleanup_fresh) { + std.debug.print("expected service worker-pool to finish multi-page HITS cleanup\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try verifyHitsFresh(alloc, db_path, target_generation); + try verifyHitsFixedIterationMetadata(alloc, db_path, target_generation, 1); +} + +fn assertOpenDbMetricPhase( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + metric_name: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_published_generation: u64, +) !void { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase != expected_phase or status.published_generation != expected_published_generation) { + std.debug.print("expected {s} phase {} published generation {d}, got phase {} published generation {d}\n", .{ + metric_name, + expected_phase, + expected_published_generation, + status.phase, + status.published_generation, + }); + return error.GraphMetricProcessProofFailed; + } +} + +fn assertOpenDbMetricActivePhase( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + metric_name: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_building_generation: u64, +) !void { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase != expected_phase or status.building_generation != expected_building_generation) { + std.debug.print("expected active {s} phase {} building generation {d}, got phase {} building generation {d}\n", .{ + metric_name, + expected_phase, + expected_building_generation, + status.phase, + status.building_generation, + }); + return error.GraphMetricProcessProofFailed; + } +} + +fn assertOpenDbHitsAfterPairedPublish( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + target_generation: u64, +) !void { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state != antfly.graph.GraphIndex.GraphMetricState.building or + authority.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations) + { + std.debug.print("expected service HITS authority to be cleaning after paired publish, got {}/{}\n", .{ authority.state, authority.phase }); + return error.GraphMetricProcessProofFailed; + } + if (hub.state != antfly.graph.GraphIndex.GraphMetricState.building or + hub.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations) + { + std.debug.print("expected service HITS hub to share paired cleanup lifecycle, got {}/{}\n", .{ hub.state, hub.phase }); + return error.GraphMetricProcessProofFailed; + } + if (authority.published_generation != target_generation or hub.published_generation != target_generation) { + std.debug.print( + "expected service paired HITS published generation {d}, got authority {d} hub {d}\n", + .{ target_generation, authority.published_generation, hub.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } +} + +fn verifyDegreeActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + try runSupervisorProcess(alloc, io, antfly_exe, db_path); + try verifyDegreeFresh(alloc, db_path, initial_generation); + + const rebuild_generation = try addDegreeDirtyEdge(alloc, db_path); + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const started = try runCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + db_path, + "degree-active-read-proof-coordinator", + "5000", + "26500", + ); + if (!started.durable_progressed or !started.stats.has_lease) { + std.debug.print("expected coordinator process to start active degree rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building or + status.published_generation != initial_generation or + status.building_generation != rebuild_generation) + { + std.debug.print( + "expected coordinator process to leave degree rebuilding at generations {d}/{d}, got state {} generations {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + status.state, + status.published_generation, + status.building_generation, + }, + ); + return error.GraphMetricDegreeProcessProofFailed; + } + } + + try verifyDegreeActivePublicReadSurface(alloc, &db, initial_generation, rebuild_generation); +} + +fn verifyDegreeServiceActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + var now_ms: u64 = 2700; + var idle_rounds: usize = 0; + var initial_fresh = false; + for (0..80) |_| { + const now_coordinator = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_coordinator); + const coordinator_summary = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-active-public-read-coordinator", + "service-degree-active-public-read-coordinator", + "200", + now_coordinator, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + const worker_summary = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-active-public-read-worker-pool", + "service-degree-active-public-read-worker-pool", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + now_ms += 1; + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + if (status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + initial_fresh = true; + break; + } + if (coordinator_summary.durable_progressed or worker_summary.durable_progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= 8) break; + } + } + if (!initial_fresh) return error.GraphMetricBuildNotComplete; + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:new", + .value = "{\"title\":\"new source\",\"body\":\"newsource graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:hub\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const rebuild_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const now_rebuild = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_rebuild); + _ = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-degree-active-public-read-coordinator", + "service-degree-active-public-read-coordinator", + "200", + now_rebuild, + ); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building or + status.published_generation != initial_generation or + status.building_generation != rebuild_generation) + { + std.debug.print( + "expected service coordinator process to leave degree rebuilding at generations {d}/{d}, got state {} generations {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + status.state, + status.published_generation, + status.building_generation, + }, + ); + return error.GraphMetricDegreeProcessProofFailed; + } + } + { + const pending = db.pendingWorkStats().graph_metric; + if (pending.active_builds == 0) { + std.debug.print("expected service coordinator process to leave active degree rebuild work\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + } + try verifyDegreeActivePublicReadSurface(alloc, &db, initial_generation, rebuild_generation); +} + +fn isProcessHarnessDoc0Node(node: []const u8) bool { + return std.mem.eql(u8, node, "doc:0") or std.mem.eql(u8, node, "0"); +} + +fn verifyDegreeActivePublicReadSurface( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + initial_generation: u64, + rebuild_generation: u64, +) !void { + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 3, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 1) { + std.debug.print("expected one degree graph metric result during service active rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + const result = published_result.graph_metric_results[0]; + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service active degree query status building, got {}\n", .{result.status.state}); + return error.GraphMetricDegreeProcessProofFailed; + } + if (result.status.published_generation != initial_generation or result.status.building_generation != rebuild_generation) { + std.debug.print("expected service degree published/building generations {d}/{d}, got {d}/{d}\n", .{ + initial_generation, + rebuild_generation, + result.status.published_generation, + result.status.building_generation, + }); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected service active degree published top-k scores\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + var found_prior_source_score = false; + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:new") or std.mem.eql(u8, score.node, "new")) { + std.debug.print("service active degree published top-k exposed rebuilding-only source {s}\n", .{score.node}); + return error.GraphMetricDegreeProcessProofFailed; + } + if (score.score == 1.0) { + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.0000001); + found_prior_source_score = true; + } + } + if (!found_prior_source_score) { + std.debug.print("expected service active degree published top-k to include a prior source score\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:side"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected one degree graph traversal result during service active rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!isProcessHarnessDoc0Node(traversal.nodes[0].key)) { + std.debug.print("expected degree traversal to return doc:0, got {s}\n", .{traversal.nodes[0].key}); + return error.GraphMetricDegreeProcessProofFailed; + } + if (traversal.nodes[0].metrics.len != 1 or traversal.nodes[0].metrics[0].score == null) { + std.debug.print("expected service traversal published projection to serve prior degree score\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + try std.testing.expectApproxEqAbs(@as(f64, 1.0), traversal.nodes[0].metrics[0].score.?, 0.0000001); + if (traversal.metric_status.len != 1 or traversal.metric_status[0].state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service traversal metric status building during active degree rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + if (traversal.metric_status[0].published_generation != initial_generation or traversal.metric_status[0].building_generation != rebuild_generation) { + std.debug.print("expected service degree traversal status to report generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 3, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during service active degree rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during service active degree rebuild\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service degree rerank status building, got {}\n", .{rerank_status.state}); + return error.GraphMetricDegreeProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation or rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected service degree rerank status generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + var found_prior_metric_score = false; + for (rerank_result.hits) |hit| { + const details = hit.score_details orelse { + std.debug.print("expected service degree reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricDegreeProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected service degree reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + if (std.mem.eql(u8, hit.id, "doc:new") or std.mem.eql(u8, hit.id, "new")) { + if (details.metric_score != null or !details.missing_score_used) { + std.debug.print("service active degree search rerank gave rebuilding-only document {s} a published metric score\n", .{hit.id}); + return error.GraphMetricDegreeProcessProofFailed; + } + continue; + } + if (details.metric_score) |metric_score| { + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_score, 0.0000001); + found_prior_metric_score = true; + } + } + if (!found_prior_metric_score) { + std.debug.print("expected service degree rerank to include a prior published metric score\n", .{}); + return error.GraphMetricDegreeProcessProofFailed; + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 3, + .include_stored = false, + })); +} + +fn verifyPageRankServiceActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + var now_ms: u64 = 3000; + var idle_rounds: usize = 0; + var initial_fresh = false; + for (0..80) |_| { + const now_coordinator = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_coordinator); + const coordinator_summary = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-active-public-read-coordinator", + "service-pagerank-active-public-read-coordinator", + "200", + now_coordinator, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + const worker_summary = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-active-public-read-worker-pool", + "service-pagerank-active-public-read-worker-pool", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + now_ms += 1; + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + if (status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + initial_fresh = true; + break; + } + if (coordinator_summary.durable_progressed or worker_summary.durable_progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= 8) break; + } + } + if (!initial_fresh) return error.GraphMetricBuildNotComplete; + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:e", + .value = "{\"title\":\"epsilon\",\"body\":\"epsilon graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const rebuild_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const now_rebuild = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_rebuild); + _ = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-pagerank-active-public-read-coordinator", + "service-pagerank-active-public-read-coordinator", + "200", + now_rebuild, + ); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building or + status.published_generation != initial_generation or + status.building_generation != rebuild_generation) + { + std.debug.print( + "expected service coordinator process to leave PageRank rebuilding at generations {d}/{d}, got state {} generations {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + status.state, + status.published_generation, + status.building_generation, + }, + ); + return error.GraphMetricPageRankProcessProofFailed; + } + } + { + const pending = db.pendingWorkStats().graph_metric; + if (pending.active_builds == 0) { + std.debug.print("expected service coordinator process to leave active PageRank rebuild work\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + try verifyPageRankActivePublicReadSurface(alloc, &db, initial_generation, rebuild_generation); +} + +fn verifyPageRankActivePublicReadSurface( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + initial_generation: u64, + rebuild_generation: u64, +) !void { + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 1) { + std.debug.print("expected one PageRank graph metric result during service active rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const result = published_result.graph_metric_results[0]; + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service active PageRank query status building, got {}\n", .{result.status.state}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (result.status.published_generation != initial_generation or result.status.building_generation != rebuild_generation) { + std.debug.print("expected service published/building generations {d}/{d}, got {d}/{d}\n", .{ + initial_generation, + rebuild_generation, + result.status.published_generation, + result.status.building_generation, + }); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected service active PageRank published read to serve prior scores\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:e")) { + std.debug.print("service active PageRank published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricPageRankProcessProofFailed; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected one PageRank graph traversal result during service active rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!std.mem.eql(u8, traversal.nodes[0].key, "doc:b")) { + std.debug.print("expected traversal to return doc:b, got {s}\n", .{traversal.nodes[0].key}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.nodes[0].metrics.len != 1 or traversal.nodes[0].metrics[0].score == null) { + std.debug.print("expected service traversal published projection to serve prior PageRank score\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.metric_status.len != 1 or traversal.metric_status[0].state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service traversal metric status building during active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (traversal.metric_status[0].published_generation != initial_generation or traversal.metric_status[0].building_generation != rebuild_generation) { + std.debug.print("expected service traversal status to report generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "pagerank", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during service active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during service active PageRank rebuild\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service rerank status building, got {}\n", .{rerank_status.state}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation or rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected service rerank status generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:e")) { + std.debug.print("service active PageRank search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricPageRankProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricPageRankProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected service reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +fn verifyEigenvectorServiceActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + var now_ms: u64 = 3300; + var idle_rounds: usize = 0; + var initial_fresh = false; + for (0..80) |_| { + const now_coordinator = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_coordinator); + const coordinator_summary = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-active-public-read-coordinator", + "service-eigenvector-active-public-read-coordinator", + "200", + now_coordinator, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + const worker_summary = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-active-public-read-worker-pool", + "service-eigenvector-active-public-read-worker-pool", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + now_ms += 1; + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state == antfly.graph.GraphIndex.GraphMetricState.fresh) { + if (status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + initial_fresh = true; + break; + } + if (coordinator_summary.durable_progressed or worker_summary.durable_progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= 8) break; + } + } + if (!initial_fresh) return error.GraphMetricBuildNotComplete; + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:e", + .value = "{\"title\":\"epsilon\",\"body\":\"epsilon graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:c\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const rebuild_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const now_rebuild = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_rebuild); + _ = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-eigenvector-active-public-read-coordinator", + "service-eigenvector-active-public-read-coordinator", + "200", + now_rebuild, + ); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building or + status.published_generation != initial_generation or + status.building_generation != rebuild_generation) + { + std.debug.print( + "expected service coordinator process to leave eigenvector rebuilding at generations {d}/{d}, got state {} generations {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + status.state, + status.published_generation, + status.building_generation, + }, + ); + return error.GraphMetricProcessProofFailed; + } + } + { + const pending = db.pendingWorkStats().graph_metric; + if (pending.active_builds == 0) { + std.debug.print("expected service coordinator process to leave active eigenvector rebuild work\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 1) { + std.debug.print("expected one eigenvector graph metric result during service active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const result = published_result.graph_metric_results[0]; + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service active eigenvector query status building, got {}\n", .{result.status.state}); + return error.GraphMetricProcessProofFailed; + } + if (result.status.published_generation != initial_generation or result.status.building_generation != rebuild_generation) { + std.debug.print("expected service eigenvector published/building generations {d}/{d}, got {d}/{d}\n", .{ + initial_generation, + rebuild_generation, + result.status.published_generation, + result.status.building_generation, + }); + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected service active eigenvector published read to serve prior scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:e")) { + std.debug.print("service active eigenvector published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricProcessProofFailed; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "eigenvector", + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len == 0) { + std.debug.print("expected eigenvector graph traversal result during service active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + for (traversal.nodes) |node| { + if (node.metrics.len != 1 or node.metrics[0].score == null) { + std.debug.print("expected service traversal published projection to serve prior eigenvector score\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + if (traversal.metric_status.len != 1 or traversal.metric_status[0].state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service traversal metric status building during active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + if (traversal.metric_status[0].published_generation != initial_generation or traversal.metric_status[0].building_generation != rebuild_generation) { + std.debug.print("expected service traversal status to report eigenvector published/building generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "eigenvector", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during service active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during service active eigenvector rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building) { + std.debug.print("expected service eigenvector rerank status building, got {}\n", .{rerank_status.state}); + return error.GraphMetricProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation or rerank_status.building_generation != rebuild_generation) { + std.debug.print("expected service eigenvector rerank status generations {d}/{d}\n", .{ initial_generation, rebuild_generation }); + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:e")) { + std.debug.print("service active eigenvector search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected service eigenvector reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + }; + if (details.published_generation != initial_generation) { + std.debug.print("expected service eigenvector reranked hit {s} to use published generation {d}, got {d}\n", .{ hit.id, initial_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +fn verifyHitsServiceActiveProcessPublicReadFreshness( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + initial_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const api_runtime = try ProcessHarnessApiRuntime.start(alloc, io, &db); + defer api_runtime.deinit(); + const base_uri = try api_runtime.baseUri(alloc); + defer alloc.free(base_uri); + + var now_ms: u64 = 3600; + var idle_rounds: usize = 0; + var initial_fresh = false; + for (0..80) |_| { + const now_coordinator = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_coordinator); + const coordinator_summary = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-active-public-read-coordinator", + "service-hits-active-public-read-coordinator", + "200", + now_coordinator, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + const worker_summary = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-active-public-read-worker-pool", + "service-hits-active-public-read-worker-pool", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + now_ms += 1; + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state == antfly.graph.GraphIndex.GraphMetricState.fresh and + hub.state == antfly.graph.GraphIndex.GraphMetricState.fresh) + { + if (authority.published_generation != initial_generation or + hub.published_generation != initial_generation) + { + return error.GraphMetricGenerationMismatch; + } + initial_fresh = true; + break; + } + if (coordinator_summary.durable_progressed or worker_summary.durable_progressed) { + idle_rounds = 0; + } else { + idle_rounds += 1; + if (idle_rounds >= 8) break; + } + } + if (!initial_fresh) return error.GraphMetricBuildNotComplete; + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:hub-c", + .value = "{\"title\":\"hub c\",\"body\":\"hub c graph\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const rebuild_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + if (rebuild_generation <= initial_generation) return error.GraphMetricGenerationMismatch; + + const now_rebuild = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_rebuild); + _ = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-active-public-read-coordinator", + "service-hits-active-public-read-coordinator", + "200", + now_rebuild, + ); + now_ms += 1; + + const now_worker = try std.fmt.allocPrint(alloc, "{d}", .{now_ms}); + defer alloc.free(now_worker); + _ = try runServiceWorkerPoolRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + "service-hits-active-public-read-worker-pool", + "service-hits-active-public-read-worker-pool", + "service-process-worker-a,service-process-worker-b", + "200", + now_worker, + ); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state != antfly.graph.GraphIndex.GraphMetricState.building or + authority.published_generation != initial_generation or + authority.building_generation != rebuild_generation) + { + std.debug.print( + "expected service coordinator process to leave HITS authority rebuilding at generations {d}/{d}, got state {} generations {d}/{d}\n", + .{ + initial_generation, + rebuild_generation, + authority.state, + authority.published_generation, + authority.building_generation, + }, + ); + return error.GraphMetricProcessProofFailed; + } + if (hub.state != antfly.graph.GraphIndex.GraphMetricState.building and + hub.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected service HITS hub status building or stale, got {}\n", .{hub.state}); + return error.GraphMetricProcessProofFailed; + } + if (hub.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + if (hub.state == antfly.graph.GraphIndex.GraphMetricState.building and hub.building_generation != rebuild_generation) { + return error.GraphMetricGenerationMismatch; + } + } + { + const pending = db.pendingWorkStats().graph_metric; + if (pending.active_builds == 0) { + std.debug.print("expected service coordinator process to leave active HITS rebuild work\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + + try verifyHitsActivePublicReadSurface(alloc, &db, initial_generation, rebuild_generation); +} + +fn verifyHitsActivePublicReadSurface( + alloc: std.mem.Allocator, + db: *antfly.db.DB, + initial_generation: u64, + rebuild_generation: u64, +) !void { + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .published, + }, + }, + }, + .limit = 0, + }); + defer published_result.deinit(); + if (published_result.graph_metric_results.len != 2) { + std.debug.print("expected two service HITS graph metric results during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (published_result.graph_metric_results) |result| { + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.building and + result.status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected service active HITS query status building or stale, got {}\n", .{result.status.state}); + return error.GraphMetricProcessProofFailed; + } + if (result.status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + if (result.status.state == antfly.graph.GraphIndex.GraphMetricState.building and + result.status.building_generation != rebuild_generation) + { + return error.GraphMetricGenerationMismatch; + } + if (result.scores.len == 0) { + std.debug.print("expected service active HITS published read to serve prior scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (result.scores) |score| { + if (std.mem.eql(u8, score.node, "doc:hub-c")) { + std.debug.print("service active HITS published read exposed rebuilding node {s}\n", .{score.node}); + return error.GraphMetricProcessProofFailed; + } + } + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 1, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 1, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .published }, + .{ .name = "hits_hub", .freshness = .published }, + }; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:hub-a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected one service HITS graph traversal result during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!std.mem.eql(u8, traversal.nodes[0].key, "doc:authority")) { + std.debug.print("expected service HITS traversal to return doc:authority, got {s}\n", .{traversal.nodes[0].key}); + return error.GraphMetricProcessProofFailed; + } + if (traversal.nodes[0].metrics.len != 2) { + std.debug.print("expected service HITS traversal to project authority and hub scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.nodes[0].metrics) |metric| { + if (metric.score == null) { + std.debug.print("expected service HITS traversal metric {s} to serve a prior published score\n", .{metric.name}); + return error.GraphMetricProcessProofFailed; + } + } + if (traversal.metric_status.len != 2) { + std.debug.print("expected two service HITS traversal metric statuses during active rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.metric_status) |status| { + if (status.state != antfly.graph.GraphIndex.GraphMetricState.building and + status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected service HITS traversal status building or stale, got {}\n", .{status.state}); + return error.GraphMetricProcessProofFailed; + } + if (status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + if (status.state == antfly.graph.GraphIndex.GraphMetricState.building and status.building_generation != rebuild_generation) { + return error.GraphMetricGenerationMismatch; + } + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 3, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected search rerank hits during service active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected search rerank status during service active HITS rebuild\n", .{}); + return error.GraphMetricProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.building and + rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.stale) + { + std.debug.print("expected service HITS search rerank status building or stale, got {}\n", .{rerank_status.state}); + return error.GraphMetricProcessProofFailed; + } + if (rerank_status.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + if (rerank_status.state == antfly.graph.GraphIndex.GraphMetricState.building and rerank_status.building_generation != rebuild_generation) { + return error.GraphMetricGenerationMismatch; + } + for (rerank_result.hits) |hit| { + if (std.mem.eql(u8, hit.id, "doc:hub-c")) { + std.debug.print("service active HITS search rerank exposed rebuilding-only document {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + } + const details = hit.score_details orelse { + std.debug.print("expected service HITS reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + }; + if (details.published_generation != initial_generation) return error.GraphMetricGenerationMismatch; + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 3, + .include_stored = false, + })); +} + +const ProcessHarnessStatusSource = struct { + fn iface(self: *ProcessHarnessStatusSource) antfly.public_api.http_server.StatusSource { + return .{ + .ptr = self, + .vtable = &.{ + .status = status, + }, + }; + } + + fn status(_: *anyopaque) !antfly.metadata_api.MetadataStatus { + return .{ .metadata_group_id = 1, .metrics = .{}, .projected_stores = 1 }; + } +}; + +/// Owns the same opaque API-kernel and `httpx` composition used by production +/// runtimes. Keeping the owner heap-stable is required because the API server +/// retains pointers to the status and table-write sources below. +const ProcessHarnessApiRuntime = struct { + alloc: std.mem.Allocator, + write_source: antfly.public_api.BoundTableWriteSource, + status_source: ProcessHarnessStatusSource, + api_server: antfly.public_api.kernel_bridge.ApiHttpServer, + handler: antfly.public_api.kernel_bridge.HttpxHandler, + http_server: httpx.Server, + listener_task: httpx.ListenerTask, + + fn start( + alloc: std.mem.Allocator, + io: std.Io, + db: *antfly.db.DB, + ) !*ProcessHarnessApiRuntime { + const runtime = try alloc.create(ProcessHarnessApiRuntime); + errdefer alloc.destroy(runtime); + + runtime.alloc = alloc; + runtime.write_source = antfly.public_api.BoundTableWriteSource.init("docs", db); + runtime.status_source = .{}; + runtime.api_server = try antfly.public_api.kernel_bridge.ApiHttpServer.initWithConfig( + alloc, + .{ + .internal_service_secret = harness_internal_service_secret, + .internal_service_issuer = harness_internal_service_issuer, + }, + runtime.status_source.iface(), + null, + runtime.write_source.source(), + ); + errdefer runtime.api_server.deinit(); + + runtime.handler = try antfly.public_api.kernel_bridge.createHandler(&runtime.api_server); + errdefer antfly.public_api.kernel_bridge.deinitHandler(&runtime.handler); + try runtime.handler.initRuntime(alloc); + + runtime.http_server = httpx.Server.initWithConfig(alloc, io, .{ + .host = "127.0.0.1", + .port = 0, + .max_connections = 32, + .max_request_tasks = 32, + }); + errdefer runtime.http_server.deinit(); + try runtime.handler.registerRoutes(&runtime.http_server); + + runtime.listener_task = httpx.ListenerTask.init(&runtime.http_server); + try runtime.listener_task.start(); + return runtime; + } + + fn deinit(self: *ProcessHarnessApiRuntime) void { + const alloc = self.alloc; + self.listener_task.shutdown(30_000); + self.listener_task.join() catch |err| { + std.log.err("graph metric process HTTP listener failed during shutdown err={s}", .{@errorName(err)}); + }; + self.http_server.deinit(); + antfly.public_api.kernel_bridge.deinitHandler(&self.handler); + self.api_server.deinit(); + alloc.destroy(self); + } + + fn baseUri(self: *ProcessHarnessApiRuntime, alloc: std.mem.Allocator) ![]u8 { + const address = self.http_server.boundAddress() orelse return error.NotListening; + return std.fmt.allocPrint(alloc, "http://{f}", .{address}); + } +}; + +fn runAndKillCoordinatorAfterReady( + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "coordinator", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--test-ready-file", + ready_file, + "--test-hold-after-run-ms", + "10000", + }; + try verifyRoleProcessArgvScoped(argv[0..]); + var child = try std.process.spawn(io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .stdin = .ignore, + .stdout = .ignore, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var ready = false; + for (0..100) |_| { + std.Io.Dir.cwd().access(io, ready_file, .{}) catch { + platform.time.sleepNs(50 * std.time.ns_per_ms); + continue; + }; + ready = true; + break; + } + if (!ready) { + child.kill(io); + std.debug.print("timed out waiting for killable coordinator ready marker\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + child.kill(io); +} + +fn runAndKillServiceCoordinatorAfterReady( + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--base-uri", + base_uri, + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "coordinator", + "--runtime-id", + runtime_id, + "--owner-id", + owner_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + "--test-now-ms", + test_now_ms, + "--test-ready-file", + ready_file, + "--test-hold-after-run-ms", + "10000", + }; + try runAndKillRoleProcessAfterReady(io, argv[0..], ready_file, error.GraphMetricLeaseProofFailed); +} + +fn runAndKillServiceWorkerPoolAfterReady( + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + worker_ids: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, + ready_file: []const u8, +) !void { + try runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io, + antfly_exe, + base_uri, + runtime_id, + owner_id, + worker_ids, + lease_ttl_ms, + test_now_ms, + "2", + ready_file, + ); +} + +fn runAndKillServiceWorkerPoolAfterReadyWithMaxPages( + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + worker_ids: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, + max_pages: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--base-uri", + base_uri, + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "worker_pool", + "--runtime-id", + runtime_id, + "--owner-id", + owner_id, + "--worker-ids", + worker_ids, + "--lease-ttl-ms", + lease_ttl_ms, + "--coordinator-start-background-builds", + "false", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + max_pages, + "--test-now-ms", + test_now_ms, + "--test-ready-file", + ready_file, + "--test-hold-after-run-ms", + "10000", + }; + try runAndKillRoleProcessAfterReady(io, argv[0..], ready_file, error.GraphMetricLeaseProofFailed); +} + +fn runAndKillRoleProcessAfterReady( + io: std.Io, + argv: []const []const u8, + ready_file: []const u8, + err: anyerror, +) !void { + try verifyRoleProcessArgvScoped(argv); + var child = try std.process.spawn(io, .{ + .environ_map = child_environ, + .argv = argv, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var ready = false; + for (0..100) |_| { + std.Io.Dir.cwd().access(io, ready_file, .{}) catch { + platform.time.sleepNs(50 * std.time.ns_per_ms); + continue; + }; + ready = true; + break; + } + if (!ready) { + child.kill(io); + std.debug.print("timed out waiting for killable service role ready marker\n", .{}); + return err; + } + + child.kill(io); +} + +fn runAndKillDegreePageOwnerAfterReady( + io: std.Io, + harness_exe: []const u8, + db_path: []const u8, + worker_id: []const u8, + now_ms: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + harness_exe, + "claim-degree-page-hold", + db_path, + worker_id, + now_ms, + ready_file, + "10000", + }; + var child = try std.process.spawn(io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .stdin = .ignore, + .stdout = .ignore, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var ready = false; + for (0..100) |_| { + std.Io.Dir.cwd().access(io, ready_file, .{}) catch { + platform.time.sleepNs(50 * std.time.ns_per_ms); + continue; + }; + ready = true; + break; + } + if (!ready) { + child.kill(io); + std.debug.print("timed out waiting for killable page owner ready marker\n", .{}); + return error.GraphMetricWorkerPageProofFailed; + } + + child.kill(io); +} + +fn runAndKillWorkerRoleAfterReady( + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + worker_id: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "worker", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--worker-id", + worker_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--test-now-ms", + test_now_ms, + "--test-ready-file", + ready_file, + "--test-hold-after-run-ms", + "10000", + }; + try verifyRoleProcessArgvScoped(argv[0..]); + var child = try std.process.spawn(io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .stdin = .ignore, + .stdout = .ignore, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var ready = false; + for (0..100) |_| { + std.Io.Dir.cwd().access(io, ready_file, .{}) catch { + platform.time.sleepNs(50 * std.time.ns_per_ms); + continue; + }; + ready = true; + break; + } + if (!ready) { + child.kill(io); + std.debug.print("timed out waiting for killable worker runtime ready marker\n", .{}); + return error.GraphMetricLeaseProofFailed; + } + + child.kill(io); +} + +fn runAndKillMetricPageOwnerAfterReady( + io: std.Io, + harness_exe: []const u8, + db_path: []const u8, + metric_name: []const u8, + phase: []const u8, + worker_id: []const u8, + now_ms: []const u8, + ready_file: []const u8, +) !void { + const argv = [_][]const u8{ + harness_exe, + "claim-metric-page-hold", + db_path, + metric_name, + phase, + worker_id, + now_ms, + ready_file, + "10000", + }; + var child = try std.process.spawn(io, .{ + .environ_map = child_environ, + .argv = argv[0..], + .stdin = .ignore, + .stdout = .ignore, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var ready = false; + for (0..100) |_| { + std.Io.Dir.cwd().access(io, ready_file, .{}) catch { + platform.time.sleepNs(50 * std.time.ns_per_ms); + continue; + }; + ready = true; + break; + } + if (!ready) { + child.kill(io); + std.debug.print("timed out waiting for killable metric page owner ready marker\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + + child.kill(io); +} + +fn readSingleLeasedDegreePage( + alloc: std.mem.Allocator, + db_path: []const u8, + expected_worker_id: []const u8, + expected_cursor: []const u8, +) !PageLeaseSnapshot { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.build_pages.len != 1) { + std.debug.print("expected one active degree page, got {d}\n", .{status.build_pages.len}); + return error.GraphMetricWorkerPageProofFailed; + } + const page = status.build_pages[0]; + if (page.state != antfly.graph.GraphIndex.GraphMetricBuildPageState.leased or + page.phase != antfly.graph.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree or + !std.mem.eql(u8, page.worker_id, expected_worker_id) or + !std.mem.eql(u8, page.cursor, expected_cursor)) + { + std.debug.print("unexpected active page state in worker page proof\n", .{}); + return error.GraphMetricWorkerPageProofFailed; + } + return .{ + .job_id = status.build_job_id, + .page_id = page.page_id, + .iteration = page.iteration, + .attempt = page.attempt, + .lease_expires_at_ms = page.lease_expires_at_ms, + .total_units = page.total_units, + }; +} + +fn readSingleLeasedMetricPage( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_worker_id: []const u8, + expected_cursor: []const u8, +) !PageLeaseSnapshot { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.build_pages.len != 1) { + std.debug.print("expected one active metric page, got {d}\n", .{status.build_pages.len}); + return error.GraphMetricPageRankProcessProofFailed; + } + const page = status.build_pages[0]; + if (page.state != antfly.graph.GraphIndex.GraphMetricBuildPageState.leased or + page.phase != expected_phase or + !std.mem.eql(u8, page.worker_id, expected_worker_id) or + !std.mem.eql(u8, page.cursor, expected_cursor)) + { + std.debug.print("unexpected active metric page state in process proof\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; + } + return .{ + .job_id = status.build_job_id, + .page_id = page.page_id, + .iteration = page.iteration, + .attempt = page.attempt, + .lease_expires_at_ms = page.lease_expires_at_ms, + .total_units = page.total_units, + }; +} + +fn readLeasedMetricPage( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_worker_id: []const u8, + expected_cursor: []const u8, +) !PageLeaseSnapshot { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + for (status.build_pages) |page| { + if (page.state == antfly.graph.GraphIndex.GraphMetricBuildPageState.leased and + page.phase == expected_phase and + std.mem.eql(u8, page.worker_id, expected_worker_id) and + std.mem.eql(u8, page.cursor, expected_cursor)) + { + return .{ + .job_id = status.build_job_id, + .page_id = page.page_id, + .iteration = page.iteration, + .attempt = page.attempt, + .lease_expires_at_ms = page.lease_expires_at_ms, + .total_units = page.total_units, + }; + } + } + std.debug.print("expected leased metric page for {s} phase {} owned by {s}\n", .{ metric_name, expected_phase, expected_worker_id }); + return error.GraphMetricPageRankProcessProofFailed; +} + +fn invalidateMetricBuildManifestConfigFingerprintForTest( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.phase != .publish_generation or status.build_job_id == 0) { + std.debug.print("expected {s} publish build before manifest invalidation\n", .{metric_name}); + return error.GraphMetricUnexpectedPhase; + } + try graph_entry.index.invalidateGraphMetricBuildManifestConfigFingerprintForTest(metric_name, status.build_job_id); +} + +fn expectStaleDegreePageAttemptRejected( + alloc: std.mem.Allocator, + db_path: []const u8, + stale_page: PageLeaseSnapshot, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + _ = graph_entry.index.completeGraphMetricBuildPageForAttempt( + "degree", + stale_page.job_id, + .scan_edges_and_out_degree, + 0, + stale_page.page_id, + "process-dead-worker", + stale_page.attempt, + stale_page.total_units, + 0, + ) catch |err| switch (err) { + error.GraphMetricBuildPageNotLeased => return, + error.GraphMetricBuildPageNotFound => return, + else => return err, + }; + std.debug.print("expected stale degree page attempt completion to be rejected\n", .{}); + return error.GraphMetricWorkerPageProofFailed; +} + +fn expectStaleMetricPageAttemptRejected( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + stale_page: PageLeaseSnapshot, + stale_worker_id: []const u8, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + _ = graph_entry.index.completeGraphMetricBuildPageForAttempt( + metric_name, + stale_page.job_id, + phase, + stale_page.iteration, + stale_page.page_id, + stale_worker_id, + stale_page.attempt, + stale_page.total_units, + 0, + ) catch |err| switch (err) { + error.GraphMetricBuildPageNotLeased => return, + error.GraphMetricBuildPageNotFound => return, + else => return err, + }; + std.debug.print("expected stale metric page attempt completion to be rejected\n", .{}); + return error.GraphMetricPageRankProcessProofFailed; +} + +fn expectReclaimedMetricPageCompleted( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + stale_page: PageLeaseSnapshot, + reclaim_worker_id: []const u8, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const page = try graph_entry.index.graphMetricBuildPageSnapshotForTest( + metric_name, + stale_page.job_id, + phase, + stale_page.iteration, + stale_page.page_id, + ) orelse { + std.debug.print("expected reclaimed {s} page record to remain durable\n", .{metric_name}); + return error.GraphMetricProcessProofFailed; + }; + if (page.state != antfly.graph.GraphIndex.GraphMetricBuildPageState.complete or + page.worker_id_hash != identityHash(reclaim_worker_id) or + page.attempt <= stale_page.attempt or + page.completed_units != page.total_units) + { + std.debug.print("expected reclaimed {s} page to complete under replacement worker and newer attempt\n", .{metric_name}); + return error.GraphMetricProcessProofFailed; + } +} + +fn runCoordinatorRoleProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, +) !RoleRunSummary { + return try runCoordinatorRoleProcessAt(alloc, io, antfly_exe, db_path, owner_id, lease_ttl_ms, null); +} + +fn runCoordinatorRoleProcessAt( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: ?[]const u8, +) !RoleRunSummary { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "coordinator", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }; + if (test_now_ms) |now_ms| { + const argv_with_now = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "coordinator", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--test-now-ms", + now_ms, + }; + const summary = try runRoleProcess(alloc, io, argv_with_now[0..]); + try verifyRoleProcessTelemetry(summary, .coordinator, owner_id, 0, 0); + return summary; + } + const summary = try runRoleProcess(alloc, io, argv[0..]); + try verifyRoleProcessTelemetry(summary, .coordinator, owner_id, 0, 0); + return summary; +} + +fn runServiceCoordinatorRoleProcessAt( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, +) !RoleRunSummary { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--base-uri", + base_uri, + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "coordinator", + "--runtime-id", + runtime_id, + "--owner-id", + owner_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + "--test-now-ms", + test_now_ms, + }; + const summary = try runRoleProcess(alloc, io, argv[0..]); + try verifyServiceRoleProcessTelemetry(summary, .coordinator, runtime_id, owner_id, 0, 0); + return summary; +} + +fn runWorkerRoleProcessAt( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + worker_id: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, +) !RoleRunSummary { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "worker", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--worker-id", + worker_id, + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--test-now-ms", + test_now_ms, + }; + const summary = try runRoleProcess(alloc, io, argv[0..]); + try verifyRoleProcessTelemetry(summary, .worker, owner_id, identityHash(worker_id), 1); + return summary; +} + +fn runWorkerPoolRoleProcess( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + db_path: []const u8, + owner_id: []const u8, + lease_ttl_ms: []const u8, +) !RoleRunSummary { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--db-path", + db_path, + "--role", + "worker_pool", + "--runtime-id", + owner_id, + "--owner-id", + owner_id, + "--worker-ids", + "lease-proof-worker-a,lease-proof-worker-b", + "--lease-ttl-ms", + lease_ttl_ms, + "--ticks", + "4", + "--max-idle-ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + }; + const summary = try runRoleProcess(alloc, io, argv[0..]); + const worker_ids = [_][]const u8{ "lease-proof-worker-a", "lease-proof-worker-b" }; + try verifyRoleProcessTelemetry(summary, .worker_pool, owner_id, workerSetHash(worker_ids[0..]), worker_ids.len); + return summary; +} + +fn runServiceWorkerPoolRoleProcessAt( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + worker_ids_csv: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, +) !RoleRunSummary { + return runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + runtime_id, + owner_id, + worker_ids_csv, + lease_ttl_ms, + test_now_ms, + "2", + ); +} + +fn runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + runtime_id: []const u8, + owner_id: []const u8, + worker_ids_csv: []const u8, + lease_ttl_ms: []const u8, + test_now_ms: []const u8, + max_pages: []const u8, +) !RoleRunSummary { + const argv = [_][]const u8{ + antfly_exe, + "__graph-metric-maintenance", + "--base-uri", + base_uri, + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "worker_pool", + "--runtime-id", + runtime_id, + "--owner-id", + owner_id, + "--worker-ids", + worker_ids_csv, + "--lease-ttl-ms", + lease_ttl_ms, + "--coordinator-start-background-builds", + "false", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + max_pages, + "--test-now-ms", + test_now_ms, + }; + const summary = try runRoleProcess(alloc, io, argv[0..]); + const worker_ids = [_][]const u8{ "service-process-worker-a", "service-process-worker-b" }; + try verifyServiceRoleProcessTelemetry(summary, .worker_pool, runtime_id, owner_id, workerSetHash(worker_ids[0..]), worker_ids.len); + return summary; +} + +fn verifyRoleProcessTelemetry( + summary: RoleRunSummary, + role: RuntimeRole, + owner_id: []const u8, + worker_id_hash: u64, + worker_count: usize, +) !void { + const stats = summary.stats; + if (stats.role != role) return error.GraphMetricRoleProcessFailed; + const logical_owner_hash = identityHash(owner_id); + if (stats.runtime_id_hash != logical_owner_hash) return error.GraphMetricRoleProcessFailed; + if (stats.owner_id_hash == 0 or stats.owner_id_hash == logical_owner_hash) return error.GraphMetricRoleProcessFailed; + if (stats.lease_key_hash == 0) return error.GraphMetricRoleProcessFailed; + if (stats.worker_id_hash != worker_id_hash) return error.GraphMetricRoleProcessFailed; + if (stats.worker_count != worker_count) return error.GraphMetricRoleProcessFailed; + if (!stats.lease_owned) return error.GraphMetricRoleProcessFailed; + try verifyRoleProcessLeaseAccounting(stats); + if (stats.ticks_started == 0) return error.GraphMetricRoleProcessFailed; + if (stats.ticks_completed == 0) return error.GraphMetricRoleProcessFailed; + if (stats.error_ticks != 0) return error.GraphMetricRoleProcessFailed; + try verifyRoleProcessTickAccounting(stats); +} + +fn verifyServiceRoleProcessTelemetry( + summary: RoleRunSummary, + role: RuntimeRole, + runtime_id: []const u8, + owner_id: []const u8, + worker_id_hash: u64, + worker_count: usize, +) !void { + const stats = summary.stats; + if (stats.role != role) return error.GraphMetricRoleProcessFailed; + if (stats.runtime_id_hash != identityHash(runtime_id)) return error.GraphMetricRoleProcessFailed; + const logical_owner_hash = identityHash(owner_id); + if (stats.owner_id_hash == 0 or stats.owner_id_hash == logical_owner_hash) return error.GraphMetricRoleProcessFailed; + if (stats.lease_key_hash == 0) return error.GraphMetricRoleProcessFailed; + if (stats.worker_id_hash != worker_id_hash) return error.GraphMetricRoleProcessFailed; + if (stats.worker_count != worker_count) return error.GraphMetricRoleProcessFailed; + if (!stats.lease_owned) return error.GraphMetricRoleProcessFailed; + try verifyRoleProcessLeaseAccounting(stats); + if (stats.ticks_started == 0) return error.GraphMetricRoleProcessFailed; + if (stats.ticks_completed == 0) return error.GraphMetricRoleProcessFailed; + if (stats.error_ticks != 0) return error.GraphMetricRoleProcessFailed; + try verifyRoleProcessTickAccounting(stats); +} + +fn verifyRoleProcessLeaseAccounting(stats: RuntimeStats) !void { + if (stats.acquisition_count == 0 and stats.lease_acquire_failures == 0) { + return error.GraphMetricRoleProcessFailed; + } + if (stats.has_lease and stats.acquisition_count == 0) { + return error.GraphMetricRoleProcessFailed; + } +} + +fn verifyRoleProcessTickAccounting(stats: RuntimeStats) !void { + if (stats.ticks_completed > stats.ticks_started) return error.GraphMetricRoleProcessFailed; + const accounted_ticks = stats.durable_progress_ticks + stats.idle_ticks + stats.error_ticks; + if (accounted_ticks > stats.ticks_completed) return error.GraphMetricRoleProcessFailed; + const fenced_ticks = stats.lease_acquire_failures + stats.lost_leases; + if (accounted_ticks + fenced_ticks < stats.ticks_completed) return error.GraphMetricRoleProcessFailed; +} + +fn identityHash(value: []const u8) u64 { + if (value.len == 0) return 0; + return std.hash.Wyhash.hash(0, value); +} + +fn workerSetHash(worker_ids: []const []const u8) u64 { + var xor_hash: u64 = 0; + var sum_hash: u64 = 0; + for (worker_ids) |worker_id| { + const item_hash = identityHash(worker_id); + xor_hash ^= item_hash; + sum_hash +%= item_hash; + } + const fingerprint_words = [_]u64{ + @intCast(worker_ids.len), + xor_hash, + sum_hash, + }; + return std.hash.Wyhash.hash(0, std.mem.asBytes(&fingerprint_words)); +} + +/// Summary leaves/root and ordinal shards add bounded checkpoints within a +/// phase. Drive those through real service processes, stopping at exactly the +/// next coordinator barrier so callers can still assert the next phase. +fn drainServicePhaseRemainder( + alloc: std.mem.Allocator, + io: std.Io, + antfly_exe: []const u8, + base_uri: []const u8, + coordinator_runtime: []const u8, + worker_runtime: []const u8, + now_ms: *u64, +) !RoleRunSummary { + for (0..64) |_| { + const worker_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms.*}); + defer alloc.free(worker_now); + const worker = try runServiceWorkerPoolRoleProcessAtWithMaxPages( + alloc, + io, + antfly_exe, + base_uri, + worker_runtime, + "bounded-phase-worker", + "service-process-worker-a,service-process-worker-b", + "5000", + worker_now, + "4", + ); + now_ms.* += 1; + const coordinator_now = try std.fmt.allocPrint(alloc, "{d}", .{now_ms.*}); + defer alloc.free(coordinator_now); + const coordinator = try runServiceCoordinatorRoleProcessAt( + alloc, + io, + antfly_exe, + base_uri, + coordinator_runtime, + "bounded-phase-coordinator", + "5000", + coordinator_now, + ); + now_ms.* += 1; + if (coordinator.result.phases_advanced != 0) return coordinator; + if (!worker.durable_progressed and !coordinator.durable_progressed) + return error.GraphMetricProcessProofFailed; + } + return error.GraphMetricProcessProofFailed; +} + +fn runRoleProcess( + alloc: std.mem.Allocator, + io: std.Io, + argv: []const []const u8, +) !RoleRunSummary { + try verifyRoleProcessArgvScoped(argv); + const result = try std.process.run(alloc, io, .{ + .environ_map = child_environ, + .argv = argv, + .reserve_amount = 512, + }); + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + switch (result.term) { + .exited => |code| if (code != 0) { + std.debug.print( + "graph metric role process exited with code {d}\nstdout:\n{s}\nstderr:\n{s}\n", + .{ code, result.stdout, result.stderr }, + ); + return error.GraphMetricRoleProcessFailed; + }, + else => { + std.debug.print( + "graph metric role process terminated unexpectedly\nstdout:\n{s}\nstderr:\n{s}\n", + .{ result.stdout, result.stderr }, + ); + return error.GraphMetricRoleProcessFailed; + }, + } + try verifyRoleProcessJsonStats(alloc, result.stdout); + var parsed = try std.json.parseFromSlice(RoleRunSummary, alloc, result.stdout, .{ + .ignore_unknown_fields = true, + }); + defer parsed.deinit(); + return parsed.value; +} + +fn verifyRoleProcessJsonStats(alloc: std.mem.Allocator, stdout: []const u8) !void { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, stdout, .{}); + defer parsed.deinit(); + try verifyJsonNoRawOperationalFields(parsed.value, error.GraphMetricRoleProcessFailed); + const object = switch (parsed.value) { + .object => |object| object, + else => return error.GraphMetricRoleProcessFailed, + }; + const stats = switch (object.get("stats") orelse return error.GraphMetricRoleProcessFailed) { + .object => |stats| stats, + else => return error.GraphMetricRoleProcessFailed, + }; + _ = stats.get("durable_progress_ticks") orelse return error.GraphMetricRoleProcessFailed; + _ = stats.get("idle_ticks") orelse return error.GraphMetricRoleProcessFailed; + _ = stats.get("error_ticks") orelse return error.GraphMetricRoleProcessFailed; + switch (stats.get("last_error_name") orelse return error.GraphMetricRoleProcessFailed) { + .null => {}, + else => return error.GraphMetricRoleProcessFailed, + } +} + +fn verifyJsonNoRawOperationalFields(value: std.json.Value, comptime failure_error: anyerror) !void { + switch (value) { + .object => |object| { + var it = object.iterator(); + while (it.next()) |entry| { + if (roleProcessJsonFieldForbidden(entry.key_ptr.*)) { + std.debug.print("graph metric process summary leaked raw graph metric field {s}\n", .{entry.key_ptr.*}); + return failure_error; + } + try verifyJsonNoRawOperationalFields(entry.value_ptr.*, failure_error); + } + }, + .array => |array| { + for (array.items) |item| { + try verifyJsonNoRawOperationalFields(item, failure_error); + } + }, + else => {}, + } +} + +fn roleProcessJsonFieldForbidden(field: []const u8) bool { + const forbidden = [_][]const u8{ + "metric_name", + "metric_names", + "index_name", + "target_generation", + "building_generation", + "job_id", + "page_id", + "attempt", + "attempt_namespace", + "manifest_path", + "score_prefix", + "output_prefix", + "metric_config", + "metric_configs", + "config_fingerprint", + "db_path", + "base_uri", + "process_id", + "pid", + "summary_file", + "writer_guard", + }; + for (forbidden) |item| { + if (std.mem.eql(u8, field, item)) return true; + } + return false; +} + +fn verifyRoleProcessArgvScoped(argv: []const []const u8) !void { + if (argv.len < 4) return error.GraphMetricRoleProcessFailed; + if (!std.mem.eql(u8, argv[1], "__graph-metric-maintenance")) return error.GraphMetricRoleProcessFailed; + try verifyRoleProcessArgvAllowlist(argv); + const has_db_path = processArgvContains(argv, "--db-path"); + const has_base_uri = processArgvContains(argv, "--base-uri") or processArgvContains(argv, "--service-base-uri"); + const has_group_id = processArgvContains(argv, "--group-id"); + const has_table_name = processArgvContains(argv, "--table-name"); + if (has_db_path and (has_base_uri or has_group_id or has_table_name)) return error.GraphMetricRoleProcessFailed; + if (!has_db_path and !(has_base_uri and has_group_id and has_table_name)) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--role")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--runtime-id")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--owner-id")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--lease-ttl-ms")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--ticks")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--max-rounds")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--max-metrics")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--max-pages")) return error.GraphMetricRoleProcessFailed; + + const forbidden = [_][]const u8{ + "--index", + "--index-name", + "--metric", + "--metric-name", + "--metric-config", + "--target-generation", + "--job-id", + "--page-id", + "--phase", + "--summary-file", + "--local-db-writer-lock", + }; + if (processArgvContainsAny(argv, forbidden[0..])) return error.GraphMetricRoleProcessFailed; + + const role = processArgvValue(argv, "--role") orelse return error.GraphMetricRoleProcessFailed; + if (std.mem.eql(u8, role, "coordinator")) { + if (processArgvContains(argv, "--worker-id")) return error.GraphMetricRoleProcessFailed; + if (processArgvContains(argv, "--worker-ids")) return error.GraphMetricRoleProcessFailed; + } else if (std.mem.eql(u8, role, "worker")) { + if (!processArgvContains(argv, "--worker-id")) return error.GraphMetricRoleProcessFailed; + if (processArgvContains(argv, "--worker-ids")) return error.GraphMetricRoleProcessFailed; + } else if (std.mem.eql(u8, role, "worker_pool")) { + if (processArgvContains(argv, "--worker-id")) return error.GraphMetricRoleProcessFailed; + if (!processArgvContains(argv, "--worker-ids")) return error.GraphMetricRoleProcessFailed; + } else { + return error.GraphMetricRoleProcessFailed; + } +} + +fn verifyRoleProcessArgvAllowlist(argv: []const []const u8) !void { + var i: usize = 2; + while (i < argv.len) : (i += 2) { + const flag = argv[i]; + if (!std.mem.startsWith(u8, flag, "--")) return error.GraphMetricRoleProcessFailed; + if (!roleProcessArgvFlagAllowed(flag)) return error.GraphMetricRoleProcessFailed; + if (i + 1 >= argv.len) return error.GraphMetricRoleProcessFailed; + if (std.mem.startsWith(u8, argv[i + 1], "--")) return error.GraphMetricRoleProcessFailed; + } +} + +fn roleProcessArgvFlagAllowed(flag: []const u8) bool { + const allowed = [_][]const u8{ + "--db-path", + "--base-uri", + "--service-base-uri", + "--group-id", + "--table-name", + "--role", + "--runtime-id", + "--owner-id", + "--worker-id", + "--worker-ids", + "--lease-ttl-ms", + "--coordinator-start-background-builds", + "--ticks", + "--max-idle-ticks", + "--max-rounds", + "--max-metrics", + "--max-pages", + "--test-now-ms", + "--test-ready-file", + "--test-hold-after-run-ms", + }; + for (allowed) |allowed_flag| { + if (std.mem.eql(u8, flag, allowed_flag)) return true; + } + return false; +} + +fn verifyRoleProcessArgvPreflightSelfTest() !void { + try verifyRoleProcessArgvScoped(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "coordinator", + "--runtime-id", + "coordinator-owner", + "--owner-id", + "coordinator-owner", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try verifyRoleProcessArgvScoped(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--test-now-ms", + "1000", + }); + try verifyRoleProcessArgvScoped(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker_pool", + "--runtime-id", + "pool-owner", + "--owner-id", + "pool-owner", + "--worker-ids", + "worker-a,worker-b", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-idle-ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + }); + try verifyRoleProcessArgvScoped(&.{ + "antfly", + "__graph-metric-maintenance", + "--base-uri", + "http://127.0.0.1:8080", + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "coordinator", + "--runtime-id", + "service-coordinator-owner", + "--owner-id", + "service-coordinator-owner", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try verifyRoleProcessArgvScoped(&.{ + "antfly", + "__graph-metric-maintenance", + "--base-uri", + "http://127.0.0.1:8080", + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "worker_pool", + "--runtime-id", + "service-pool-owner", + "--owner-id", + "service-pool-owner", + "--worker-ids", + "worker-a,worker-b", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-idle-ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + }); + + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "coordinator", + "--runtime-id", + "coordinator-owner", + "--owner-id", + "coordinator-owner", + "--worker-ids", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker_pool", + "--runtime-id", + "pool-owner", + "--owner-id", + "pool-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "2", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--base-uri", + "http://127.0.0.1:8080", + "--group-id", + "7", + "--table-name", + "docs", + "--role", + "coordinator", + "--runtime-id", + "coordinator-owner", + "--owner-id", + "coordinator-owner", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--base-uri", + "http://127.0.0.1:8080", + "--group-id", + "7", + "--role", + "coordinator", + "--runtime-id", + "coordinator-owner", + "--owner-id", + "coordinator-owner", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--metric-name", + "pagerank", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--local-db-writer-lock", + "true", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + "1", + "--unexpected-owner-input", + "value", + }); + try expectRoleProcessArgvRejected(&.{ + "antfly", + "__graph-metric-maintenance", + "--db-path", + "/tmp/db", + "--role", + "worker", + "--runtime-id", + "worker-owner", + "--owner-id", + "worker-owner", + "--worker-id", + "worker-a", + "--lease-ttl-ms", + "5000", + "--ticks", + "1", + "--max-rounds", + "1", + "--max-metrics", + "4", + "--max-pages", + }); +} + +fn expectRoleProcessArgvRejected(argv: []const []const u8) !void { + verifyRoleProcessArgvScoped(argv) catch return; + return error.GraphMetricProcessProofFailed; +} + +fn processArgvContains(argv: []const []const u8, needle: []const u8) bool { + for (argv) |arg| { + if (std.mem.eql(u8, arg, needle)) return true; + } + return false; +} + +fn processArgvContainsAny(argv: []const []const u8, needles: []const []const u8) bool { + for (needles) |needle| { + if (processArgvContains(argv, needle)) return true; + } + return false; +} + +fn processArgvValue(argv: []const []const u8, flag: []const u8) ?[]const u8 { + for (argv, 0..) |arg, i| { + if (std.mem.eql(u8, arg, flag)) { + if (i + 1 >= argv.len) return null; + return argv[i + 1]; + } + } + return null; +} + +fn verifyDegreeFresh(alloc: std.mem.Allocator, db_path: []const u8, target_generation: u64) !void { + return verifyMetricFresh(alloc, db_path, "degree", target_generation); +} + +fn verifyMetricFresh( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.fresh) { + std.debug.print("expected fresh graph metric, got {}\n", .{status.state}); + return error.GraphMetricNotFresh; + } + if (status.published_generation != target_generation) { + std.debug.print( + "expected published generation {d}, got {d}\n", + .{ target_generation, status.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } +} + +fn verifyHitsFresh(alloc: std.mem.Allocator, db_path: []const u8, target_generation: u64) !void { + try verifyMetricFresh(alloc, db_path, "hits_authority", target_generation); + try verifyMetricFresh(alloc, db_path, "hits_hub", target_generation); + + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const authority_top = try graph_entry.index.graphMetricTopK("hits_authority", 3); + defer { + for (authority_top) |*score| score.deinit(alloc); + alloc.free(authority_top); + } + const hub_top = try graph_entry.index.graphMetricTopK("hits_hub", 3); + defer { + for (hub_top) |*score| score.deinit(alloc); + alloc.free(hub_top); + } + if (authority_top.len == 0 or hub_top.len == 0) { + std.debug.print("expected paired HITS top-k scores after process supervisor publish\n", .{}); + return error.GraphMetricProcessProofFailed; + } +} + +fn verifyPageRankFixedIterationMetadata( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, + expected_iterations_completed: u32, +) !void { + return verifyFixedIterationMetadata( + alloc, + db_path, + "pagerank", + target_generation, + expected_iterations_completed, + ); +} + +fn verifyFixedIterationMetadata( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + target_generation: u64, + expected_iterations_completed: u32, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.fresh) { + std.debug.print("expected fresh {s} metric, got {}\n", .{ metric_name, status.state }); + return error.GraphMetricNotFresh; + } + if (status.published_generation != target_generation) { + std.debug.print( + "expected {s} published generation {d}, got {d}\n", + .{ metric_name, target_generation, status.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } + if (status.converged) { + std.debug.print("expected bounded {s} process publish to report converged=false\n", .{metric_name}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (status.iterations_completed != expected_iterations_completed) { + std.debug.print( + "expected {s} iterations_completed {d}, got {d}\n", + .{ metric_name, expected_iterations_completed, status.iterations_completed }, + ); + return error.GraphMetricPageRankProcessProofFailed; + } + if (!std.math.isFinite(status.delta) or status.delta <= 0.0) { + std.debug.print("expected positive finite {s} fixed-iteration delta, got {d}\n", .{ metric_name, status.delta }); + return error.GraphMetricPageRankProcessProofFailed; + } +} + +fn verifyHitsFixedIterationMetadata( + alloc: std.mem.Allocator, + db_path: []const u8, + target_generation: u64, + expected_iterations_completed: u32, +) !void { + try verifyFixedIterationMetadata( + alloc, + db_path, + "hits_authority", + target_generation, + expected_iterations_completed, + ); + try verifyFixedIterationMetadata( + alloc, + db_path, + "hits_hub", + target_generation, + expected_iterations_completed, + ); +} + +fn verifyMetricFailedPreservesPublished( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + published_generation: u64, + expected_last_error: []const u8, +) !void { + return verifyMetricFailedPreservesPublishedAtPhase( + alloc, + db_path, + metric_name, + published_generation, + expected_last_error, + .publish_generation, + null, + ); +} + +fn verifyMetricFailedPreservesPublishedAtPhase( + alloc: std.mem.Allocator, + db_path: []const u8, + metric_name: []const u8, + published_generation: u64, + expected_last_error: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_iteration: ?u32, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.state != antfly.graph.GraphIndex.GraphMetricState.failed) { + std.debug.print("expected failed graph metric, got {}\n", .{status.state}); + return error.GraphMetricPageRankProcessProofFailed; + } + if (status.published_generation != published_generation) { + std.debug.print( + "expected failed metric to preserve published generation {d}, got {d}\n", + .{ published_generation, status.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } + if (!std.mem.eql(u8, status.last_error, expected_last_error)) { + std.debug.print("expected last error {s}, got {s}\n", .{ expected_last_error, status.last_error }); + return error.GraphMetricPageRankProcessProofFailed; + } + if (status.recent_failures.len == 0 or status.recent_failures[0].phase != expected_phase) { + std.debug.print("expected retained {s} failure diagnostics for phase {}\n", .{ metric_name, expected_phase }); + return error.GraphMetricPageRankProcessProofFailed; + } + if (expected_iteration) |iteration| { + if (status.recent_failures[0].iteration != iteration) { + std.debug.print("expected retained {s} failure diagnostics for phase {} iteration {d}\n", .{ metric_name, expected_phase, iteration }); + return error.GraphMetricPageRankProcessProofFailed; + } + } +} + +fn verifyHitsFailedPreservesPublished( + alloc: std.mem.Allocator, + db_path: []const u8, + published_generation: u64, + expected_last_error: []const u8, +) !void { + return verifyHitsFailedPreservesPublishedAtPhase( + alloc, + db_path, + published_generation, + expected_last_error, + .publish_generation, + 0, + ); +} + +fn verifyHitsFailedPreservesPublishedAtPhase( + alloc: std.mem.Allocator, + db_path: []const u8, + published_generation: u64, + expected_last_error: []const u8, + expected_phase: antfly.graph.GraphIndex.GraphMetricBuildPhase, + expected_iteration: u32, +) !void { + var db = try antfly.db.DB.open(alloc, db_path, .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state != antfly.graph.GraphIndex.GraphMetricState.failed or + hub.state != antfly.graph.GraphIndex.GraphMetricState.failed) + { + std.debug.print("expected failed HITS pair, got authority {} hub {}\n", .{ authority.state, hub.state }); + return error.GraphMetricProcessProofFailed; + } + if (authority.published_generation != published_generation or hub.published_generation != published_generation) { + std.debug.print( + "expected failed HITS pair to preserve published generation {d}, got authority {d} hub {d}\n", + .{ published_generation, authority.published_generation, hub.published_generation }, + ); + return error.GraphMetricGenerationMismatch; + } + if (!std.mem.eql(u8, authority.last_error, expected_last_error) or !std.mem.eql(u8, hub.last_error, expected_last_error)) { + std.debug.print( + "expected HITS pair last error {s}, got authority {s} hub {s}\n", + .{ expected_last_error, authority.last_error, hub.last_error }, + ); + return error.GraphMetricProcessProofFailed; + } + if (authority.recent_failures.len == 0 or hub.recent_failures.len == 0 or + authority.recent_failures[0].phase != expected_phase or + hub.recent_failures[0].phase != expected_phase or + authority.recent_failures[0].iteration != expected_iteration or + hub.recent_failures[0].iteration != expected_iteration) + { + std.debug.print("expected retained paired HITS failure diagnostics for phase {} iteration {d}\n", .{ expected_phase, expected_iteration }); + return error.GraphMetricProcessProofFailed; + } + + const authority_top = try graph_entry.index.graphMetricTopK("hits_authority", 3); + defer { + for (authority_top) |*score| score.deinit(alloc); + alloc.free(authority_top); + } + const hub_top = try graph_entry.index.graphMetricTopK("hits_hub", 3); + defer { + for (hub_top) |*score| score.deinit(alloc); + alloc.free(hub_top); + } + if (authority_top.len == 0 or hub_top.len == 0) { + std.debug.print("expected failed HITS pair to keep prior published top-k output visible\n", .{}); + return error.GraphMetricProcessProofFailed; + } + + var direct_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .published, + }, + }, + }, + .limit = 0, + }); + defer direct_result.deinit(); + if (direct_result.graph_metric_results.len != 2) { + std.debug.print("expected failed HITS direct query to return two graph metric results\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (direct_result.graph_metric_results) |result| { + if (result.status.state != antfly.graph.GraphIndex.GraphMetricState.failed or + result.status.published_generation != published_generation) + { + std.debug.print( + "expected failed HITS direct result to preserve generation {d}, got state {} generation {d}\n", + .{ published_generation, result.status.state, result.status.published_generation }, + ); + return error.GraphMetricProcessProofFailed; + } + if (result.scores.len == 0) { + std.debug.print("expected failed HITS direct result to keep prior scores visible\n", .{}); + return error.GraphMetricProcessProofFailed; + } + } + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .published }, + .{ .name = "hits_hub", .freshness = .published }, + }; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:hub-a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1, .max_results = 10 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_result.deinit(); + if (traversal_result.graph_results.len != 1 or traversal_result.graph_results[0].nodes.len != 1) { + std.debug.print("expected failed HITS traversal query to return one graph result\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const traversal = traversal_result.graph_results[0]; + if (!std.mem.eql(u8, traversal.nodes[0].key, "doc:authority") or traversal.nodes[0].metrics.len != 2) { + std.debug.print("expected failed HITS traversal to project prior authority and hub scores\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.nodes[0].metrics) |metric| { + if (metric.score == null) { + std.debug.print("expected failed HITS traversal metric {s} to keep prior score visible\n", .{metric.name}); + return error.GraphMetricProcessProofFailed; + } + } + if (traversal.metric_status.len != 2) { + std.debug.print("expected failed HITS traversal to return two metric statuses\n", .{}); + return error.GraphMetricProcessProofFailed; + } + for (traversal.metric_status) |status| { + if (status.state != antfly.graph.GraphIndex.GraphMetricState.failed or + status.published_generation != published_generation) + { + std.debug.print( + "expected failed HITS traversal status to preserve generation {d}, got state {} generation {d}\n", + .{ published_generation, status.state, status.published_generation }, + ); + return error.GraphMetricProcessProofFailed; + } + } + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + var rerank_result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .published, + .base_weight = 0.0, + .weight = 1.0, + .missing_score = -1.0, + }, + .limit = 3, + .include_stored = false, + }); + defer rerank_result.deinit(); + if (rerank_result.hits.len == 0) { + std.debug.print("expected failed HITS rerank to return hits from prior generation\n", .{}); + return error.GraphMetricProcessProofFailed; + } + const rerank_status = rerank_result.graph_metric_rerank_status orelse { + std.debug.print("expected failed HITS rerank status\n", .{}); + return error.GraphMetricProcessProofFailed; + }; + if (rerank_status.state != antfly.graph.GraphIndex.GraphMetricState.failed or + rerank_status.published_generation != published_generation) + { + std.debug.print( + "expected failed HITS rerank status to preserve generation {d}, got state {} generation {d}\n", + .{ published_generation, rerank_status.state, rerank_status.published_generation }, + ); + return error.GraphMetricProcessProofFailed; + } + for (rerank_result.hits) |hit| { + const details = hit.score_details orelse { + std.debug.print("expected failed HITS reranked hit score details for {s}\n", .{hit.id}); + return error.GraphMetricProcessProofFailed; + }; + if (details.published_generation != published_generation) { + std.debug.print("expected failed HITS reranked hit {s} to use generation {d}, got {d}\n", .{ hit.id, published_generation, details.published_generation }); + return error.GraphMetricGenerationMismatch; + } + } + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 3, + .include_stored = false, + })); +} diff --git a/zig/pkg/antfly/src/cmd/mod.zig b/zig/pkg/antfly/src/cmd/mod.zig index 521e162ee8..eef8308ca2 100644 --- a/zig/pkg/antfly/src/cmd/mod.zig +++ b/zig/pkg/antfly/src/cmd/mod.zig @@ -13,6 +13,7 @@ // limitations. pub const data = @import("data.zig"); +pub const graph_metric_maintenance = @import("graph_metric_maintenance.zig"); pub const metadata = @import("metadata.zig"); pub const serverless = @import("serverless.zig"); pub const serverless_api = @import("serverless_api.zig"); @@ -27,6 +28,7 @@ pub const lite = @import("lite.zig"); test "cmd module compiles" { _ = data; + _ = graph_metric_maintenance; _ = metadata; _ = serverless; _ = serverless_api; diff --git a/zig/pkg/antfly/src/cmd_graph_metric_maintenance_test_root.zig b/zig/pkg/antfly/src/cmd_graph_metric_maintenance_test_root.zig new file mode 100644 index 0000000000..4fb8227a69 --- /dev/null +++ b/zig/pkg/antfly/src/cmd_graph_metric_maintenance_test_root.zig @@ -0,0 +1,19 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const graph_metric_maintenance = @import("cmd/graph_metric_maintenance.zig"); + +test { + _ = graph_metric_maintenance; +} diff --git a/zig/pkg/antfly/src/common/cancellation.zig b/zig/pkg/antfly/src/common/cancellation.zig index 4e9c1bd1fd..74519f3522 100644 --- a/zig/pkg/antfly/src/common/cancellation.zig +++ b/zig/pkg/antfly/src/common/cancellation.zig @@ -1,5 +1,16 @@ // Copyright 2026 Antfly, Inc. -// SPDX-License-Identifier: Elastic-2.0 +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. //! Transport-neutral borrowed cancellation contract. //! @@ -12,6 +23,10 @@ const std = @import("std"); pub const CancellationToken = struct { ptr: ?*const anyopaque = null, is_cancelled_fn: ?*const fn (*const anyopaque) bool = null, + /// Optional fallible checkpoint for scoped execution (for example a + /// renewable publication lease). This is authoritative when supplied; + /// the boolean callback remains available to transport-only adapters. + check_fn: ?*const fn (*const anyopaque) anyerror!void = null, pub const none: CancellationToken = .{}; @@ -29,12 +44,19 @@ pub const CancellationToken = struct { pub fn isCancelled(self: CancellationToken) bool { const ptr = self.ptr orelse return false; + if (self.check_fn) |check_fn| { + check_fn(ptr) catch return true; + return false; + } const callback = self.is_cancelled_fn orelse return false; return callback(ptr); } pub fn check(self: CancellationToken) !void { - if (self.isCancelled()) return error.Canceled; + if (self.ptr) |ptr| if (self.check_fn) |check_fn| return check_fn(ptr); + if (self.ptr) |ptr| if (self.is_cancelled_fn) |callback| { + if (callback(ptr)) return error.Canceled; + }; } }; diff --git a/zig/pkg/antfly/src/completion.zig b/zig/pkg/antfly/src/completion.zig index afeda6762e..ecec994c11 100644 --- a/zig/pkg/antfly/src/completion.zig +++ b/zig/pkg/antfly/src/completion.zig @@ -11,6 +11,7 @@ const std = @import("std"); pub const Route = enum { cli, data, + graph_metric_maintenance, ha, inference, metadata, @@ -27,6 +28,7 @@ pub const Command = struct { description: []const u8, route: Route, subcommands: []const []const u8 = &.{}, + hidden: bool = false, }; const table_subcommands = [_][]const u8{ "create", "drop", "list", "get" }; @@ -56,6 +58,8 @@ const completion_subcommands = [_][]const u8{ "bash", "zsh", "fish" }; /// completions behind. pub const commands = [_]Command{ .{ .name = "data", .description = "Run a data node", .route = .data }, + .{ .name = "graph-metric-maintenance", .description = "Run resumable graph metric maintenance", .route = .graph_metric_maintenance }, + .{ .name = "__graph-metric-maintenance", .description = "Run internal graph metric maintenance", .route = .graph_metric_maintenance, .hidden = true }, .{ .name = "metadata", .description = "Run a metadata node", .route = .metadata }, .{ .name = "standalone", .description = "Run a standalone server", .route = .standalone }, .{ .name = "swarm", .description = "Run a standalone server (legacy alias)", .route = .standalone }, @@ -111,9 +115,12 @@ pub fn write(shell: Shell, writer: *std.Io.Writer) !void { } fn writeCommandNames(writer: *std.Io.Writer, command_list: []const Command) !void { - for (command_list, 0..) |command, index| { - if (index != 0) try writer.writeByte(' '); + var written: usize = 0; + for (command_list) |command| { + if (command.hidden) continue; + if (written != 0) try writer.writeByte(' '); try writer.writeAll(command.name); + written += 1; } } @@ -143,6 +150,7 @@ fn writeBash(writer: *std.Io.Writer) !void { ); try writer.writeByte('\n'); for (commands) |command| { + if (command.hidden) continue; if (command.subcommands.len == 0) continue; try writer.print(" {s}) COMPREPLY=($(compgen -W \"", .{command.name}); try writeSubcommandNames(writer, command.subcommands); @@ -167,6 +175,7 @@ fn writeZsh(writer: *std.Io.Writer) !void { ); try writer.writeByte('\n'); for (commands) |command| { + if (command.hidden) continue; try writer.print(" '{s}:{s}'\n", .{ command.name, command.description }); } try writer.writeAll( @@ -180,6 +189,7 @@ fn writeZsh(writer: *std.Io.Writer) !void { ); try writer.writeByte('\n'); for (commands) |command| { + if (command.hidden) continue; if (command.subcommands.len == 0) continue; try writer.print(" {s}) subcommands=(", .{command.name}); try writeSubcommandNames(writer, command.subcommands); @@ -204,6 +214,7 @@ fn writeFish(writer: *std.Io.Writer) !void { \\ ); for (commands) |command| { + if (command.hidden) continue; try writer.print("complete -c antfly -n '__fish_use_subcommand' -a '{s}' -d '{s}'\n", .{ command.name, command.description }); if (command.subcommands.len == 0) continue; try writer.print("complete -c antfly -n '__fish_seen_subcommand_from {s}' -a '", .{command.name}); @@ -216,9 +227,20 @@ test "command table drives routes and completion entries" { try std.testing.expectEqual(Route.standalone, findCommand("swarm").?.route); try std.testing.expectEqual(Route.cli, findCommand("table").?.route); try std.testing.expectEqual(Route.completion, findCommand("completion").?.route); + try std.testing.expectEqual(Route.graph_metric_maintenance, findCommand("__graph-metric-maintenance").?.route); try std.testing.expect(findCommand("termite") == null); } +test "completion output omits hidden commands" { + inline for (std.meta.tags(Shell)) |shell| { + var output: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer output.deinit(); + try write(shell, &output.writer); + try std.testing.expect(std.mem.indexOf(u8, output.written(), "__graph-metric-maintenance") == null); + try std.testing.expect(std.mem.indexOf(u8, output.written(), "graph-metric-maintenance") != null); + } +} + test "zsh completion contains nested inference and completion commands" { var output: std.Io.Writer.Allocating = .init(std.testing.allocator); defer output.deinit(); diff --git a/zig/pkg/antfly/src/db_test_root.zig b/zig/pkg/antfly/src/db_test_root.zig index bcded3a50e..128ed6f958 100644 --- a/zig/pkg/antfly/src/db_test_root.zig +++ b/zig/pkg/antfly/src/db_test_root.zig @@ -13,7 +13,9 @@ // limitations. test { + _ = @import("graph/query.zig"); _ = @import("storage/db/db.zig"); + _ = @import("storage/db/graph_runtime.zig"); _ = @import("storage/db_split_vopr.zig"); _ = @import("storage/db/promotion_runtime.zig"); _ = @import("storage/db/resolution_runtime.zig"); diff --git a/zig/pkg/antfly/src/graph/adjacency.zig b/zig/pkg/antfly/src/graph/adjacency.zig new file mode 100644 index 0000000000..6a9b6c244d --- /dev/null +++ b/zig/pkg/antfly/src/graph/adjacency.zig @@ -0,0 +1,93 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Durable packing state for one output-vector chunk. Partial tiles and input +//! cursor commit together. A completion receipt selects one immutable attempt. +const std = @import("std"); +const ordinal = @import("ordinal.zig"); +pub const tile_entries = 256; + +pub const Receipt = struct { + attempt: u64, + edges: u64, + pub fn blocks(self: Receipt) u64 { + return self.edges / tile_entries + @intFromBool(self.edges % tile_entries != 0); + } + pub fn encode(self: Receipt) [20]u8 { + var bytes: [20]u8 = undefined; + @memcpy(bytes[0..4], "GAR1"); + std.mem.writeInt(u64, bytes[4..12], self.attempt, .little); + std.mem.writeInt(u64, bytes[12..20], self.edges, .little); + return bytes; + } + pub fn decode(bytes: []const u8) !Receipt { + if (bytes.len != 20 or !std.mem.eql(u8, bytes[0..4], "GAR1")) return error.InvalidGraphMetricBuildManifest; + const result = Receipt{ .attempt = std.mem.readInt(u64, bytes[4..12], .little), .edges = std.mem.readInt(u64, bytes[12..20], .little) }; + if (result.attempt == 0) return error.InvalidGraphMetricBuildManifest; + return result; + } +}; + +pub const State = struct { + attempt: u64, + blocks: u64 = 0, + count: usize = 0, + pending: [tile_entries]ordinal.Edge = undefined, + /// Borrows the decoded state bytes. + cursor: []const u8 = "", + + pub fn encode(self: *const State, alloc: std.mem.Allocator) ![]u8 { + if (self.attempt == 0 or self.count >= tile_entries or self.cursor.len > std.math.maxInt(u32)) return error.InvalidGraphMetricBuildManifest; + const bytes = try alloc.alloc(u8, 26 + self.count * 16 + self.cursor.len); + @memcpy(bytes[0..4], "GAP1"); + std.mem.writeInt(u64, bytes[4..12], self.attempt, .little); + std.mem.writeInt(u64, bytes[12..20], self.blocks, .little); + std.mem.writeInt(u16, bytes[20..22], @intCast(self.count), .little); + std.mem.writeInt(u32, bytes[22..26], @intCast(self.cursor.len), .little); + for (self.pending[0..self.count], 0..) |edge, i| { + std.mem.writeInt(u64, bytes[26 + i * 16 ..][0..8], edge.source, .little); + std.mem.writeInt(u64, bytes[34 + i * 16 ..][0..8], edge.target, .little); + } + @memcpy(bytes[26 + self.count * 16 ..], self.cursor); + return bytes; + } + + pub fn decode(bytes: []const u8) !State { + if (bytes.len < 26 or !std.mem.eql(u8, bytes[0..4], "GAP1")) return error.InvalidGraphMetricBuildManifest; + var state = State{ .attempt = std.mem.readInt(u64, bytes[4..12], .little), .blocks = std.mem.readInt(u64, bytes[12..20], .little), .count = std.mem.readInt(u16, bytes[20..22], .little) }; + const cursor_len = std.mem.readInt(u32, bytes[22..26], .little); + if (state.attempt == 0 or state.count >= tile_entries or bytes.len != 26 + state.count * 16 + @as(usize, cursor_len)) return error.InvalidGraphMetricBuildManifest; + for (state.pending[0..state.count], 0..) |*edge, i| { + edge.* = .{ .source = std.mem.readInt(u64, bytes[26 + i * 16 ..][0..8], .little), .target = std.mem.readInt(u64, bytes[34 + i * 16 ..][0..8], .little) }; + if (edge.source == 0 or edge.target == 0) return error.InvalidGraphMetricBuildManifest; + } + state.cursor = bytes[26 + state.count * 16 ..]; + return state; + } +}; + +test "ordinal blocks adjacency packing state and receipts reject malformed progress" { + var state = State{ .attempt = 2, .blocks = 3, .count = 1, .cursor = "checkpoint" }; + state.pending[0] = .{ .source = 4, .target = 5 }; + const raw = try state.encode(std.testing.allocator); + defer std.testing.allocator.free(raw); + const decoded = try State.decode(raw); + try std.testing.expectEqual(@as(u64, 3), decoded.blocks); + try std.testing.expectEqualStrings("checkpoint", decoded.cursor); + try std.testing.expectEqual(state.pending[0], decoded.pending[0]); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, State.decode(raw[0 .. raw.len - 1])); + const receipt = Receipt{ .attempt = 2, .edges = 769 }; + const encoded = receipt.encode(); + try std.testing.expectEqual(@as(u64, 4), (try Receipt.decode(&encoded)).blocks()); +} diff --git a/zig/pkg/antfly/src/graph/graph.zig b/zig/pkg/antfly/src/graph/graph.zig index dec83e94c0..56a274c662 100644 --- a/zig/pkg/antfly/src/graph/graph.zig +++ b/zig/pkg/antfly/src/graph/graph.zig @@ -26,12 +26,22 @@ const Allocator = std.mem.Allocator; const platform_time = @import("antfly_platform").time; const edge_type_mod = @import("edge_type.zig"); const edge_weight = @import("edge_weight.zig"); +const metric_kernels = @import("metrics.zig"); +pub const score_read = @import("score_read.zig"); +pub const vector_chunk = @import("vector_chunk.zig"); +const partition_census = @import("partition_census.zig"); +const membership = @import("membership.zig"); +const ordinal_blocks = @import("ordinal.zig"); +const adjacency_blocks = @import("adjacency.zig"); +const topology_owner = @import("topology_owner.zig"); +const typed_edges = @import("typed_edges.zig"); const backend_erased = @import("../storage/backend_erased.zig"); const backend_scan = @import("../storage/backend_scan.zig"); const docstore = @import("../storage/docstore.zig"); const internal_keys = @import("../storage/internal_keys.zig"); const backfill_state_mod = @import("../storage/db/backfill_state.zig"); -const supports_native_reverse_lmdb = builtin.os.tag != .freestanding and build_options.lmdb_enabled; +const supports_native_reverse_lmdb = builtin.is_test or + (builtin.os.tag != .freestanding and build_options.lmdb_enabled); const lmdb_backend = if (supports_native_reverse_lmdb) @import("../storage/lmdb_backend.zig") else struct { pub const Backend = struct { pub fn close(_: *@This()) void {} @@ -316,6 +326,122 @@ fn parseReverseEdgeKeyAlloc(alloc: Allocator, key: []const u8) !?ParsedGraphEdge }; } +/// Checks the structural edge-key prefix and index component without decoding +/// or allocating the document and edge components. Planning scans can visit +/// millions of keys, so ownership should only be materialized for the handful +/// of persisted partition boundaries. +fn graphIndexEdgeKeyMatchesIndex(key: []const u8, index_name: []const u8) bool { + if (!internal_keys.isInternalUserKey(key)) return false; + const doc_term = internal_keys.findComponentTerminator(key, 1) orelse return false; + var pos = doc_term + 2; + if (pos >= key.len or key[pos] != internal_keys.artifact_kind) return false; + pos += 1; + if (!internal_keys.componentEquals(key, pos, graph_index_edge_artifact_type)) return false; + pos = (internal_keys.findComponentTerminator(key, pos) orelse return false) + 2; + if (!internal_keys.componentEquals(key, pos, index_name)) return false; + pos = (internal_keys.findComponentTerminator(key, pos) orelse return false) + 2; + return pos < key.len and key[pos] == internal_keys.graph_edge_record_kind; +} + +/// Metric state can dwarf the edge set. Never enumerate it while finding the +/// end of an edge partition, including the final unbounded partition. +fn graphMetricSkipMetadata(cur: anytype, entry: anytype) !@TypeOf(entry) { + if (entry) |value| if (std.mem.startsWith(u8, value.key, "meta:")) + return try cur.seekAtOrAfter("meta;"); + return entry; +} + +test "graph metric edge scan skips an arbitrarily large metadata tail with one seek" { + const Entry = struct { key: []const u8 }; + const Cursor = struct { + seeks: usize = 0, + pub fn seekAtOrAfter(self: *@This(), key: []const u8) !?Entry { + try std.testing.expectEqualStrings("meta;", key); + self.seeks += 1; + return .{ .key = "next-record" }; + } + }; + var cur = Cursor{}; + const skipped = try graphMetricSkipMetadata(&cur, @as(?Entry, .{ .key = "meta:graph-metric/vector/0" })); + try std.testing.expectEqualStrings("next-record", skipped.?.key); + try std.testing.expectEqual(@as(usize, 1), cur.seeks); + const edge = try graphMetricSkipMetadata(&cur, @as(?Entry, .{ .key = "edge" })); + try std.testing.expectEqualStrings("edge", edge.?.key); + try std.testing.expectEqual(@as(usize, 1), cur.seeks); + try std.testing.expect((try graphMetricSkipMetadata(&cur, @as(?Entry, null))) == null); +} + +const DecodedGraphKeyComponent = struct { + bytes: []const u8, + owned: bool = false, + + fn deinit(self: *@This(), alloc: Allocator) void { + if (self.owned) alloc.free(self.bytes); + self.* = undefined; + } +}; + +const ParsedReverseEdgeKeyView = struct { + source: DecodedGraphKeyComponent, + edge_type: DecodedGraphKeyComponent, + target: DecodedGraphKeyComponent, + + fn deinit(self: *@This(), alloc: Allocator) void { + self.source.deinit(alloc); + self.edge_type.deinit(alloc); + self.target.deinit(alloc); + self.* = undefined; + } +}; + +fn decodeGraphKeyComponent(alloc: Allocator, encoded: []const u8) !DecodedGraphKeyComponent { + if (try internal_keys.decodeBodyView(encoded)) |view| return .{ .bytes = view }; + return .{ .bytes = try internal_keys.decodeBodyAlloc(alloc, encoded), .owned = true }; +} + +/// Parses the reverse-edge fields needed by metric kernels. Ordinary UTF-8 +/// identifiers borrow directly from the cursor key; only identifiers containing +/// escaped NUL bytes require allocation. +fn parseMetricReverseEdgeKeyView( + alloc: Allocator, + key: []const u8, + index_name: []const u8, +) !?ParsedReverseEdgeKeyView { + if (!internal_keys.isInternalUserKey(key)) return null; + const target_term = internal_keys.findComponentTerminator(key, 1) orelse return null; + var pos = target_term + 2; + if (pos >= key.len or key[pos] != internal_keys.artifact_kind) return null; + pos += 1; + if (!internal_keys.componentEquals(key, pos, graph_index_edge_artifact_type)) return null; + pos = (internal_keys.findComponentTerminator(key, pos) orelse return null) + 2; + if (!internal_keys.componentEquals(key, pos, index_name)) return null; + pos = (internal_keys.findComponentTerminator(key, pos) orelse return null) + 2; + if (pos >= key.len or key[pos] != internal_keys.graph_edge_record_kind) return null; + pos += 1; + + const edge_type_term = internal_keys.findComponentTerminator(key, pos) orelse return null; + const edge_type_start = pos; + pos = edge_type_term + 2; + const source_term = internal_keys.findComponentTerminator(key, pos) orelse return null; + if (source_term + 2 != key.len) return null; + + var target = try decodeGraphKeyComponent(alloc, key[1..target_term]); + errdefer target.deinit(alloc); + var edge_type = try decodeGraphKeyComponent(alloc, key[edge_type_start..edge_type_term]); + errdefer edge_type.deinit(alloc); + var source = try decodeGraphKeyComponent(alloc, key[pos..source_term]); + errdefer source.deinit(alloc); + + return .{ .source = source, .edge_type = edge_type, .target = target }; +} + +fn graphMetricFirstComponentAfterPrefixAlloc(alloc: Allocator, key: []const u8, prefix: []const u8) !?[]u8 { + if (!std.mem.startsWith(u8, key, prefix)) return null; + const pos = prefix.len; + const term = internal_keys.findComponentTerminator(key, pos) orelse return null; + return try internal_keys.decodeBodyAlloc(alloc, key[pos..term]); +} + // ============================================================================ // GraphIndex // ============================================================================ @@ -328,6 +454,172 @@ pub const EdgeTypeConfig = struct { topology: TopologyMode = .graph, }; +pub const GraphMetricKind = enum { + pagerank, + degree, + eigenvector, + hits_authority, + hits_hub, +}; + +pub const GraphMetricRefreshMode = enum { + background, + manual, +}; + +pub const GraphMetricEdgeFilterMode = enum { + all, + types, +}; + +pub const GraphMetricEdgeFilter = struct { + mode: GraphMetricEdgeFilterMode = .all, + types: []const []const u8 = &.{}, + + pub fn equivalent(self: @This(), other: @This()) bool { + if (self.mode != other.mode) return false; + return switch (self.mode) { + .all => true, + .types => { + if (self.types.len != other.types.len) return false; + for (self.types) |edge_type| { + if (!other.includesType(edge_type)) return false; + } + for (other.types) |edge_type| { + if (!self.includesType(edge_type)) return false; + } + return true; + }, + }; + } + + pub fn includesType(self: @This(), edge_type: []const u8) bool { + for (self.types) |allowed| { + if (std.mem.eql(u8, allowed, edge_type)) return true; + } + return false; + } + + pub fn cloneAlloc(self: @This(), alloc: Allocator) !@This() { + if (self.types.len == 0) return .{ .mode = self.mode }; + const types = try alloc.alloc([]const u8, self.types.len); + var initialized: usize = 0; + errdefer { + for (types[0..initialized]) |edge_type| alloc.free(edge_type); + alloc.free(types); + } + for (self.types, 0..) |edge_type, i| { + types[i] = try alloc.dupe(u8, edge_type); + initialized += 1; + } + return .{ .mode = self.mode, .types = types }; + } + + pub fn deinit(self: *@This(), alloc: Allocator) void { + for (self.types) |edge_type| alloc.free(edge_type); + if (self.types.len > 0) alloc.free(self.types); + self.* = undefined; + } +}; + +pub const GraphMetricConfig = struct { + name: []const u8, + kind: GraphMetricKind = .pagerank, + damping: f64 = 0.85, + tolerance: f64 = 0.000001, + max_iterations: u32 = 50, + refresh: GraphMetricRefreshMode = .background, + edge_filter: GraphMetricEdgeFilter = .{}, +}; + +/// Iterative graph metrics are intentionally bounded even when configuration +/// is supplied by an internal caller. This keeps a malformed index definition +/// from creating effectively unbounded foreground or background work. +pub const graph_metric_max_iterations: u32 = 1_000; + +pub fn freeGraphMetricConfigs(alloc: Allocator, configs: []GraphMetricConfig) void { + for (configs) |*cfg| { + alloc.free(cfg.name); + cfg.edge_filter.deinit(alloc); + } + if (configs.len > 0) alloc.free(configs); +} + +pub const GraphMetricValidationError = error{ + UnknownGraphMetricEdgeType, + InvalidGraphMetricName, + DuplicateGraphMetricName, + InvalidGraphMetricEdgeFilter, + DuplicateGraphMetricEdgeType, + InvalidGraphMetricIterations, + InvalidGraphMetricHitsPair, + AmbiguousGraphMetricHitsPair, +}; + +pub fn validateGraphMetricEdgeFilters( + edge_type_configs: []const EdgeTypeConfig, + metric_configs: []const GraphMetricConfig, +) GraphMetricValidationError!void { + for (metric_configs, 0..) |metric_cfg, i| { + if (metric_cfg.name.len == 0 or std.mem.indexOfScalar(u8, metric_cfg.name, 0) != null) return error.InvalidGraphMetricName; + if (metric_cfg.max_iterations == 0 or metric_cfg.max_iterations > graph_metric_max_iterations) { + return error.InvalidGraphMetricIterations; + } + for (metric_configs[0..i]) |prior| { + if (std.mem.eql(u8, prior.name, metric_cfg.name)) return error.DuplicateGraphMetricName; + } + if (metric_cfg.edge_filter.mode == .all) { + if (metric_cfg.edge_filter.types.len != 0) return error.InvalidGraphMetricEdgeFilter; + } else { + if (metric_cfg.edge_filter.types.len == 0) return error.InvalidGraphMetricEdgeFilter; + for (metric_cfg.edge_filter.types, 0..) |edge_type, edge_type_i| { + if (edge_type.len == 0) return error.InvalidGraphMetricEdgeFilter; + for (metric_cfg.edge_filter.types[0..edge_type_i]) |prior_edge_type| { + if (std.mem.eql(u8, prior_edge_type, edge_type)) return error.DuplicateGraphMetricEdgeType; + } + if (edge_type_configs.len > 0 and !hasConfiguredEdgeType(edge_type_configs, edge_type)) return error.UnknownGraphMetricEdgeType; + } + } + + const opposite_kind = graphMetricOppositeHitsKind(metric_cfg.kind) orelse continue; + var pair_count: usize = 0; + var pair_refresh: GraphMetricRefreshMode = undefined; + for (metric_configs) |candidate| { + if (candidate.kind != opposite_kind) continue; + if (candidate.max_iterations != metric_cfg.max_iterations) continue; + if (candidate.tolerance != metric_cfg.tolerance) continue; + if (!candidate.edge_filter.equivalent(metric_cfg.edge_filter)) continue; + pair_count += 1; + pair_refresh = candidate.refresh; + } + if (pair_count > 1) return error.AmbiguousGraphMetricHitsPair; + if (pair_count == 1 and pair_refresh != metric_cfg.refresh) return error.InvalidGraphMetricHitsPair; + } +} + +pub fn graphMetricOppositeHitsKind(kind: GraphMetricKind) ?GraphMetricKind { + return switch (kind) { + .hits_authority => .hits_hub, + .hits_hub => .hits_authority, + else => null, + }; +} + +pub fn graphMetricHitsPairCompatible(a: GraphMetricConfig, b: GraphMetricConfig) bool { + return graphMetricOppositeHitsKind(a.kind) == b.kind and + a.refresh == b.refresh and + a.max_iterations == b.max_iterations and + a.tolerance == b.tolerance and + a.edge_filter.equivalent(b.edge_filter); +} + +fn hasConfiguredEdgeType(edge_type_configs: []const EdgeTypeConfig, edge_type: []const u8) bool { + for (edge_type_configs) |cfg| { + if (std.mem.eql(u8, cfg.name, edge_type)) return true; + } + return false; +} + pub const GraphIndexOptions = struct { map_size: usize = 64 * 1024 * 1024, no_sync: bool = false, @@ -338,6 +630,10 @@ pub const GraphIndexOptions = struct { reverse_lsm_options: lsm_backend.Options = .{ .flush_threshold = 1 }, reverse_lsm_root_generation: u64 = 0, edge_type_configs: []const EdgeTypeConfig = &.{}, + metric_configs: []const GraphMetricConfig = &.{}, + /// Optional host-owned pool; must outlive the index. Null uses the shared + /// process pool, not a separate allowance for each graph index. + sealed_vector_budget: ?*@import("sealed_vector_cache.zig").Budget = null, rebuild_root_path: ?[]const u8 = null, rebuild_owner_generation: u64 = 0, algebraic_semiring_traversal: bool = false, @@ -360,7 +656,7 @@ test "graph index routes reverse lsm profile options" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .reverse_backend = .lsm_memory, .reverse_lsm_options = .{ .flush_threshold = 91 }, }); @@ -372,11 +668,136 @@ test "graph index routes reverse lsm profile options" { } } +test "graph metric edge filter validation uses configured edge type metadata" { + const edge_types = [_]EdgeTypeConfig{ + .{ .name = "cites" }, + .{ .name = "mentions" }, + }; + const valid_filter_types = [_][]const u8{ "cites", "mentions" }; + const invalid_filter_types = [_][]const u8{"related"}; + const valid_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .types, .types = &valid_filter_types }, + }}; + const invalid_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .types, .types = &invalid_filter_types }, + }}; + const empty_name_metrics = [_]GraphMetricConfig{.{ + .name = "", + }}; + const duplicate_metrics = [_]GraphMetricConfig{ + .{ .name = "pagerank" }, + .{ .name = "pagerank", .kind = .degree }, + }; + const empty_filter_types = [_][]const u8{}; + const empty_filter_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .types, .types = &empty_filter_types }, + }}; + const blank_filter_types = [_][]const u8{""}; + const blank_filter_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .types, .types = &blank_filter_types }, + }}; + const duplicate_filter_types = [_][]const u8{ "cites", "cites" }; + const duplicate_filter_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .types, .types = &duplicate_filter_types }, + }}; + const all_with_types_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .edge_filter = .{ .mode = .all, .types = &valid_filter_types }, + }}; + const zero_iteration_metrics = [_]GraphMetricConfig{.{ .name = "pagerank", .max_iterations = 0 }}; + const excessive_iteration_metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .max_iterations = graph_metric_max_iterations + 1, + }}; + const mismatched_hits_refresh = [_]GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority, .refresh = .background }, + .{ .name = "hub", .kind = .hits_hub, .refresh = .manual }, + }; + const ambiguous_hits = [_]GraphMetricConfig{ + .{ .name = "authority-a", .kind = .hits_authority }, + .{ .name = "authority-b", .kind = .hits_authority }, + .{ .name = "hub", .kind = .hits_hub }, + }; + const cites_only = [_][]const u8{"cites"}; + const mentions_only = [_][]const u8{"mentions"}; + const distinct_hits_pairs = [_]GraphMetricConfig{ + .{ .name = "authority-cites", .kind = .hits_authority, .edge_filter = .{ .mode = .types, .types = &cites_only } }, + .{ .name = "hub-cites", .kind = .hits_hub, .edge_filter = .{ .mode = .types, .types = &cites_only } }, + .{ .name = "authority-mentions", .kind = .hits_authority, .edge_filter = .{ .mode = .types, .types = &mentions_only } }, + .{ .name = "hub-mentions", .kind = .hits_hub, .edge_filter = .{ .mode = .types, .types = &mentions_only } }, + }; + + try validateGraphMetricEdgeFilters(&edge_types, &valid_metrics); + try validateGraphMetricEdgeFilters(&.{}, &invalid_metrics); + try std.testing.expectError(error.UnknownGraphMetricEdgeType, validateGraphMetricEdgeFilters(&edge_types, &invalid_metrics)); + try std.testing.expectError(error.InvalidGraphMetricName, validateGraphMetricEdgeFilters(&edge_types, &empty_name_metrics)); + try std.testing.expectError(error.DuplicateGraphMetricName, validateGraphMetricEdgeFilters(&edge_types, &duplicate_metrics)); + try std.testing.expectError(error.InvalidGraphMetricEdgeFilter, validateGraphMetricEdgeFilters(&edge_types, &empty_filter_metrics)); + try std.testing.expectError(error.InvalidGraphMetricEdgeFilter, validateGraphMetricEdgeFilters(&edge_types, &blank_filter_metrics)); + try std.testing.expectError(error.DuplicateGraphMetricEdgeType, validateGraphMetricEdgeFilters(&edge_types, &duplicate_filter_metrics)); + try std.testing.expectError(error.InvalidGraphMetricEdgeFilter, validateGraphMetricEdgeFilters(&edge_types, &all_with_types_metrics)); + try std.testing.expectError(error.InvalidGraphMetricIterations, validateGraphMetricEdgeFilters(&edge_types, &zero_iteration_metrics)); + try std.testing.expectError(error.InvalidGraphMetricIterations, validateGraphMetricEdgeFilters(&edge_types, &excessive_iteration_metrics)); + try std.testing.expectError(error.InvalidGraphMetricHitsPair, validateGraphMetricEdgeFilters(&edge_types, &mismatched_hits_refresh)); + try std.testing.expectError(error.AmbiguousGraphMetricHitsPair, validateGraphMetricEdgeFilters(&edge_types, &ambiguous_hits)); + try validateGraphMetricEdgeFilters(&edge_types, &distinct_hits_pairs); +} + const reverse_rebuild_batch_size: usize = 1024; pub var test_abort_reverse_rebuild_after_batches: ?usize = null; const graph_meta_prefix = "meta:"; const graph_edge_count_key = "meta:edge_count"; const graph_node_count_key = "meta:node_count"; +const graph_edge_generation_key = "meta:edge_generation"; +const graph_metric_type_epoch_floor_key = "meta:metric_type_epoch_floor:v1"; +const graph_metric_key_prefix = "meta:metric:"; +const graph_metric_control_key_prefix = "meta:metric_control:"; +const graph_metric_type_epochs_prefix = "meta:metric_type_epochs:v1/"; +const graph_metric_filter_plan_prefix = "meta:metric_filter_plan:v1/"; +const topology_task_prefix = "meta:metric_topology:tasks/"; +const topology_task_name_prefix = "\x00topology/"; +const topology_task_control_prefix = "meta:metric_topology:control/"; +const topology_task_incarnation_key = "meta:metric_topology:task-incarnation"; +const graph_metric_recent_event_limit = 8; +const graph_metric_status_page_limit = 8; +const default_graph_metric_deferred_cleanup_ms: u64 = 60_000; +const graph_metric_local_build_worker_id = "local"; +const graph_metric_local_build_lease_ms: u64 = 300_000; +const graph_metric_build_max_page_attempts: u64 = 3; +const graph_metric_build_target_scan_page_units: usize = 64; +const graph_metric_build_target_reduce_page_units: usize = 64; +// A worker checkpoint is a durable write transaction. Keep partition planning +// stable, but amortize execution checkpoints over enough immutable input to +// avoid millions of tiny transactions on production-sized graphs. Tests pass +// explicit smaller limits when exercising resumability. +const graph_metric_build_checkpoint_scan_units: usize = 4096; +const graph_metric_build_checkpoint_reduce_units: usize = 4096; +const graph_metric_build_summary_leaf_base: u64 = 65536; +// Keep the durable scheduler surface bounded while distributing the immutable +// keyspace evenly across workers. Per-tick execution remains independently +// bounded by the target unit constants below. +// Large graph builds must be able to occupy a real maintenance pool. The +// planner still chooses pages from bounded target work units, while this high +// safety ceiling prevents unbounded durable control records. +const graph_metric_build_max_partition_pages: usize = 256; +const graph_metric_build_cleanup_delete_page_units: usize = 512; +const graph_metric_build_adoption_page_units: usize = 512; +const graph_metric_packed_f64_magic: u64 = 0xA17F_5046_3634_0001; +const graph_metric_packed_f64_header_len: usize = 24; +const graph_metric_build_adoption_cursor_prefix = "@adopt:"; +const graph_metric_partition_plan_key = "meta:metric_partition_plan:v7"; +const graph_metric_partition_census_key = "meta:metric_partition_census:v3"; +const graph_metric_partition_plan_version: u32 = 9; +const graph_metric_partition_plan_checksum_seed: u64 = 0xA17F_504C_414E_0007; +// The public API caps top-K at this value. Retaining a rank entry for every +// score doubles write/storage amplification without improving any supported +// query, so each immutable generation keeps only this exact ordered prefix. +const graph_metric_rank_entry_limit: usize = 10_000; pub const ReverseBackend = enum { lmdb, @@ -386,6 +807,18 @@ pub const ReverseBackend = enum { }; pub const GraphIndex = struct { + /// Explicit small-page injection for recovery fixtures; production uses + /// byte/work-bounded checkpoints with 4096-unit scheduling ranges. + test_partition_target_units: ?usize = null, + sealed_vectors: @import("sealed_vector_cache.zig").Cache = .{}, + topology_gc_mutex: std.atomic.Mutex = .unlocked, + // Census position is only a fairness hint. Deletion tombstones and removed + // keys are the durable recovery state; idle scans must not write a WAL. + topology_gc_cursor: ?topology_owner.Id = null, + filter_plan_gc_cursor: ?struct { bytes: [68]u8, len: u8 } = null, + topology_preparation_only: bool = false, + topology_preparation_mutex: std.atomic.Mutex = .unlocked, + topology_preparation_cursor: ?[64]u8 = null, alloc: Allocator, index_name: []const u8, outgoing_store: backend_erased.Store, @@ -393,12 +826,14 @@ pub const GraphIndex = struct { reverse_store: backend_erased.Store, reverse_owner: ReverseStoreOwner, edge_type_configs: []const EdgeTypeConfig, + metric_configs: []const GraphMetricConfig, rebuild_root_path: ?[]u8, rebuild_storage: ?lsm_backend.Storage, rebuild_owner_generation: u64, algebraic_semiring_traversal: bool, edge_count: u64, node_count: u64, + edge_generation: u64, algebraic_traversal_attempt_count: u64, algebraic_traversal_proven_count: u64, algebraic_traversal_rejected_count: u64, @@ -549,6 +984,7 @@ pub const GraphIndex = struct { return .{ .edge_count = try readU64OrZero(&txn, graph_edge_count_key), .node_count = try readU64OrZero(&txn, graph_node_count_key), + .edge_generation = try readU64OrZero(&txn, graph_edge_generation_key), }; } @@ -557,10 +993,527 @@ pub const GraphIndex = struct { error.NotFound => return 0, else => return err, }; - if (raw.len < 8) return 0; + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; return std.mem.readInt(u64, raw[0..8], .little); } + const IndexedReadKey = struct { + original_index: usize, + key: []const u8, + + fn lessThan(_: void, left: @This(), right: @This()) bool { + const order = std.mem.order(u8, left.key, right.key); + return order == .lt or (order == .eq and left.original_index < right.original_index); + } + }; + + /// Resolve an arbitrary key vector through the backend's sorted multi-get + /// planner while preserving caller order. LSM backends can turn this into a + /// cursor/run merge instead of paying one complete lookup per graph node. + fn getManyValuesAlloc( + self: *GraphIndex, + txn: anytype, + keys: []const []const u8, + ) ![]?[]const u8 { + const values = try self.alloc.alloc(?[]const u8, keys.len); + errdefer self.alloc.free(values); + @memset(values, null); + if (keys.len == 0) return values; + + const indexed = try self.alloc.alloc(IndexedReadKey, keys.len); + defer self.alloc.free(indexed); + for (keys, 0..) |key, i| indexed[i] = .{ .original_index = i, .key = key }; + std.mem.sort(IndexedReadKey, indexed, {}, IndexedReadKey.lessThan); + + const sorted_keys = try self.alloc.alloc([]const u8, keys.len); + defer self.alloc.free(sorted_keys); + const sorted_values = try self.alloc.alloc(?[]const u8, keys.len); + defer self.alloc.free(sorted_values); + @memset(sorted_values, null); + for (indexed, 0..) |item, i| sorted_keys[i] = item.key; + try txn.getManySorted(sorted_keys, sorted_values); + for (indexed, sorted_values) |item, value| values[item.original_index] = value; + return values; + } + + fn readF64KeysAlloc(self: *GraphIndex, txn: anytype, keys: []const []const u8) ![]f64 { + const raw_values = try self.getManyValuesAlloc(txn, keys); + defer self.alloc.free(raw_values); + const values = try self.alloc.alloc(f64, keys.len); + errdefer self.alloc.free(values); + for (raw_values, 0..) |maybe_raw, i| { + values[i] = if (maybe_raw) |raw| decodeF64(raw) orelse return error.InvalidGraphMetricScore else 0.0; + if (!std.math.isFinite(values[i])) return error.InvalidGraphMetricScore; + } + return values; + } + + fn readRequiredF64KeysAlloc(self: *GraphIndex, txn: anytype, keys: []const []const u8) ![]f64 { + const raw_values = try self.getManyValuesAlloc(txn, keys); + defer self.alloc.free(raw_values); + const values = try self.alloc.alloc(f64, keys.len); + errdefer self.alloc.free(values); + for (raw_values, 0..) |maybe_raw, i| { + const raw = maybe_raw orelse return error.InvalidGraphMetricScore; + values[i] = decodeF64(raw) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(values[i])) return error.InvalidGraphMetricScore; + } + return values; + } + + fn readDeterministicF64KeysAlloc(self: *GraphIndex, txn: anytype, keys: []const []const u8) ![]f64 { + const raw_values = try self.getManyValuesAlloc(txn, keys); + defer self.alloc.free(raw_values); + const values = try self.alloc.alloc(f64, keys.len); + errdefer self.alloc.free(values); + for (raw_values, 0..) |maybe_raw, i| { + values[i] = if (maybe_raw) |raw| try decodeDeterministicF64Sum(raw) else 0.0; + } + return values; + } + + fn readU64KeysAlloc(self: *GraphIndex, txn: anytype, keys: []const []const u8) ![]u64 { + const raw_values = try self.getManyValuesAlloc(txn, keys); + defer self.alloc.free(raw_values); + const values = try self.alloc.alloc(u64, keys.len); + errdefer self.alloc.free(values); + for (raw_values, 0..) |maybe_raw, i| { + values[i] = if (maybe_raw) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else 0; + } + return values; + } + + fn addU64DeltasInBatch( + self: *GraphIndex, + batch: anytype, + keys: []const []const u8, + deltas: []const u64, + include_prior: bool, + ) !void { + if (keys.len != deltas.len) return error.InvalidGraphMetricBuildManifest; + const priors = if (include_prior) try self.readU64KeysAlloc(batch, keys) else null; + defer if (priors) |values| self.alloc.free(values); + for (keys, deltas, 0..) |key, delta, i| { + const prior = if (priors) |values| values[i] else 0; + try putU64(batch, key, std.math.add(u64, prior, delta) catch + return error.InvalidGraphMetricBuildManifest); + } + } + + fn addF64DeltasInBatch( + self: *GraphIndex, + batch: anytype, + keys: []const []const u8, + deltas: []const f64, + include_prior: bool, + ) !void { + if (keys.len != deltas.len) return error.InvalidGraphMetricBuildManifest; + const priors = if (include_prior) try self.readF64KeysAlloc(batch, keys) else null; + defer if (priors) |values| self.alloc.free(values); + for (keys, deltas, 0..) |key, delta, i| { + const prior = if (priors) |values| values[i] else 0.0; + const value = prior + delta; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + try putF64(batch, key, value); + } + } + + /// Idempotently replace each page shard and adjust its canonical total by + /// the delta in one vector read. Adoption is serialized by the reverse + /// store writer, so balanced scan pages may safely share hot graph nodes + /// without sacrificing scan parallelism. + fn replaceU64PageDeltasInBatch( + self: *GraphIndex, + batch: anytype, + page_keys: []const []const u8, + total_keys: []const []const u8, + replacements: []const u64, + ) !void { + if (page_keys.len != total_keys.len or page_keys.len != replacements.len) return error.InvalidGraphMetricBuildManifest; + const read_key_count = std.math.mul(usize, page_keys.len, 2) catch return error.InvalidGraphMetricBuildManifest; + const read_keys = try self.alloc.alloc([]const u8, read_key_count); + defer self.alloc.free(read_keys); + @memcpy(read_keys[0..page_keys.len], page_keys); + @memcpy(read_keys[page_keys.len..], total_keys); + const raw_values = try self.getManyValuesAlloc(batch, read_keys); + defer self.alloc.free(raw_values); + const combined_values = try self.alloc.alloc(u64, replacements.len); + defer self.alloc.free(combined_values); + for (replacements, 0..) |replacement, i| { + const prior_page = if (raw_values[i]) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else 0; + const prior_total = if (raw_values[page_keys.len + i]) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else if (prior_page == 0) 0 else return error.InvalidGraphMetricBuildManifest; + const without_prior = std.math.sub(u64, prior_total, prior_page) catch + return error.InvalidGraphMetricBuildManifest; + combined_values[i] = std.math.add(u64, without_prior, replacement) catch + return error.InvalidGraphMetricBuildManifest; + } + // Backend getMany values are borrowed from the transaction. Decode the + // complete vector before the first mutation so this remains valid for + // in-memory backends whose put may relocate value storage. + for (page_keys, total_keys, replacements, combined_values) |page_key, total_key, replacement, combined| { + try putU64(batch, page_key, replacement); + try putU64(batch, total_key, combined); + } + } + + fn replaceF64PageDeltasInBatch( + self: *GraphIndex, + batch: anytype, + page_keys: []const []const u8, + total_keys: []const []const u8, + replacements: []const f64, + deterministic_nodes: []const bool, + page_id: u64, + ) !void { + if (page_keys.len != total_keys.len or page_keys.len != replacements.len or + page_keys.len != deterministic_nodes.len) + { + return error.InvalidGraphMetricBuildManifest; + } + const read_key_count = std.math.mul(usize, page_keys.len, 2) catch return error.InvalidGraphMetricBuildManifest; + const read_keys = try self.alloc.alloc([]const u8, read_key_count); + defer self.alloc.free(read_keys); + @memcpy(read_keys[0..page_keys.len], page_keys); + @memcpy(read_keys[page_keys.len..], total_keys); + const raw_values = try self.getManyValuesAlloc(batch, read_keys); + defer self.alloc.free(raw_values); + const combined_values = try self.alloc.alloc([]u8, replacements.len); + var initialized_values: usize = 0; + defer { + for (combined_values[0..initialized_values]) |value| self.alloc.free(value); + self.alloc.free(combined_values); + } + for (replacements, 0..) |replacement, i| { + if (!std.math.isFinite(replacement)) return error.InvalidGraphMetricScore; + if (!deterministic_nodes[i]) { + if (raw_values[page_keys.len + i]) |raw| { + if (raw.len != 8 or decodeF64(raw) == null) return error.InvalidGraphMetricBuildManifest; + } + combined_values[i] = try self.alloc.alloc(u8, 8); + std.mem.writeInt(u64, combined_values[i][0..8], @bitCast(replacement), .little); + initialized_values += 1; + continue; + } + const legacy_page_id: ?u64 = if (raw_values[i]) |raw| blk: { + const prior_page = decodeF64(raw) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(prior_page)) return error.InvalidGraphMetricScore; + break :blk page_id; + } else null; + combined_values[i] = try self.replaceDeterministicF64PageValueAlloc( + raw_values[page_keys.len + i], + legacy_page_id, + page_id, + replacement, + ); + initialized_values += 1; + } + for (page_keys, total_keys, replacements, combined_values) |page_key, total_key, replacement, combined| { + try putF64(batch, page_key, replacement); + try batch.put(total_key, combined); + } + } + + const deterministic_f64_page_values_magic: u64 = 0xA17F_4636_3450_4147; + const DeterministicF64PageValue = struct { page_id: u64, value: f64 }; + const PackedF64Entry = struct { + node: []const u8, + value: f64, + + fn lessThan(_: void, left: @This(), right: @This()) bool { + return std.mem.lessThan(u8, left.node, right.node); + } + }; + + /// Reverse-edge scans are target-major. Preserve that ordering directly + /// instead of hashing and copying every distinct target and sorting it back + /// into the order the storage cursor already provided. + const TargetContributionRuns = struct { + entries: std.ArrayListUnmanaged(PackedF64Entry) = .empty, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.entries.items) |entry| alloc.free(entry.node); + self.entries.deinit(alloc); + self.* = undefined; + } + + fn indexFor(self: *@This(), alloc: Allocator, target: []const u8) !usize { + if (self.entries.items.len > 0) { + const prior = self.entries.items[self.entries.items.len - 1].node; + switch (std.mem.order(u8, prior, target)) { + .eq => return self.entries.items.len - 1, + .gt => return error.InvalidGraphMetricBuildManifest, + .lt => {}, + } + } + const owned = try alloc.dupe(u8, target); + errdefer alloc.free(owned); + try self.entries.append(alloc, .{ .node = owned, .value = 0 }); + return self.entries.items.len - 1; + } + }; + + fn encodePackedF64EntriesAlloc(self: *GraphIndex, entries: []const PackedF64Entry) ![]u8 { + if (entries.len > graph_metric_build_adoption_page_units) return error.InvalidGraphMetricBuildManifest; + var encoded_len = graph_metric_packed_f64_header_len; + for (entries, 0..) |entry, i| { + if (entry.node.len == 0 or !std.math.isFinite(entry.value) or entry.node.len > std.math.maxInt(u32)) + return error.InvalidGraphMetricScore; + if (i > 0 and std.mem.order(u8, entries[i - 1].node, entry.node) != .lt) + return error.InvalidGraphMetricBuildManifest; + encoded_len = std.math.add(usize, encoded_len, 4 + entry.node.len + 8) catch + return error.InvalidGraphMetricBuildManifest; + } + const encoded = try self.alloc.alloc(u8, encoded_len); + errdefer self.alloc.free(encoded); + std.mem.writeInt(u64, encoded[0..8], graph_metric_packed_f64_magic, .little); + std.mem.writeInt(u32, encoded[8..12], 1, .little); + std.mem.writeInt(u32, encoded[12..16], @intCast(entries.len), .little); + var offset: usize = graph_metric_packed_f64_header_len; + for (entries) |entry| { + std.mem.writeInt(u32, encoded[offset..][0..4], @intCast(entry.node.len), .little); + offset += 4; + @memcpy(encoded[offset .. offset + entry.node.len], entry.node); + offset += entry.node.len; + std.mem.writeInt(u64, encoded[offset..][0..8], @bitCast(entry.value), .little); + offset += 8; + } + std.debug.assert(offset == encoded.len); + std.mem.writeInt(u64, encoded[16..24], std.hash.Wyhash.hash(graph_metric_packed_f64_magic, encoded[graph_metric_packed_f64_header_len..]), .little); + return encoded; + } + + fn decodePackedF64EntriesAlloc(self: *GraphIndex, encoded: []const u8) ![]PackedF64Entry { + if (encoded.len < graph_metric_packed_f64_header_len or + std.mem.readInt(u64, encoded[0..8], .little) != graph_metric_packed_f64_magic or + std.mem.readInt(u32, encoded[8..12], .little) != 1 or + std.mem.readInt(u64, encoded[16..24], .little) != std.hash.Wyhash.hash(graph_metric_packed_f64_magic, encoded[graph_metric_packed_f64_header_len..])) + { + return error.InvalidGraphMetricBuildManifest; + } + const count: usize = std.mem.readInt(u32, encoded[12..16], .little); + if (count > graph_metric_build_adoption_page_units) return error.InvalidGraphMetricBuildManifest; + const entries = try self.alloc.alloc(PackedF64Entry, count); + errdefer self.alloc.free(entries); + var offset: usize = graph_metric_packed_f64_header_len; + for (entries, 0..) |*entry, i| { + if (offset + 4 > encoded.len) return error.InvalidGraphMetricBuildManifest; + const node_len: usize = std.mem.readInt(u32, encoded[offset..][0..4], .little); + offset += 4; + const value_offset = std.math.add(usize, offset, node_len) catch return error.InvalidGraphMetricBuildManifest; + if (value_offset + 8 > encoded.len) return error.InvalidGraphMetricBuildManifest; + entry.* = .{ + .node = encoded[offset..value_offset], + .value = @bitCast(std.mem.readInt(u64, encoded[value_offset..][0..8], .little)), + }; + if (entry.node.len == 0 or !std.math.isFinite(entry.value) or + (i > 0 and std.mem.order(u8, entries[i - 1].node, entry.node) != .lt)) + { + return error.InvalidGraphMetricBuildManifest; + } + offset = value_offset + 8; + } + if (offset != encoded.len) return error.InvalidGraphMetricBuildManifest; + return entries; + } + + fn ordinalAttemptContributionForNodeForTest( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + node: []const u8, + ) !f64 { + // Fixture oracle for inspecting an uncommitted attempt. Production + // readers only select a producer after its completion barrier. + std.debug.assert(builtin.is_test); + const slots = try self.graphMetricNodeSlotsAlloc(txn, metric_name, job_id, &.{node}); + defer self.alloc.free(slots); + const chunk_prefix = try self.ordinalAdjacencyPrefixAlloc(metric_name, job_id, phase, slots[0] / vector_chunk.entries); + defer self.alloc.free(chunk_prefix); + const prefix = try std.fmt.allocPrint(self.alloc, "{s}{d:0>20}:{d:0>20}:", .{ chunk_prefix, page_id, attempt }); + defer self.alloc.free(prefix); + var sum: f64 = 0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const entries = try self.ordinalAdjacencyValuesAlloc(txn, metric_name, job_id, phase, iteration, entry.value); + defer self.alloc.free(entries); + for (entries) |entry_value| { + if (entry_value.ordinal == slots[0]) sum += entry_value.value; + } + } + if (!std.math.isFinite(sum)) return error.InvalidGraphMetricScore; + return sum; + } + + fn publishPackedF64CheckpointShardsInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + entries: []const PackedF64Entry, + page: GraphMetricBuildPage, + checkpoint_suffix: []const u8, + ) !void { + // The suffix contains the checkpoint offset and chunk ordinal emitted + // by the attempt writer. Both are stable across retries, so a reclaimed + // attempt overwrites exactly the same immutable shard without reading + // or merging prior state. + var suffix_pos: usize = 0; + for (0..2) |_| { + suffix_pos = (internal_keys.findComponentTerminator(checkpoint_suffix, suffix_pos) orelse + return error.InvalidGraphMetricBuildManifest) + 2; + } + if (suffix_pos != checkpoint_suffix.len) return error.InvalidGraphMetricBuildManifest; + + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d:0>20}", .{page.page_id}); + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(self.alloc); + for (entries) |entry| { + if (!std.math.isFinite(entry.value)) return error.InvalidGraphMetricScore; + try self.writeGraphMetricControlKey(&key, &.{ metric_name, "job", job_id_text, "pagerank_contribution", iteration_text, entry.node, page_id_text }); + try key.appendSlice(self.alloc, checkpoint_suffix); + try putF64(batch, key.items, entry.value); + } + } + + /// Canonical per-node floating aggregates retain one bounded value per + /// balanced input page. Encoding entries in page-id order makes the final + /// fold independent of worker completion order while adding only one + /// extra entry for each partition boundary that splits a target. + fn deterministicF64PageValuesAlloc( + self: *GraphIndex, + raw: ?[]const u8, + legacy_page_id: ?u64, + ) ![]DeterministicF64PageValue { + const encoded = raw orelse return try self.alloc.alloc(DeterministicF64PageValue, 0); + if (encoded.len == 8) { + const page = legacy_page_id orelse return error.InvalidGraphMetricBuildManifest; + const value = decodeF64(encoded) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + const values = try self.alloc.alloc(DeterministicF64PageValue, 1); + values[0] = .{ .page_id = page, .value = value }; + return values; + } + if (encoded.len < 24 or (encoded.len - 8) % 16 != 0 or + std.mem.readInt(u64, encoded[0..8], .little) != deterministic_f64_page_values_magic) + { + return error.InvalidGraphMetricBuildManifest; + } + const count = (encoded.len - 8) / 16; + if (count > graph_metric_build_max_partition_pages) return error.InvalidGraphMetricBuildManifest; + const values = try self.alloc.alloc(DeterministicF64PageValue, count); + errdefer self.alloc.free(values); + for (values, 0..) |*entry, i| { + const offset = 8 + i * 16; + entry.* = .{ + .page_id = std.mem.readInt(u64, encoded[offset..][0..8], .little), + .value = @bitCast(std.mem.readInt(u64, encoded[offset + 8 ..][0..8], .little)), + }; + if (!std.math.isFinite(entry.value) or + (i > 0 and values[i - 1].page_id >= entry.page_id)) + { + return error.InvalidGraphMetricBuildManifest; + } + } + return values; + } + + fn replaceDeterministicF64PageValueAlloc( + self: *GraphIndex, + raw: ?[]const u8, + legacy_page_id: ?u64, + page_id: u64, + replacement: f64, + ) ![]u8 { + const prior = try self.deterministicF64PageValuesAlloc(raw, legacy_page_id); + defer self.alloc.free(prior); + var insertion_index: usize = 0; + while (insertion_index < prior.len and prior[insertion_index].page_id < page_id) : (insertion_index += 1) {} + const replaces_existing = insertion_index < prior.len and prior[insertion_index].page_id == page_id; + const next_count = prior.len + @intFromBool(!replaces_existing); + const encoded = try self.alloc.alloc(u8, 8 + next_count * 16); + errdefer self.alloc.free(encoded); + std.mem.writeInt(u64, encoded[0..8], deterministic_f64_page_values_magic, .little); + var out_index: usize = 0; + for (prior, 0..) |entry, prior_index| { + if (prior_index == insertion_index) { + const value = DeterministicF64PageValue{ .page_id = page_id, .value = replacement }; + const offset = 8 + out_index * 16; + std.mem.writeInt(u64, encoded[offset..][0..8], value.page_id, .little); + std.mem.writeInt(u64, encoded[offset + 8 ..][0..8], @bitCast(value.value), .little); + out_index += 1; + if (replaces_existing) continue; + } + const offset = 8 + out_index * 16; + std.mem.writeInt(u64, encoded[offset..][0..8], entry.page_id, .little); + std.mem.writeInt(u64, encoded[offset + 8 ..][0..8], @bitCast(entry.value), .little); + out_index += 1; + } + if (insertion_index == prior.len) { + const offset = 8 + out_index * 16; + std.mem.writeInt(u64, encoded[offset..][0..8], page_id, .little); + std.mem.writeInt(u64, encoded[offset + 8 ..][0..8], @bitCast(replacement), .little); + out_index += 1; + } + std.debug.assert(out_index == next_count); + return encoded; + } + + fn decodeDeterministicF64Sum(raw: []const u8) !f64 { + if (raw.len == 8) { + const value = decodeF64(raw) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + return value; + } + if (raw.len < 24 or (raw.len - 8) % 16 != 0 or + std.mem.readInt(u64, raw[0..8], .little) != deterministic_f64_page_values_magic) + { + return error.InvalidGraphMetricBuildManifest; + } + const count = (raw.len - 8) / 16; + if (count > graph_metric_build_max_partition_pages) return error.InvalidGraphMetricBuildManifest; + var sum: f64 = 0.0; + var correction: f64 = 0.0; + var prior_page_id: ?u64 = null; + for (0..count) |i| { + const offset = 8 + i * 16; + const page_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + const value: f64 = @bitCast(std.mem.readInt(u64, raw[offset + 8 ..][0..8], .little)); + if (!std.math.isFinite(value) or (prior_page_id != null and prior_page_id.? >= page_id)) { + return error.InvalidGraphMetricBuildManifest; + } + const next = sum + value; + correction += if (@abs(sum) >= @abs(value)) (sum - next) + value else (value - next) + sum; + sum = next; + prior_page_id = page_id; + } + const value = sum + correction; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + return value; + } + fn putU64(txn: anytype, key: []const u8, value: u64) !void { var buf: [8]u8 = undefined; std.mem.writeInt(u64, &buf, value, .little); @@ -618,1353 +1571,31781 @@ pub const GraphIndex = struct { fn persistGraphCounters(self: *GraphIndex, batch: anytype) !void { try putU64(batch, graph_edge_count_key, self.edge_count); try putU64(batch, graph_node_count_key, self.node_count); + try putU64(batch, graph_edge_generation_key, self.edge_generation); } - fn rememberNodeRefCount(self: *GraphIndex, counts: *std.StringHashMapUnmanaged(u64), node: []const u8) !void { - const result = try counts.getOrPut(self.alloc, node); - if (result.found_existing) { - result.value_ptr.* += 1; - return; - } - errdefer _ = counts.remove(node); - result.key_ptr.* = try self.alloc.dupe(u8, node); - result.value_ptr.* = 1; + fn graphMetricKeyAlloc(self: *GraphIndex, parts: []const []const u8) ![]u8 { + return try graphMetricKeyWithAllocator(self.alloc, parts); } - fn rebuildCounterMetadata(self: *GraphIndex) !void { - const prev_edge_count = self.edge_count; - const prev_node_count = self.node_count; - errdefer { - self.edge_count = prev_edge_count; - self.node_count = prev_node_count; - } + fn graphMetricKeyWithAllocator(alloc: Allocator, parts: []const []const u8) ![]u8 { + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(alloc); + try list.appendSlice(alloc, graphMetricNamespacePrefix(parts, graph_metric_key_prefix)); + for (parts) |part| try internal_keys.appendEncodedComponent(&list, alloc, part); + return try list.toOwnedSlice(alloc); + } - var read_txn = try self.beginReadReverseTxn(); - defer read_txn.abort(); + fn writeGraphMetricKey(self: *GraphIndex, list: *std.ArrayListUnmanaged(u8), parts: []const []const u8) !void { + list.clearRetainingCapacity(); + try list.appendSlice(self.alloc, graphMetricNamespacePrefix(parts, graph_metric_key_prefix)); + for (parts) |part| try internal_keys.appendEncodedComponent(list, self.alloc, part); + } - var meta_keys = std.ArrayListUnmanaged([]u8).empty; - defer { - for (meta_keys.items) |key| self.alloc.free(key); - meta_keys.deinit(self.alloc); - } - var node_refs = std.StringHashMapUnmanaged(u64).empty; - defer { - var key_it = node_refs.keyIterator(); - while (key_it.next()) |key| self.alloc.free(key.*); - node_refs.deinit(self.alloc); - } + fn graphMetricControlKeyAlloc(self: *GraphIndex, parts: []const []const u8) ![]u8 { + return try graphMetricControlKeyWithAllocator(self.alloc, parts); + } - var edge_count: u64 = 0; - var cur = try read_txn.openCursor(); - defer cur.close(); - var maybe_entry = try cur.first(); - while (maybe_entry) |entry| { - if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) { - try meta_keys.append(self.alloc, try self.alloc.dupe(u8, entry.key)); - } else { - edge_count += 1; - if (try parseReverseEdgeKeyAlloc(self.alloc, entry.key)) |parsed_owned| { - var parsed = parsed_owned; - defer parsed.deinit(self.alloc); - try self.rememberNodeRefCount(&node_refs, parsed.source); - try self.rememberNodeRefCount(&node_refs, parsed.target); - } - } - maybe_entry = try cur.next(); - } + fn writeGraphMetricControlKey(self: *GraphIndex, list: *std.ArrayListUnmanaged(u8), parts: []const []const u8) !void { + list.clearRetainingCapacity(); + try list.appendSlice(self.alloc, graphMetricControlPrefix(parts)); + for (parts) |part| try internal_keys.appendEncodedComponent(list, self.alloc, part); + } - var batch = try self.beginWriteReverseBatch(); - errdefer batch.abort(); - for (meta_keys.items) |key| { - batch.delete(key) catch |err| switch (err) { - error.NotFound => {}, - else => return err, - }; - } - - var node_count: u64 = 0; - var refs_it = node_refs.iterator(); - while (refs_it.next()) |entry| { - const ref_key = try graphNodeRefKeyAlloc(self.alloc, entry.key_ptr.*); - defer self.alloc.free(ref_key); - try putU64(&batch, ref_key, entry.value_ptr.*); - node_count += 1; - } - - self.edge_count = edge_count; - self.node_count = node_count; - try self.persistGraphCounters(&batch); - try batch.commit(); + fn graphMetricControlKeyWithAllocator(alloc: Allocator, parts: []const []const u8) ![]u8 { + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(alloc); + try list.appendSlice(alloc, graphMetricControlPrefix(parts)); + for (parts) |part| try internal_keys.appendEncodedComponent(&list, alloc, part); + return try list.toOwnedSlice(alloc); } - fn openEdgeStore(alloc: Allocator, path: [*:0]const u8, opts: GraphIndexOptions) !OpenedReverseStore { - switch (opts.reverse_backend) { - .lmdb => { - if (!supports_native_reverse_lmdb) return error.UnsupportedPlatform; - const backend = try alloc.create(lmdb_backend.Backend); - errdefer alloc.destroy(backend); - backend.* = try lmdb_backend.Backend.open(alloc, path, .{ - .backend = .{ - .durability = if (opts.no_sync) .none else .full, - }, - .env = .{ - .map_size = opts.map_size, - .no_sync = opts.no_sync, - .no_meta_sync = opts.no_meta_sync, - .no_tls = true, - .max_dbs = 1, - }, - }); - errdefer backend.close(); + fn graphMetricControlPrefix(parts: []const []const u8) []const u8 { + return graphMetricNamespacePrefix(parts, graph_metric_control_key_prefix); + } - var runtime = try backend.runtimeStore(alloc, .{}); - errdefer runtime.deinit(); - return .{ - .store = runtime, - .owner = .{ .lmdb = backend }, - }; - }, - .mem => { - const backend = try alloc.create(mem_backend.Backend); - errdefer alloc.destroy(backend); - backend.* = mem_backend.Backend.init(alloc, .{}); - errdefer backend.close(); + fn graphMetricNamespacePrefix(parts: []const []const u8, ordinary: []const u8) []const u8 { + // Task diagnostics/counters belong to the same bounded retirement + // namespace as its pages; none may leak into public metric metadata. + return if (parts.len != 0 and std.mem.startsWith(u8, parts[0], topology_task_name_prefix)) topology_task_control_prefix else ordinary; + } - var runtime = try backend.runtimeStore(alloc, .{}); - errdefer runtime.deinit(); - return .{ - .store = runtime, - .owner = .{ .mem = backend }, - }; - }, - .lsm_memory => { - var handle = try lsm_backend.BackendHandle.init(alloc, resolvedReverseLsmOptions(opts, true)); - errdefer handle.close(); + fn graphMetricScoreKeyAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64, node: []const u8) ![]u8 { + return try graphMetricScoreKeyWithAllocator(self.alloc, metric_name, generation, node); + } - var runtime = try handle.backend.runtimeStore(alloc, .{}); - errdefer runtime.deinit(); - return .{ - .store = runtime, - .owner = .{ .lsm = handle }, - }; - }, - .lsm => { - var handle = try lsm_backend.BackendHandle.open(alloc, std.mem.span(path), resolvedReverseLsmOptions(opts, false)); - errdefer handle.close(); + fn graphMetricScoreKeyWithAllocator(alloc: Allocator, metric_name: []const u8, generation: u64, node: []const u8) ![]u8 { + var generation_buf: [20]u8 = undefined; + const generation_text = try std.fmt.bufPrint(&generation_buf, "{d}", .{generation}); + return try graphMetricKeyWithAllocator(alloc, &.{ metric_name, "score", generation_text, node }); + } - var runtime = try handle.backend.runtimeStore(alloc, .{}); - errdefer runtime.deinit(); - return .{ - .store = runtime, - .owner = .{ .lsm = handle }, - }; - }, - } + fn graphMetricScorePrefixAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64) ![]u8 { + var generation_buf: [20]u8 = undefined; + const generation_text = try std.fmt.bufPrint(&generation_buf, "{d}", .{generation}); + return try self.graphMetricKeyAlloc(&.{ metric_name, "score", generation_text }); } - fn openReverseStore(alloc: Allocator, reverse_path: [*:0]const u8, opts: GraphIndexOptions) !OpenedReverseStore { - return try openEdgeStore(alloc, reverse_path, opts); + fn graphMetricRankPrefixAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64) ![]u8 { + var generation_buf: [20]u8 = undefined; + const generation_text = try std.fmt.bufPrint(&generation_buf, "{d}", .{generation}); + return try self.graphMetricKeyAlloc(&.{ metric_name, "rank", generation_text }); } - /// Test/backward-compatible opener. The supplied store is ignored: graph - /// edges live in private forward/reverse stores rooted under reverse_path. - pub fn open(alloc: Allocator, main_store: anytype, reverse_path: [*:0]const u8, index_name: []const u8, opts: GraphIndexOptions) !GraphIndex { - _ = main_store; - const root = std.mem.span(reverse_path); - const outgoing_raw = try std.fmt.allocPrint(alloc, "{s}/forward", .{root}); - defer alloc.free(outgoing_raw); - const outgoing_path = try alloc.dupeZ(u8, outgoing_raw); - defer alloc.free(outgoing_path); - const reverse_raw = try std.fmt.allocPrint(alloc, "{s}/reverse", .{root}); - defer alloc.free(reverse_raw); - const private_reverse_path = try alloc.dupeZ(u8, reverse_raw); - defer alloc.free(private_reverse_path); - return try openWithPrivateStores(alloc, outgoing_path, private_reverse_path, index_name, opts); + fn graphMetricRankKeyAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64, score: f64, node: []const u8) ![]u8 { + if (!std.math.isFinite(score)) return error.InvalidGraphMetricScore; + var generation_buf: [20]u8 = undefined; + const generation_text = try std.fmt.bufPrint(&generation_buf, "{d}", .{generation}); + var descending_score: [8]u8 = undefined; + // Canonicalize signed zero so equal numeric scores remain grouped and + // the following node component provides the stable tie-break. + const canonical_score: f64 = if (score == 0.0) 0.0 else score; + const bits: u64 = @bitCast(canonical_score); + const ascending_bits = if (bits & (@as(u64, 1) << 63) != 0) ~bits else bits ^ (@as(u64, 1) << 63); + std.mem.writeInt(u64, &descending_score, ~ascending_bits, .big); + return try self.graphMetricKeyAlloc(&.{ metric_name, "rank", generation_text, &descending_score, node }); } - pub fn openWithPrivateStores(alloc: Allocator, outgoing_path: [*:0]const u8, reverse_path: [*:0]const u8, index_name: []const u8, opts: GraphIndexOptions) !GraphIndex { - var outgoing_store = try openEdgeStore(alloc, outgoing_path, opts); - errdefer { - outgoing_store.store.deinit(); - outgoing_store.owner.close(alloc); - } - var reverse_store = try openReverseStore(alloc, reverse_path, opts); - errdefer { - reverse_store.store.deinit(); - reverse_store.owner.close(alloc); - } - try outgoing_store.owner.ensureDurableEmptyManifest(); - try reverse_store.owner.ensureDurableEmptyManifest(); - const loaded_stats = try loadGraphCounters(&reverse_store.store); + fn graphMetricPublishedGenerationKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricKeyAlloc(&.{ metric_name, "published_generation" }); + } - return .{ - .alloc = alloc, - .index_name = index_name, - .outgoing_store = outgoing_store.store, - .outgoing_owner = outgoing_store.owner, - .reverse_store = reverse_store.store, - .reverse_owner = reverse_store.owner, - .edge_type_configs = opts.edge_type_configs, - .rebuild_root_path = if (opts.rebuild_root_path) |path| try alloc.dupe(u8, path) else null, - .rebuild_storage = opts.reverse_lsm_storage, - .rebuild_owner_generation = opts.rebuild_owner_generation, - .algebraic_semiring_traversal = opts.algebraic_semiring_traversal, - .edge_count = loaded_stats.edge_count, - .node_count = loaded_stats.node_count, - .algebraic_traversal_attempt_count = 0, - .algebraic_traversal_proven_count = 0, - .algebraic_traversal_rejected_count = 0, - .algebraic_traversal_fallback_count = 0, - .algebraic_traversal_result_node_count = 0, - }; + fn graphMetricScoreGenerationSequenceKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "score_generation_sequence" }); } - pub fn close(self: *GraphIndex) void { - self.outgoing_store.deinit(); - self.outgoing_owner.close(self.alloc); - self.reverse_store.deinit(); - self.reverse_owner.close(self.alloc); - if (self.rebuild_root_path) |path| self.alloc.free(path); - self.* = undefined; + fn graphMetricRetiredScoreGenerationKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "retired_score_generation" }); } - pub fn abandonAfterCrash(self: *GraphIndex) void { - self.outgoing_store.deinit(); - self.outgoing_owner.abandonAfterCrash(self.alloc); - self.reverse_store.deinit(); - self.reverse_owner.abandonAfterCrash(self.alloc); - if (self.rebuild_root_path) |path| self.alloc.free(path); - self.* = undefined; + fn graphMetricNextRetiredScoreGenerationKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "next_retired_score_generation" }); } - pub fn sync(self: *GraphIndex, force: bool) !void { - try self.outgoing_owner.sync(force); - try self.reverse_owner.sync(force); + fn graphMetricRetiredScoreCleanupPhaseKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "retired_score_cleanup_phase" }); } - pub fn syncReplayState(self: *GraphIndex) !void { - try self.outgoing_owner.sync(false); - try self.reverse_owner.sync(false); + fn graphMetricRetiredScoreCleanupCursorKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "retired_score_cleanup_cursor" }); } - pub fn supportsAlgebraicSemiringTraversal(self: *const GraphIndex) bool { - return self.algebraic_semiring_traversal; + fn graphMetricDirtyGenerationKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricKeyAlloc(&.{ metric_name, "dirty_generation" }); } - pub const AlgebraicTraversalRuntimeStats = struct { - attempt_count: u64 = 0, - proven_count: u64 = 0, - rejected_count: u64 = 0, - fallback_count: u64 = 0, - result_node_count: u64 = 0, - }; + fn graphMetricMaintenancePausedKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "maintenance_paused" }); + } - pub fn noteAlgebraicTraversalAttempt(self: *GraphIndex) void { - self.algebraic_traversal_attempt_count += 1; + fn graphMetricDisabledKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "disabled" }); } - pub fn noteAlgebraicTraversalProven(self: *GraphIndex, result_node_count: usize) void { - self.algebraic_traversal_proven_count += 1; - self.algebraic_traversal_result_node_count += @intCast(result_node_count); + fn graphMetricDeleteCleanupPhaseKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "delete_cleanup_phase" }); } - pub fn noteAlgebraicTraversalRejected(self: *GraphIndex) void { - self.algebraic_traversal_rejected_count += 1; + fn graphMetricDeleteCleanupCursorKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "delete_cleanup_cursor" }); } - pub fn noteAlgebraicTraversalFallback(self: *GraphIndex) void { - self.algebraic_traversal_fallback_count += 1; + fn graphMetricDeleteCleanupJobIdKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "delete_cleanup_job_id" }); } - pub fn algebraicTraversalRuntimeStats(self: *const GraphIndex) AlgebraicTraversalRuntimeStats { - return .{ - .attempt_count = self.algebraic_traversal_attempt_count, - .proven_count = self.algebraic_traversal_proven_count, - .rejected_count = self.algebraic_traversal_rejected_count, - .fallback_count = self.algebraic_traversal_fallback_count, - .result_node_count = self.algebraic_traversal_result_node_count, - }; + fn graphMetricBuildLeaseKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "build_lease" }); } - pub const Stats = struct { - edge_count: u64 = 0, - node_count: u64 = 0, - }; + fn graphMetricBuildJobKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "build_job" }); + } - pub fn stats(self: *GraphIndex, alloc: Allocator) !Stats { - _ = alloc; - if (self.edge_count == 0 and self.node_count == 0) { - const persisted = try loadGraphCounters(&self.reverse_store); - if (persisted.edge_count != 0 or persisted.node_count != 0) { - self.edge_count = persisted.edge_count; - self.node_count = persisted.node_count; - return persisted; - } - } - return .{ - .edge_count = self.edge_count, - .node_count = self.node_count, - }; + fn graphMetricFailedJobCleanupPrefixAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "failed_job_cleanup" }); } - pub fn scanStats(self: *GraphIndex, alloc: Allocator) !Stats { - var txn = try self.beginReadReverseTxn(); - defer txn.abort(); + fn graphMetricFailedJobCleanupKeyAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "failed_job_cleanup", job_id_text }); + } - var cur = try txn.openCursor(); - defer cur.close(); + fn graphMetricBuildManifestKeyAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "manifest" }); + } - var seen_nodes = std.StringHashMapUnmanaged(void).empty; - defer { - var it = seen_nodes.keyIterator(); - while (it.next()) |key| alloc.free(key.*); - seen_nodes.deinit(alloc); - } + fn graphMetricBuildPageKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + // Page keys are cursor-scanned. Fixed-width decimal preserves numeric + // ordering across digit boundaries without a heap allocation. + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d:0>20}", .{page_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "page", @tagName(phase), iteration_text, page_id_text }); + } - var first = (try cur.first()) orelse return .{}; - while (std.mem.startsWith(u8, first.key, graph_meta_prefix)) { - first = (try cur.next()) orelse return .{}; - } - var edge_count: u64 = 0; - try rememberStatsNode(alloc, &seen_nodes, first.key); - edge_count += 1; + fn graphMetricBuildPagePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "page", @tagName(phase), iteration_text }); + } - while (try cur.next()) |entry| { - if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; - try rememberStatsNode(alloc, &seen_nodes, entry.key); - edge_count += 1; - } - return .{ - .edge_count = edge_count, - .node_count = seen_nodes.count(), - }; + fn graphMetricBuildPhaseSummaryKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "phase", @tagName(phase), iteration_text }); } - fn rememberStatsNode( - alloc: Allocator, - seen_nodes: *std.StringHashMapUnmanaged(void), - key: []const u8, - ) !void { - var parsed = (try parseReverseEdgeKeyAlloc(alloc, key)) orelse return; - defer parsed.deinit(alloc); - try rememberStatsNodeValue(alloc, seen_nodes, parsed.source); - try rememberStatsNodeValue(alloc, seen_nodes, parsed.target); + fn graphMetricBuildPhaseProgressKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + // Computational progress belongs to the job artifact namespace, so all + // retirement paths remain one bounded cleanup protocol. Cleanup pages + // use their existing page/job cursor and never create these records. + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "progress", @tagName(phase), iteration_text }); } - fn rememberStatsNodeValue( - alloc: Allocator, - seen_nodes: *std.StringHashMapUnmanaged(void), - key: []const u8, - ) !void { - const result = try seen_nodes.getOrPut(alloc, key); - if (result.found_existing) return; - errdefer _ = seen_nodes.remove(key); - result.key_ptr.* = try alloc.dupe(u8, key); + fn graphMetricBuildIterationSummaryKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "iteration", iteration_text }); } - fn getTopologyMode(self: *const GraphIndex, edge_type: []const u8) TopologyMode { - for (self.edge_type_configs) |cfg| { - if (std.mem.eql(u8, cfg.name, edge_type)) return cfg.topology; - } - return .graph; + fn graphMetricBuildDegreePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "degree_partial" }); } - /// Add an edge (writes outgoing and reverse edge rows to private graph stores). - /// Returns TreeTopologyViolation if the edge type has tree topology and - /// the source already has an outgoing edge of that type to a different target. - pub fn addEdge( + fn graphMetricBuildDegreePartialKeyAlloc( self: *GraphIndex, - source: []const u8, - target: []const u8, - edge_type: []const u8, - weight: f64, - created_at: u64, - updated_at: u64, - metadata: []const u8, - ) !void { - // Tree topology: source can have at most one outgoing edge of this type - if (self.getTopologyMode(edge_type) == .tree) { - const existing = try self.getEdges(self.alloc, source, edge_type, .out); - defer freeEdges(self.alloc, existing); - for (existing) |e| { - if (!std.mem.eql(u8, e.target, target)) { - return TreeTopologyViolation.TreeTopologyViolation; - } - } - } + metric_name: []const u8, + job_id: u64, + node: []const u8, + page_id: u64, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const page_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{page_id}); + defer self.alloc.free(page_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "degree_partial", node, page_id_text }); + } - return try self.batchApply(&.{.{ - .source = source, - .target = target, - .edge_type = edge_type, - .weight = weight, - .created_at = created_at, - .updated_at = updated_at, - .metadata_json = metadata, - }}, &.{}); + fn graphMetricBuildAttemptPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + const page_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{page_id}); + defer self.alloc.free(page_id_text); + const attempt_text = try std.fmt.allocPrint(self.alloc, "{d}", .{attempt}); + defer self.alloc.free(attempt_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text }); } - pub fn batchApply(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete) !void { - if (writes.len == 0 and deletes.len == 0) return; + fn graphMetricBuildAttemptDegreePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const attempt_prefix = try self.graphMetricBuildAttemptPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(attempt_prefix); + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(self.alloc); + try list.appendSlice(self.alloc, attempt_prefix); + try internal_keys.appendEncodedComponent(&list, self.alloc, "degree_partial"); + return try list.toOwnedSlice(self.alloc); + } - // Validate the complete batch before opening either physical write - // batch, so invalid durable fields cannot partially mutate one - // direction or create records that the public graph wire contract - // cannot represent. - for (writes) |write| { - try edge_type_mod.validateStored(write.edge_type); - try edge_weight.validateStored(write.weight); - } - for (deletes) |delete| try edge_type_mod.validateStored(delete.edge_type); - try self.validateTreeBatchWrites(writes, deletes); + fn graphMetricBuildAttemptDegreePartialKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + var attempt_buf: [20]u8 = undefined; + const attempt_text = try std.fmt.bufPrint(&attempt_buf, "{d}", .{attempt}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text, "degree_partial", node }); + } - var main_batch = try self.beginWriteOutgoingBatch(); - errdefer main_batch.abort(); + fn graphMetricBuildAttemptPageRankContributionPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const attempt_prefix = try self.graphMetricBuildAttemptPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(attempt_prefix); + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(self.alloc); + try list.appendSlice(self.alloc, attempt_prefix); + try internal_keys.appendEncodedComponent(&list, self.alloc, "pagerank_contribution"); + return try list.toOwnedSlice(self.alloc); + } - var reverse_batch = try self.beginWriteReverseBatch(); - errdefer reverse_batch.abort(); - const prev_edge_count = self.edge_count; - const prev_node_count = self.node_count; - errdefer { - self.edge_count = prev_edge_count; - self.node_count = prev_node_count; - } + fn graphMetricBuildAttemptPageRankContributionKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + target_node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + var attempt_buf: [20]u8 = undefined; + const attempt_text = try std.fmt.bufPrint(&attempt_buf, "{d}", .{attempt}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text, "pagerank_contribution", target_node }); + } - for (deletes) |delete| { - const out_key = try edgeKeyAlloc(self.alloc, delete.source, self.index_name, delete.edge_type, delete.target); - defer self.alloc.free(out_key); - main_batch.delete(out_key) catch |err| switch (err) { - error.NotFound => {}, - else => return err, - }; + fn graphMetricBuildAttemptPageRankContributionChunkKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + checkpoint_offset: u64, + chunk_index: u64, + ) ![]u8 { + const prefix = try self.graphMetricBuildAttemptPageRankContributionPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(prefix); + var checkpoint_bytes: [8]u8 = undefined; + var chunk_bytes: [8]u8 = undefined; + std.mem.writeInt(u64, &checkpoint_bytes, checkpoint_offset, .big); + std.mem.writeInt(u64, &chunk_bytes, chunk_index, .big); + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(self.alloc); + try key.appendSlice(self.alloc, prefix); + try internal_keys.appendEncodedComponent(&key, self.alloc, &checkpoint_bytes); + try internal_keys.appendEncodedComponent(&key, self.alloc, &chunk_bytes); + return try key.toOwnedSlice(self.alloc); + } - const rev_key = try reverseEdgeKeyAlloc(self.alloc, delete.target, self.index_name, delete.edge_type, delete.source); - defer self.alloc.free(rev_key); - try self.accountReverseDelete(&reverse_batch, delete.source, delete.target, rev_key); - reverse_batch.delete(rev_key) catch |err| switch (err) { - error.NotFound => {}, - else => return err, - }; - } + fn graphMetricBuildAttemptPageRankOutDegreePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const attempt_prefix = try self.graphMetricBuildAttemptPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(attempt_prefix); + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(self.alloc); + try list.appendSlice(self.alloc, attempt_prefix); + try internal_keys.appendEncodedComponent(&list, self.alloc, "pagerank_out_degree"); + return try list.toOwnedSlice(self.alloc); + } - for (writes) |write| { - const edge_val = try encodeEdgeValueAlloc( - self.alloc, - write.weight, - write.created_at, - write.updated_at, - write.metadata_json, - ); - defer self.alloc.free(edge_val); + fn graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + var attempt_buf: [20]u8 = undefined; + const attempt_text = try std.fmt.bufPrint(&attempt_buf, "{d}", .{attempt}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text, "pagerank_out_degree", node }); + } - const out_key = try edgeKeyAlloc(self.alloc, write.source, self.index_name, write.edge_type, write.target); - defer self.alloc.free(out_key); - try main_batch.put(out_key, edge_val); + fn graphMetricBuildAttemptPageRankNodePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const attempt_prefix = try self.graphMetricBuildAttemptPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(attempt_prefix); + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(self.alloc); + try list.appendSlice(self.alloc, attempt_prefix); + try internal_keys.appendEncodedComponent(&list, self.alloc, "pagerank_node"); + return try list.toOwnedSlice(self.alloc); + } - const rev_key = try reverseEdgeKeyAlloc(self.alloc, write.target, self.index_name, write.edge_type, write.source); - defer self.alloc.free(rev_key); - try self.accountReverseInsert(&reverse_batch, write.source, write.target, rev_key); - try reverse_batch.put(rev_key, edge_val); - } + fn graphMetricBuildAttemptPageRankNodePartialKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + var attempt_buf: [20]u8 = undefined; + const attempt_text = try std.fmt.bufPrint(&attempt_buf, "{d}", .{attempt}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text, "pagerank_node", node }); + } - try self.persistGraphCounters(&reverse_batch); - try main_batch.commit(); - try reverse_batch.commit(); + fn graphMetricBuildAttemptHitsHubRawPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + ) ![]u8 { + const attempt_prefix = try self.graphMetricBuildAttemptPrefixAlloc(metric_name, job_id, phase, iteration, page_id, attempt); + defer self.alloc.free(attempt_prefix); + var list = std.ArrayListUnmanaged(u8).empty; + defer list.deinit(self.alloc); + try list.appendSlice(self.alloc, attempt_prefix); + try internal_keys.appendEncodedComponent(&list, self.alloc, "hits_hub_raw"); + return try list.toOwnedSlice(self.alloc); } - /// Delete an edge (removes from both private graph stores). - pub fn deleteEdge(self: *GraphIndex, source: []const u8, target: []const u8, edge_type: []const u8) !void { - return try self.batchApply(&.{}, &.{.{ - .source = source, - .target = target, - .edge_type = edge_type, - }}); + fn graphMetricBuildAttemptHitsHubRawKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + source_node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + var attempt_buf: [20]u8 = undefined; + const attempt_text = try std.fmt.bufPrint(&attempt_buf, "{d}", .{attempt}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "attempt", @tagName(phase), iteration_text, page_id_text, attempt_text, "hits_hub_raw", source_node }); } - /// Get edges connected to a key. Caller owns the returned slice and edge data. - pub fn getEdges(self: *GraphIndex, alloc: Allocator, key: []const u8, edge_type: []const u8, direction: EdgeDirection) ![]Edge { - var results = std.ArrayListUnmanaged(Edge).empty; - errdefer { - for (results.items) |e| freeEdge(alloc, e); - results.deinit(alloc); - } + fn graphMetricBuildPageRankOutDegreePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_out_degree" }); + } - if (direction == .out or direction == .both) { - try self.scanOutgoingEdges(alloc, &results, key, edge_type); - } - if (direction == .in or direction == .both) { - try self.scanIncomingEdges(alloc, &results, key, edge_type, direction == .both); - } + fn graphMetricBuildPageRankOutDegreePartialKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + node: []const u8, + page_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_out_degree", node, page_id_text }); + } - return try results.toOwnedSlice(alloc); + fn graphMetricBuildPageRankOutDegreeKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + node: []const u8, + ) ![]u8 { + return graphMetricBuildPageRankOutDegreeKeyWithAllocator(self.alloc, metric_name, job_id, node); } - /// Get edges connected to a key, restricting storage scans to the requested - /// relationship types. An empty type list retains the unfiltered behavior. - pub fn getEdgesByTypes(self: *GraphIndex, alloc: Allocator, key: []const u8, edge_types: []const []const u8, direction: EdgeDirection) ![]Edge { - if (edge_types.len == 0) return try self.getEdges(alloc, key, "", direction); + fn graphMetricBuildPageRankOutDegreeKeyWithAllocator( + alloc: Allocator, + metric_name: []const u8, + job_id: u64, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + return try graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", job_id_text, "pagerank_out_degree_total", node }); + } - var results = std.ArrayListUnmanaged(Edge).empty; - errdefer { - for (results.items) |e| freeEdge(alloc, e); - results.deinit(alloc); - } - for (edge_types, 0..) |edge_type, type_index| { - var duplicate = false; - for (edge_types[0..type_index]) |prior| { - if (std.mem.eql(u8, edge_type, prior)) { - duplicate = true; - break; - } - } - if (duplicate) continue; - if (direction == .out or direction == .both) { - try self.scanOutgoingEdges(alloc, &results, key, edge_type); - } - if (direction == .in or direction == .both) { - try self.scanIncomingEdges(alloc, &results, key, edge_type, direction == .both); - } - } - return try results.toOwnedSlice(alloc); + fn graphMetricBuildPageRankNodePartialPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_node" }); } - /// Read one bounded page of an adjacency in the same deterministic order - /// as `getEdgesByTypes`. The cursor contains logical edge identity rather - /// than backend state, so a caller can resume against the same pinned DB - /// generation across an internal RPC boundary. - pub fn getEdgesByTypesPage( + fn graphMetricBuildPageRankNodePartialKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + node: []const u8, + page_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d}", .{page_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_node", node, page_id_text }); + } + + fn graphMetricBuildPageRankKeyAlloc( self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + return graphMetricBuildPageRankKeyWithAllocator(self.alloc, metric_name, job_id, iteration, node); + } + + fn graphMetricBuildPageRankKeyWithAllocator( alloc: Allocator, - key: []const u8, - edge_types: []const []const u8, - direction: EdgeDirection, - scan_cursor: ?EdgeScanCursor, - limits: EdgePageLimits, - ) !EdgePage { - if (limits.max_edges == 0) return error.GraphExploredEdgesBudgetExceeded; - if (limits.max_owned_bytes == 0) return error.GraphExploredEdgeBytesBudgetExceeded; + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", job_id_text, "pagerank_rank", iteration_text, node }); + } - var results = std.ArrayListUnmanaged(Edge).empty; - errdefer { - for (results.items) |edge| freeEdge(alloc, edge); - results.deinit(alloc); - } - var owned_bytes: usize = 0; - const type_count: usize = if (edge_types.len == 0) 1 else edge_types.len; - var type_index: usize = if (scan_cursor) |cursor| cursor.type_index else 0; - if (type_index >= type_count) return error.InvalidArgument; - if (scan_cursor) |cursor| { - if (cursor.direction == .both or - (direction == .out and cursor.direction != .out) or - (direction == .in and cursor.direction != .in) or - (edge_types.len > 0 and !std.mem.eql(u8, cursor.edge_type, edge_types[type_index]))) - return error.InvalidArgument; - if (edge_types.len > 0) { - for (edge_types[0..type_index]) |prior| { - if (std.mem.eql(u8, edge_types[type_index], prior)) return error.InvalidArgument; - } - } - } + fn graphMetricBuildPageRankSourceFactorKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + return graphMetricBuildPageRankSourceFactorKeyWithAllocator(self.alloc, metric_name, job_id, iteration, node); + } - while (type_index < type_count) : (type_index += 1) { - if (edge_types.len > 0) { - var duplicate = false; - for (edge_types[0..type_index]) |prior| { - if (std.mem.eql(u8, edge_types[type_index], prior)) { - duplicate = true; - break; - } - } - if (duplicate) continue; - } - const requested_type = if (edge_types.len == 0) "" else edge_types[type_index]; - var phase: EdgeDirection = if (scan_cursor) |cursor| - if (cursor.type_index == type_index) cursor.direction else firstScanDirection(direction) - else - firstScanDirection(direction); + fn graphMetricBuildPageRankSourceFactorKeyWithAllocator( + alloc: Allocator, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", job_id_text, "pagerank_source_factor", iteration_text, node }); + } - while (true) { - if (results.items.len >= limits.max_edges or owned_bytes >= limits.max_owned_bytes) { - var next_cursor = try edgeScanStartCursor( - alloc, - phase, - @intCast(type_index), - requested_type, - ); - errdefer next_cursor.deinit(alloc); - return .{ - .edges = try results.toOwnedSlice(alloc), - .next_cursor = next_cursor, - .owned_bytes = owned_bytes, - }; - } - const active_resume = if (scan_cursor) |cursor| - cursor.type_index == type_index and cursor.direction == phase - else - false; - const capped = try self.scanEdgePagePhase( - alloc, - &results, - &owned_bytes, - key, - requested_type, - @intCast(type_index), - phase, - if (active_resume) scan_cursor else null, - direction == .both and phase == .in, - limits, - ); - if (capped) |cursor| { - return .{ - .edges = try results.toOwnedSlice(alloc), - .next_cursor = cursor, - .owned_bytes = owned_bytes, - }; - } - if (direction != .both or phase == .in) break; - phase = .in; - } - } + fn graphMetricBuildPageRankContributionKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + target_node: []const u8, + page_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d:0>20}", .{page_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_contribution", iteration_text, target_node, page_id_text }); + } - return .{ - .edges = try results.toOwnedSlice(alloc), - .owned_bytes = owned_bytes, - }; + fn graphMetricBuildPageRankContributionCheckpointKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + target_node: []const u8, + page_id: u64, + checkpoint_offset: u64, + chunk_index: u64, + ) ![]u8 { + const prefix = try self.graphMetricBuildPageRankContributionKeyAlloc(metric_name, job_id, iteration, target_node, page_id); + defer self.alloc.free(prefix); + var checkpoint_bytes: [8]u8 = undefined; + var chunk_bytes: [8]u8 = undefined; + std.mem.writeInt(u64, &checkpoint_bytes, checkpoint_offset, .big); + std.mem.writeInt(u64, &chunk_bytes, chunk_index, .big); + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(self.alloc); + try key.appendSlice(self.alloc, prefix); + try internal_keys.appendEncodedComponent(&key, self.alloc, &checkpoint_bytes); + try internal_keys.appendEncodedComponent(&key, self.alloc, &chunk_bytes); + return try key.toOwnedSlice(self.alloc); } - /// Materialize an adjacency only while it fits the caller's explicit - /// request budget. Page-sized scans ensure the budget is checked before a - /// high-degree node can force unbounded allocation. - pub fn getEdgesByTypesBounded( + fn graphMetricBuildPageRankContributionNodePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + target_node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_contribution", iteration_text, target_node }); + } + + fn graphMetricBuildPageRankContributionPrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "pagerank_contribution", iteration_text }); + } + + fn graphMetricBuildHitsRankKeyAlloc( self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + vector_name: []const u8, + iteration: u32, + node: []const u8, + ) ![]u8 { + return graphMetricBuildHitsRankKeyWithAllocator(self.alloc, metric_name, job_id, vector_name, iteration, node); + } + + fn graphMetricBuildHitsRankKeyWithAllocator( alloc: Allocator, - key: []const u8, - edge_types: []const []const u8, - direction: EdgeDirection, - max_edges: usize, - max_owned_bytes: usize, - ) ![]Edge { - const page_edge_cap: usize = 4096; - const page_byte_cap: usize = 4 * 1024 * 1024; - var results = std.ArrayListUnmanaged(Edge).empty; - errdefer { - for (results.items) |edge| freeEdge(alloc, edge); - results.deinit(alloc); - } - var total_bytes: usize = 0; - var cursor: ?EdgeScanCursor = null; - defer if (cursor) |*value| value.deinit(alloc); + metric_name: []const u8, + job_id: u64, + vector_name: []const u8, + iteration: u32, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", job_id_text, "hits_rank", vector_name, iteration_text, node }); + } - while (true) { - const edge_room = if (results.items.len < max_edges) max_edges - results.items.len else 0; - const byte_room = if (total_bytes < max_owned_bytes) max_owned_bytes - total_bytes else 0; - var page = try self.getEdgesByTypesPage( - alloc, - key, - edge_types, - direction, - cursor, - .{ - .max_edges = @min(page_edge_cap, std.math.add(usize, edge_room, 1) catch std.math.maxInt(usize)), - .max_owned_bytes = @min(page_byte_cap, @max(byte_room, 1)), - }, - ); - if (cursor) |*value| value.deinit(alloc); - cursor = null; - errdefer page.deinit(alloc); + fn graphMetricBuildHitsRankNamespacePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_rank" }); + } - const next_count = std.math.add(usize, results.items.len, page.edges.len) catch - return error.GraphExploredEdgesBudgetExceeded; - const next_bytes = std.math.add(usize, total_bytes, page.owned_bytes) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - if (next_count > max_edges) return error.GraphExploredEdgesBudgetExceeded; - if (next_bytes > max_owned_bytes) return error.GraphExploredEdgeBytesBudgetExceeded; + fn graphMetricBuildHitsHubRawKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + page_id: u64, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + var page_id_buf: [20]u8 = undefined; + const page_id_text = try std.fmt.bufPrint(&page_id_buf, "{d:0>20}", .{page_id}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw", iteration_text, node, page_id_text }); + } - try results.ensureUnusedCapacity(alloc, page.edges.len); - for (page.edges) |edge| results.appendAssumeCapacity(edge); - alloc.free(page.edges); - page.edges = @constCast((&[_]Edge{})[0..]); - total_bytes = next_bytes; - cursor = page.next_cursor; - page.next_cursor = null; - page.deinit(alloc); - if (cursor == null) break; - } - return try results.toOwnedSlice(alloc); + fn graphMetricBuildHitsHubRawNodePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw", iteration_text, node }); } - fn firstScanDirection(direction: EdgeDirection) EdgeDirection { - return if (direction == .in) .in else .out; + fn graphMetricBuildHitsHubRawTotalKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) ![]u8 { + var job_id_buf: [20]u8 = undefined; + const job_id_text = try std.fmt.bufPrint(&job_id_buf, "{d}", .{job_id}); + var iteration_buf: [10]u8 = undefined; + const iteration_text = try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw_total", iteration_text, node }); } - fn scanEdgePagePhase( + fn graphMetricBuildHitsHubRawPrefixAlloc( self: *GraphIndex, - alloc: Allocator, - results: *std.ArrayListUnmanaged(Edge), - owned_bytes: *usize, - key: []const u8, - requested_type: []const u8, - type_index: u32, - phase: EdgeDirection, - scan_cursor: ?EdgeScanCursor, - skip_mirrored_self_loops: bool, - limits: EdgePageLimits, - ) !?EdgeScanCursor { - std.debug.assert(phase != .both); - const phase_start_len = results.items.len; - const prefix = if (phase == .out) - try edgePrefixAlloc(alloc, key, self.index_name, requested_type) - else - try reverseEdgePrefixAlloc(alloc, key, self.index_name, requested_type); - defer alloc.free(prefix); + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw", iteration_text }); + } - const resume_key = if (scan_cursor) |cursor| - if (cursor.at_phase_start) - null - else if (phase == .out) - try edgeKeyAlloc(alloc, key, self.index_name, cursor.edge_type, cursor.adjacent_key) - else - try reverseEdgeKeyAlloc(alloc, key, self.index_name, cursor.edge_type, cursor.adjacent_key) - else - null; - defer if (resume_key) |value| alloc.free(value); + fn graphMetricBuildHitsHubRawNamespacePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw" }); + } - var txn = if (phase == .out) try self.beginReadOutgoingTxn() else try self.beginReadReverseTxn(); - defer txn.abort(); - var cursor = try txn.openCursor(); - defer cursor.close(); + fn graphMetricBuildHitsHubRawSummaryKeyAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + const iteration_text = try std.fmt.allocPrint(self.alloc, "{d}", .{iteration}); + defer self.alloc.free(iteration_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw_summary", iteration_text }); + } - var entry = (try cursor.seekAtOrAfter(resume_key orelse prefix)) orelse return null; - if (resume_key) |value| { - if (std.mem.eql(u8, entry.key, value)) entry = (try cursor.next()) orelse return null; - } + fn graphMetricBuildHitsHubRawSummaryNamespacePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text, "hits_hub_raw_summary" }); + } - while (std.mem.startsWith(u8, entry.key, prefix)) { - if (results.items.len >= limits.max_edges) - return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, results.items[results.items.len - 1]); + fn graphMetricFailureKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "last_failure" }); + } - const before = results.items.len; - if (phase == .out) - try appendEdgeFromKV(alloc, results, entry.key, entry.value) - else - try appendReverseEdgeFromKV(alloc, results, entry.key, entry.value, skip_mirrored_self_loops); - if (results.items.len != before) { - const appended = results.items[results.items.len - 1]; - const edge_bytes = edgeOwnedBytes(appended); - const next_bytes = std.math.add(usize, owned_bytes.*, edge_bytes) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - if (next_bytes > limits.max_owned_bytes) { - _ = results.pop(); - freeEdge(alloc, appended); - if (results.items.len == 0) return error.GraphExploredEdgeBytesBudgetExceeded; - if (results.items.len == phase_start_len) - return try edgeScanStartCursor(alloc, phase, type_index, requested_type); - return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, results.items[results.items.len - 1]); - } - owned_bytes.* = next_bytes; - if (results.items.len >= limits.max_edges) - return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, appended); - } - entry = (try cursor.next()) orelse break; - } - return null; + fn graphMetricFailureSequenceKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "failure_sequence" }); } - /// Resolve exact physical relationships with one snapshot and one sorted - /// backend multi-get. Results remain aligned with `probes`; null means the - /// relationship does not exist. Only found edges allocate edge payloads. - pub fn probeEdgesAlloc(self: *GraphIndex, alloc: Allocator, probes: []const EdgeProbe) ![]?Edge { - return try self.probeEdgesAllocBounded(alloc, probes, std.math.maxInt(usize)); + fn graphMetricFailureRecordKeyAlloc(self: *GraphIndex, metric_name: []const u8, sequence: u64) ![]u8 { + const sequence_text = try std.fmt.allocPrint(self.alloc, "{d}", .{sequence}); + defer self.alloc.free(sequence_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "failure", sequence_text }); } - /// Resolve exact physical relationships without allowing decoded edge - /// payloads to exceed the caller's remaining request budget. The limit is - /// checked against values borrowed from the read transaction before any - /// found edge payload is copied into caller-owned memory. - pub fn probeEdgesAllocBounded( - self: *GraphIndex, - alloc: Allocator, - probes: []const EdgeProbe, - max_owned_bytes: usize, - ) ![]?Edge { - const ProbeKey = struct { - encoded: []u8, - result_index: usize, + fn graphMetricEventSequenceKeyAlloc(self: *GraphIndex, metric_name: []const u8) ![]u8 { + return try self.graphMetricKeyAlloc(&.{ metric_name, "event_sequence" }); + } - fn lessThan(_: void, left: @This(), right: @This()) bool { - return std.mem.order(u8, left.encoded, right.encoded) == .lt; - } + fn graphMetricEventKeyAlloc(self: *GraphIndex, metric_name: []const u8, sequence: u64) ![]u8 { + const sequence_text = try std.fmt.allocPrint(self.alloc, "{d}", .{sequence}); + defer self.alloc.free(sequence_text); + return try self.graphMetricKeyAlloc(&.{ metric_name, "event", sequence_text }); + } + + fn graphMetricMetaKeyAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64) ![]u8 { + const generation_text = try std.fmt.allocPrint(self.alloc, "{d}", .{generation}); + defer self.alloc.free(generation_text); + return try self.graphMetricKeyAlloc(&.{ metric_name, "meta", generation_text }); + } + + fn graphMetricMetaEdgeFilterKeyAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64) ![]u8 { + const generation_text = try std.fmt.allocPrint(self.alloc, "{d}", .{generation}); + defer self.alloc.free(generation_text); + return try self.graphMetricKeyAlloc(&.{ metric_name, "meta_edge_filter", generation_text }); + } + + fn graphMetricMetaConfigFingerprintKeyAlloc(self: *GraphIndex, metric_name: []const u8, generation: u64) ![]u8 { + const generation_text = try std.fmt.allocPrint(self.alloc, "{d}", .{generation}); + defer self.alloc.free(generation_text); + return try self.graphMetricKeyAlloc(&.{ metric_name, "meta_config_fingerprint", generation_text }); + } + + fn metricPublishedGeneration(self: *GraphIndex, txn: anytype, metric_name: []const u8) !u64 { + const key = try self.graphMetricPublishedGenerationKeyAlloc(metric_name); + defer self.alloc.free(key); + return try readU64OrZero(txn, key); + } + + /// Translate the internal score-namespace pointer into the public edge + /// snapshot generation. Pre-v4 materializations used the same value for + /// both and remain readable without migration. + fn metricPublishedEdgeGeneration(self: *GraphIndex, txn: anytype, metric_name: []const u8) !u64 { + const score_generation = try self.metricPublishedGeneration(txn, metric_name); + if (score_generation == 0) return 0; + + const meta_key = try self.graphMetricMetaKeyAlloc(metric_name, score_generation); + defer self.alloc.free(meta_key); + const raw = txn.get(meta_key) catch |err| switch (err) { + error.NotFound => return score_generation, + else => return err, }; + const meta = decodeGraphMetricMeta(raw) orelse return score_generation; + return if (meta.target_edge_generation != 0) + meta.target_edge_generation + else + score_generation; + } - const results = try alloc.alloc(?Edge, probes.len); - errdefer alloc.free(results); - @memset(results, null); - if (probes.len == 0) return results; + /// Allocate a private, durable score namespace. Edge generations identify + /// the input snapshot; they are deliberately not reused as storage epochs + /// because a config-only rebuild can target the same edge snapshot. + fn allocateGraphMetricScoreGenerationInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + cfg: GraphMetricConfig, + target_edge_generation: u64, + ) !u64 { + const sequence_key = try self.graphMetricScoreGenerationSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + var greatest = @max( + try readU64OrZero(batch, sequence_key), + try self.metricPublishedGeneration(batch, metric_name), + ); - const keys = try alloc.alloc(ProbeKey, probes.len); - var initialized_keys: usize = 0; - defer { - for (keys[0..initialized_keys]) |item| alloc.free(item.encoded); - alloc.free(keys); - } - for (probes, 0..) |probe, i| { - keys[i] = .{ - .encoded = try edgeKeyAlloc(alloc, probe.source, self.index_name, probe.edge_type, probe.target), - .result_index = i, - }; - initialized_keys += 1; + const pair_cfg = self.pairedHitsMetricConfig(cfg); + var pair_sequence_key: ?[]u8 = null; + defer if (pair_sequence_key) |key| self.alloc.free(key); + if (pair_cfg) |pair| { + const key = try self.graphMetricScoreGenerationSequenceKeyAlloc(pair.name); + pair_sequence_key = key; + greatest = @max(greatest, try readU64OrZero(batch, key)); + greatest = @max(greatest, try self.metricPublishedGeneration(batch, pair.name)); } - std.mem.sort(ProbeKey, keys, {}, ProbeKey.lessThan); + const next = if (greatest < target_edge_generation) + target_edge_generation + else blk: { + if (greatest == std.math.maxInt(u64)) return error.GraphMetricGenerationExhausted; + break :blk greatest + 1; + }; + try putU64(batch, sequence_key, next); + if (pair_sequence_key) |key| try putU64(batch, key, next); + return next; + } - const sorted_key_refs = try alloc.alloc([]const u8, keys.len); - defer alloc.free(sorted_key_refs); - for (keys, 0..) |item, i| sorted_key_refs[i] = item.encoded; - const values = try alloc.alloc(?[]const u8, keys.len); - defer alloc.free(values); + fn metricDirtyGeneration(self: *GraphIndex, txn: anytype, metric_name: []const u8) !u64 { + const key = try self.graphMetricDirtyGenerationKeyAlloc(metric_name); + defer self.alloc.free(key); + return try readU64OrZero(txn, key); + } - var txn = try self.beginReadOutgoingTxn(); - defer txn.abort(); - try txn.getManySorted(sorted_key_refs, values); + fn metricMaintenancePaused(self: *GraphIndex, txn: anytype, metric_name: []const u8) !bool { + const key = try self.graphMetricMaintenancePausedKeyAlloc(metric_name); + defer self.alloc.free(key); + return (try readU64OrZero(txn, key)) != 0; + } - errdefer { - for (results) |maybe_edge| if (maybe_edge) |edge| freeEdge(alloc, edge); - } - var owned_bytes: usize = 0; - for (keys, values) |item, maybe_value| { - const value = maybe_value orelse continue; - const decoded = try decodeEdgeValue(value); - const probe = probes[item.result_index]; - var edge_bytes: usize = @sizeOf(Edge); - edge_bytes = std.math.add(usize, edge_bytes, probe.source.len) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - edge_bytes = std.math.add(usize, edge_bytes, probe.target.len) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - edge_bytes = std.math.add(usize, edge_bytes, probe.edge_type.len) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - edge_bytes = std.math.add(usize, edge_bytes, decoded.metadata.len) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - owned_bytes = std.math.add(usize, owned_bytes, edge_bytes) catch - return error.GraphExploredEdgeBytesBudgetExceeded; - if (owned_bytes > max_owned_bytes) return error.GraphExploredEdgeBytesBudgetExceeded; - const source = try alloc.dupe(u8, probe.source); - errdefer alloc.free(source); - const target = try alloc.dupe(u8, probe.target); - errdefer alloc.free(target); - const edge_type = try alloc.dupe(u8, probe.edge_type); - errdefer alloc.free(edge_type); - const metadata = if (decoded.metadata.len > 0) - try alloc.dupe(u8, decoded.metadata) - else - ""; - errdefer if (metadata.len > 0) alloc.free(metadata); - results[item.result_index] = .{ - .source = source, - .target = target, - .edge_type = edge_type, - .weight = decoded.weight, - .created_at = decoded.created_at, - .updated_at = decoded.updated_at, - .metadata = metadata, - }; - } - return results; + fn metricDisabled(self: *GraphIndex, txn: anytype, metric_name: []const u8) !bool { + const key = try self.graphMetricDisabledKeyAlloc(metric_name); + defer self.alloc.free(key); + return (try readU64OrZero(txn, key)) != 0; } - pub fn freeProbedEdges(alloc: Allocator, edges: []?Edge) void { - for (edges) |maybe_edge| if (maybe_edge) |edge| freeEdge(alloc, edge); - alloc.free(edges); + fn metricBuildLease(self: *GraphIndex, txn: anytype, metric_name: []const u8) !?GraphMetricBuildLease { + const key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildLease(raw); } - /// Probe incoming-edge existence for a key batch using one reverse-store - /// snapshot and one cursor. Results are aligned with `keys`. - pub fn hasIncomingEdgesManyAlloc( - self: *GraphIndex, - alloc: Allocator, - keys: []const []const u8, - ) ![]bool { - const result = try alloc.alloc(bool, keys.len); - errdefer alloc.free(result); - @memset(result, false); - if (keys.len == 0) return result; - - var txn = try self.beginReadReverseTxn(); - defer txn.abort(); - var cursor = try txn.openCursor(); - defer cursor.close(); - - for (keys, 0..) |key, i| { - const prefix = try reverseEdgePrefixAlloc(alloc, key, self.index_name, ""); - defer alloc.free(prefix); - const first = (try cursor.seekAtOrAfter(prefix)) orelse continue; - result[i] = std.mem.startsWith(u8, first.key, prefix); - } - return result; + fn metricBuildJob(self: *GraphIndex, txn: anytype, metric_name: []const u8) !?GraphMetricBuildJob { + const key = try self.graphMetricBuildJobKeyAlloc(metric_name); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildJob(raw); } - fn scanOutgoingEdges(self: *GraphIndex, alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), key: []const u8, edge_type: []const u8) !void { - const prefix = try edgePrefixAlloc(alloc, key, self.index_name, edge_type); - defer alloc.free(prefix); + fn cloneGraphMetricBuildJobAlloc(self: *GraphIndex, job: GraphMetricBuildJob) !GraphMetricBuildJob { + var cloned = job; + cloned.worker_id = try self.alloc.dupe(u8, job.worker_id); + errdefer self.alloc.free(cloned.worker_id); + cloned.last_error = try self.alloc.dupe(u8, job.last_error); + errdefer self.alloc.free(cloned.last_error); + cloned.cursor = try self.alloc.dupe(u8, job.cursor); + return cloned; + } - var txn = try self.beginReadOutgoingTxn(); - defer txn.abort(); - var cur = try txn.openCursor(); - defer cur.close(); + fn deinitClonedGraphMetricBuildJob(self: *GraphIndex, job: GraphMetricBuildJob) void { + self.alloc.free(job.worker_id); + self.alloc.free(job.last_error); + self.alloc.free(job.cursor); + } - const first = (try cur.seekAtOrAfter(prefix)) orelse return; - if (!std.mem.startsWith(u8, first.key, prefix)) return; - try appendEdgeFromKV(alloc, results, first.key, first.value); - while (try cur.next()) |entry| { - if (!std.mem.startsWith(u8, entry.key, prefix)) break; - try appendEdgeFromKV(alloc, results, entry.key, entry.value); - } + fn metricBuildManifest(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64) !?GraphMetricBuildManifest { + const key = try self.graphMetricBuildManifestKeyAlloc(metric_name, job_id); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildManifest(raw); } - fn scanIncomingEdges( + fn metricBuildPage( self: *GraphIndex, - alloc: Allocator, - results: *std.ArrayListUnmanaged(Edge), - key: []const u8, - edge_type: []const u8, - skip_mirrored_self_loops: bool, - ) !void { - const prefix = try reverseEdgePrefixAlloc(alloc, key, self.index_name, edge_type); - defer alloc.free(prefix); + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + ) !?GraphMetricBuildPage { + const key = try self.graphMetricBuildPageKeyAlloc(metric_name, job_id, phase, iteration, page_id); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildPage(raw); + } + pub fn graphMetricBuildPageSnapshotForTest( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + ) !?GraphMetricBuildPageSnapshotForTest { var txn = try self.beginReadReverseTxn(); defer txn.abort(); + const page = try self.metricBuildPage(&txn, metric_name, job_id, phase, iteration, page_id) orelse return null; + return .{ + .state = page.state, + .worker_id_hash = std.hash.Wyhash.hash(0, page.worker_id), + .attempt = page.attempt, + .completed_units = page.completed_units, + .total_units = page.total_units, + .output_fingerprint = page.output_fingerprint, + }; + } - var cur = try txn.openCursor(); - defer cur.close(); - - const first = (try cur.seekAtOrAfter(prefix)) orelse return; - - if (std.mem.startsWith(u8, first.key, prefix)) { - try appendReverseEdgeFromKV(alloc, results, first.key, first.value, skip_mirrored_self_loops); - } else { - return; - } + fn metricBuildPhaseSummary( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !?GraphMetricBuildPhaseSummary { + const key = try self.graphMetricBuildPhaseSummaryKeyAlloc(metric_name, job_id, phase, iteration); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildPhaseSummary(raw); + } - while (try cur.next()) |entry| { - if (!std.mem.startsWith(u8, entry.key, prefix)) break; - try appendReverseEdgeFromKV(alloc, results, entry.key, entry.value, skip_mirrored_self_loops); - } + fn metricBuildPhaseProgress( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !?GraphMetricBuildPhaseProgress { + const key = try self.graphMetricBuildPhaseProgressKeyAlloc(metric_name, job_id, phase, iteration); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildPhaseProgress(raw); } - fn appendEdgeFromKV(alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), key: []const u8, value: []const u8) !void { - var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, key)) orelse return; - defer parsed.deinit(alloc); - try appendParsedEdge(alloc, results, parsed, value); + fn metricBuildIterationSummary( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) !?GraphMetricBuildIterationSummary { + const key = try self.graphMetricBuildIterationSummaryKeyAlloc(metric_name, job_id, iteration); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricBuildIterationSummary(raw); } - fn appendReverseEdgeFromKV( - alloc: Allocator, - results: *std.ArrayListUnmanaged(Edge), - key: []const u8, - value: []const u8, - skip_mirrored_self_loops: bool, - ) !void { - var parsed = (try parseReverseEdgeKeyAlloc(alloc, key)) orelse return; - defer parsed.deinit(alloc); - // A physical self-loop is indexed once in each adjacency direction so - // independent `out` and `in` reads remain complete. A `both` read has - // already emitted the outgoing copy, so suppress only its mirrored - // reverse-index representation. Reciprocal non-self edges remain - // distinct because their physical source/target identities differ. - if (skip_mirrored_self_loops and std.mem.eql(u8, parsed.source, parsed.target)) return; - try appendParsedEdge(alloc, results, parsed, value); + fn metricFailureDetail(self: *GraphIndex, txn: anytype, metric_name: []const u8) !?GraphMetricFailureDetail { + const key = try self.graphMetricFailureKeyAlloc(metric_name); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return try decodeGraphMetricFailureDetailAlloc(self.alloc, raw); } - fn appendParsedEdge(alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), parsed: ParsedGraphEdgeKey, value: []const u8) !void { - const decoded = try decodeEdgeValue(value); - const source = try alloc.dupe(u8, parsed.source); - errdefer alloc.free(source); - const target = try alloc.dupe(u8, parsed.target); - errdefer alloc.free(target); - const edge_type = try alloc.dupe(u8, parsed.edge_type); - errdefer alloc.free(edge_type); - const metadata = if (decoded.metadata.len > 0) try alloc.dupe(u8, decoded.metadata) else ""; - errdefer if (metadata.len > 0) alloc.free(metadata); - try results.append(alloc, .{ - .source = source, - .target = target, - .edge_type = edge_type, - .weight = decoded.weight, - .created_at = decoded.created_at, - .updated_at = decoded.updated_at, - .metadata = metadata, - }); + fn acquireGraphMetricBuildLease( + self: *GraphIndex, + metric_name: []const u8, + target_generation: u64, + ) !void { + return self.acquireGraphMetricBuildLeaseWithPlanning(metric_name, target_generation, true); } - /// Delete all outgoing edges for a document (cleanup on doc deletion). - pub fn deleteEdgesForDoc(self: *GraphIndex, doc_key: []const u8) !void { - const edges = try self.getEdges(self.alloc, doc_key, "", .both); - defer freeEdges(self.alloc, edges); + fn acquireGraphMetricBuildLeaseWithPlanning(self: *GraphIndex, metric_name: []const u8, target_generation: u64, comptime drain: bool) !void { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const observed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + // Avoid the boundary scan for the common duplicate-scheduler case. + // The write transaction below repeats both checks to close the race. + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.metricDisabled(&txn, metric_name)) return error.GraphMetricDisabled; + if (try self.metricBuildLease(&txn, metric_name)) |lease| { + if (lease.lease_expires_at_ms > observed_at_ms) return error.GraphMetricBuildAlreadyRunning; + } + } + var partition_plan = if (drain) try self.prepareGraphMetricPartitionPlan(cfg) else try self.cachedGraphMetricPartitionPlanForConfig(cfg); + defer partition_plan.deinit(self.alloc); + // Boundary planning can be substantial for a cold generation. Base + // both lease takeover and the new expiry on the time at which the + // transaction is actually ready to commit, not the pre-scan probe. + const lease_started_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); - var deletes = try self.alloc.alloc(BatchDelete, edges.len); - defer self.alloc.free(deletes); - for (edges, 0..) |edge, i| { - deletes[i] = .{ - .source = edge.source, - .target = edge.target, - .edge_type = edge.edge_type, + // The disabled marker is checked in the same transaction that creates + // the lease. A scheduler decision made before an operator delete can + // therefore never resurrect the metric after that delete commits. + if (try self.metricDisabled(&batch, metric_name)) return error.GraphMetricDisabled; + if (self.topology_preparation_only) { + const task_key = try topologyTaskKeyAlloc(self.alloc, metric_name); + defer self.alloc.free(task_key); + const task_raw = batch.get(task_key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildSuperseded, + else => return err, }; + if (try topologyTaskIncarnation(task_raw) != try topologyTaskNameIncarnation(metric_name) or task_raw[8] & 2 != 0) return error.GraphMetricBuildSuperseded; + const filter = try topology_owner.filterDigest(self.alloc, cfg.edge_filter); + const identity = topology_owner.identity(filter, try self.graphMetricPartitionPlanRaw(&batch, cfg)); + const ready = try topology_owner.readyKey(self.alloc, identity, cfg.kind == .hits_authority); + defer self.alloc.free(ready); + if (batch.get(ready)) |_| return error.GraphMetricBuildSuperseded else |err| if (err != error.NotFound) return err; } - try self.batchApply(&.{}, deletes); - } - - fn validateTreeBatchWrites(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete) !void { - for (writes, 0..) |write, i| { - if (self.getTopologyMode(write.edge_type) != .tree) continue; - - const existing = try self.getEdges(self.alloc, write.source, write.edge_type, .out); - defer freeEdges(self.alloc, existing); - for (existing) |edge| { - if (containsBatchDelete(deletes, edge.source, edge.target, edge.edge_type)) continue; - if (!std.mem.eql(u8, edge.target, write.target)) { - return TreeTopologyViolation.TreeTopologyViolation; - } + const key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(key); + if (batch.get(key)) |raw| { + if (decodeGraphMetricBuildLease(raw)) |lease| { + if (lease.lease_expires_at_ms > lease_started_at_ms) return error.GraphMetricBuildAlreadyRunning; } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } - for (writes[0..i]) |prior| { - if (!std.mem.eql(u8, prior.source, write.source)) continue; - if (!std.mem.eql(u8, prior.edge_type, write.edge_type)) continue; - if (containsBatchDelete(deletes, prior.source, prior.target, prior.edge_type)) continue; - if (!std.mem.eql(u8, prior.target, write.target)) { - return TreeTopologyViolation.TreeTopologyViolation; + if (try self.metricBuildJob(&batch, metric_name)) |prior_job| { + if (prior_job.phase != .complete) { + // Explicit failure already recorded diagnostics and enqueued + // bounded namespace retirement. Replacing that failed active + // pointer must not count the same build a second time as a + // lease-takeover failure. Unrecorded abandoned work still goes + // through the superseded-build path here. + if (prior_job.retry_count == 0 and prior_job.last_error.len == 0) { + try self.retireSupersededGraphMetricBuildInBatch( + &batch, + metric_name, + cfg, + prior_job, + "GraphMetricBuildSupersededByLeaseTakeover", + ); } } } - } - pub fn rebuildReverseFromOwnedOutgoingEdges(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { - var io_impl = std.Io.Threaded.init(alloc, .{}); - defer io_impl.deinit(); - return try self.rebuildReverseFromOwnedOutgoingEdgesResumeWithIo(alloc, io_impl.io(), lower, upper, null); + try partition_plan.validateMetricSnapshot(self, &batch, cfg, target_generation); + const score_generation = try self.allocateGraphMetricScoreGenerationInBatch(&batch, metric_name, cfg, target_generation); + const lease = GraphMetricBuildLease{ + // The durable score epoch is already monotonic and unique for this + // metric. Reuse it as the job identity so two rebuilds started in + // the same millisecond can never alias a completed job namespace. + .job_id = score_generation, + .target_generation = target_generation, + .started_at_ms = lease_started_at_ms, + .lease_expires_at_ms = lease_started_at_ms + graph_metric_local_build_lease_ms, + .phase = .prepare_generation, + .iteration = 0, + .worker_id = graph_metric_local_build_worker_id, + }; + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(lease, encoded); + try batch.put(key, encoded); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = lease.job_id, + .target_generation = target_generation, + .score_generation = score_generation, + .started_at_ms = lease.started_at_ms, + .updated_at_ms = lease_started_at_ms, + .lease_expires_at_ms = lease.lease_expires_at_ms, + .phase = lease.phase, + .iteration = lease.iteration, + .worker_id = lease.worker_id, + }); + try self.planGraphMetricBuildManifestInBatch(&batch, metric_name, cfg, .{ + .job_id = lease.job_id, + .target_generation = target_generation, + .score_generation = score_generation, + .started_at_ms = lease.started_at_ms, + .updated_at_ms = lease_started_at_ms, + .lease_expires_at_ms = lease.lease_expires_at_ms, + .phase = lease.phase, + .iteration = lease.iteration, + .worker_id = lease.worker_id, + }, partition_plan); + const requested = try self.graphMetricControlKeyAlloc(&.{ metric_name, "requested" }); + defer self.alloc.free(requested); + batch.delete(requested) catch |err| if (err != error.NotFound) return err; + try batch.commit(); } - pub fn copyOwnedOutgoingEdgesTo(self: *GraphIndex, dest: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { - const range_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; - defer if (range_lower_owned) |key| alloc.free(key); - const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; - defer if (range_upper_owned) |key| alloc.free(key); - const range_lower = range_lower_owned orelse ""; - const range_upper = range_upper_owned orelse ""; - - const pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); - defer backend_scan.freeResults(alloc, pairs); + fn retireSupersededGraphMetricBuildInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + reason: []const u8, + ) !void { + self.sealed_vectors.retire(self.alloc, sealedVectorScope(metric_name)); + const retry_count = try self.nextGraphMetricFailureRetryCountInBatch(batch, metric_name); + try self.putGraphMetricFailureDetailInBatch(batch, metric_name, retry_count, reason); + const published_generation = try self.metricPublishedGeneration(batch, metric_name); + if (job.score_generation != 0 and job.score_generation != published_generation and + try self.scoreGenerationHasKeysInBatch(batch, metric_name, job.score_generation)) + { + try self.enqueueRetiredScoreGenerationInBatch(batch, metric_name, job.score_generation); + } + if (self.pairedHitsMetricConfig(cfg)) |pair| { + const pair_retry_count = try self.nextGraphMetricFailureRetryCountInBatch(batch, pair.name); + try self.putGraphMetricFailureDetailInBatch(batch, pair.name, pair_retry_count, reason); + const pair_published_generation = try self.metricPublishedGeneration(batch, pair.name); + if (job.score_generation != 0 and job.score_generation != pair_published_generation and + try self.scoreGenerationHasKeysInBatch(batch, pair.name, job.score_generation)) + { + try self.enqueueRetiredScoreGenerationInBatch(batch, pair.name, job.score_generation); + } + } - var batch = try dest.beginWriteOutgoingBatch(); - errdefer batch.abort(); - var copied: usize = 0; - for (pairs) |pair| { - var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; - defer parsed.deinit(alloc); - if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; - if (!std.mem.eql(u8, dest.index_name, self.index_name)) continue; - try batch.put(pair.key, pair.value); - copied += 1; + const cleanup_key = try self.graphMetricFailedJobCleanupKeyAlloc(metric_name, job.job_id); + defer self.alloc.free(cleanup_key); + const job_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(job_prefix); + var cleanup = try self.deleteKeysWithPrefixPageInBatch( + batch, + job_prefix, + "", + graph_metric_build_cleanup_delete_page_units, + ); + defer cleanup.deinit(self.alloc); + if (cleanup.reached_end) { + batch.delete(cleanup_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } else { + try batch.put(cleanup_key, cleanup.cursor); } - try batch.commit(); - return copied; } - pub fn rebuildReverseFromOwnedOutgoingEdgesResume( + fn ensureGraphMetricBuildManifestForJob( self: *GraphIndex, - alloc: Allocator, - lower: []const u8, - upper: []const u8, - resume_from: ?[]const u8, - ) !usize { - var io_impl = std.Io.Threaded.init(alloc, .{}); - defer io_impl.deinit(); - return try self.rebuildReverseFromOwnedOutgoingEdgesResumeWithIo(alloc, io_impl.io(), lower, upper, resume_from); + metric_name: []const u8, + job: GraphMetricBuildJob, + ) !void { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.metricBuildManifest(&txn, metric_name, job.job_id)) |manifest| { + try validateGraphMetricBuildExecution(manifest, job, cfg); + const raw_plan = try self.graphMetricPartitionPlanRaw(&txn, cfg); + var plan = (try self.decodeGraphMetricPartitionPlanAlloc(raw_plan)) orelse + return error.InvalidGraphMetricBuildManifest; + defer plan.deinit(self.alloc); + try plan.validateMetricSnapshot(self, &txn, cfg, job.target_generation); + if (plan.edge_count != manifest.edge_count or plan.node_count != manifest.node_count) + return error.InvalidGraphMetricBuildManifest; + return; + } + } + var partition_plan = try self.prepareGraphMetricPartitionPlan(cfg); + defer partition_plan.deinit(self.alloc); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + try partition_plan.validateMetricSnapshot(self, &batch, cfg, job.target_generation); + try self.planGraphMetricBuildManifestInBatch(&batch, metric_name, cfg, job, partition_plan); + try batch.commit(); } - pub fn rebuildReverseFromOwnedOutgoingEdgesResumeWithIo( - self: *GraphIndex, - alloc: Allocator, - io: std.Io, - lower: []const u8, - upper: []const u8, - resume_from: ?[]const u8, - ) !usize { - const base_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; - defer if (base_lower_owned) |key| alloc.free(key); - const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; - defer if (range_upper_owned) |key| alloc.free(key); - const base_lower = base_lower_owned orelse ""; - const range_lower = if (resume_from) |key| - if (key.len > 0 and std.mem.order(u8, key, base_lower) == .gt) key else base_lower - else - base_lower; - const range_upper = range_upper_owned orelse ""; + const GraphMetricPartitionPlan = struct { + edge_generation: u64, + edge_count: u64, + node_count: u64, + boundary_digest: [32]u8 = @splat(0), + edge_page_count: usize, + node_page_count: usize, + edge_boundaries: std.ArrayListUnmanaged([]u8) = .empty, + edge_page_units: std.ArrayListUnmanaged(u64) = .empty, + node_boundaries: std.ArrayListUnmanaged([]u8) = .empty, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.edge_boundaries.items) |key| alloc.free(key); + self.edge_boundaries.deinit(alloc); + self.edge_page_units.deinit(alloc); + for (self.node_boundaries.items) |key| alloc.free(key); + self.node_boundaries.deinit(alloc); + self.* = undefined; + } - const pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); - defer backend_scan.freeResults(alloc, pairs); + fn validateSnapshot(self: @This(), txn: anytype, target_generation: u64) !void { + if (self.edge_generation != target_generation or + try readU64OrZero(txn, graph_edge_generation_key) != self.edge_generation or + try readU64OrZero(txn, graph_edge_count_key) != self.edge_count or + try readU64OrZero(txn, graph_node_count_key) != self.node_count) + { + return error.GraphMetricBuildSnapshotChanged; + } + } - var rebuilt: usize = 0; - var batch_count: usize = 0; - var flushed_batches: usize = 0; - var matching_edges: usize = 0; - var txn = try self.beginWriteReverseTxn(); - var txn_active = true; - errdefer if (txn_active) txn.abort(); - const rebuild_state = if (self.rebuild_root_path) |path| - if (self.rebuild_owner_generation != 0) - backfill_state_mod.RebuildState.initOwned(path, self.rebuild_storage, self.rebuild_owner_generation) - else - backfill_state_mod.RebuildState.initWithStorage(path, self.rebuild_storage) - else - null; + fn validateMetricSnapshot(self: @This(), index: *GraphIndex, txn: anytype, cfg: GraphMetricConfig, target_generation: u64) !void { + if (cfg.edge_filter.mode == .all) return self.validateSnapshot(txn, target_generation); + // Boundaries are frozen scheduling ranges, not a live census of + // unrelated relationship types. Selected topology must be exact. + if (self.edge_generation != target_generation or + try index.graphMetricFilterGeneration(txn, cfg.edge_filter) != target_generation) + return error.GraphMetricBuildSnapshotChanged; + } + }; - for (pairs) |pair| { - if (resume_from) |resume_key| { - if (resume_key.len > 0 and std.mem.order(u8, pair.key, resume_key) != .gt) continue; + /// Control stays independent of boundary length. Lease validation, + /// topology ownership and numerical iterations read no boundary blocks. + fn decodeGraphMetricPartitionPlanAlloc(self: *GraphIndex, raw: []const u8) !?GraphMetricPartitionPlan { + if (raw.len != 76 or std.mem.readInt(u32, raw[0..4], .little) != graph_metric_partition_plan_version) return null; + if (std.hash.Wyhash.hash(graph_metric_partition_plan_checksum_seed, raw[0..68]) != std.mem.readInt(u64, raw[68..76], .little)) return null; + const edges = std.math.cast(usize, std.mem.readInt(u64, raw[12..20], .little)) orelse return null; + const nodes = std.math.cast(usize, std.mem.readInt(u64, raw[20..28], .little)) orelse return null; + const edge_pages = std.mem.readInt(u32, raw[28..32], .little); + const node_pages = std.mem.readInt(u32, raw[32..36], .little); + if (edge_pages != self.graphMetricDegreeScanPageCount(edges) or node_pages != self.graphMetricDegreeReducePageCount(nodes)) return null; + return .{ + .edge_generation = std.mem.readInt(u64, raw[4..12], .little), + .edge_count = edges, + .node_count = nodes, + .edge_page_count = edge_pages, + .node_page_count = node_pages, + .boundary_digest = raw[36..68].*, + }; + } + + fn materializeGraphMetricPartitionPlan(self: *GraphIndex, txn: anytype, key: []const u8, plan: *GraphMetricPartitionPlan) !void { + var state = partition_census.State{ + .generation = plan.edge_generation, + .edge_count = plan.edge_count, + .node_count = plan.node_count, + .persisted_edges = if (plan.edge_count == 0) 0 else plan.edge_page_count, + .persisted_nodes = if (plan.node_count == 0) 0 else plan.node_page_count, + }; + defer state.deinit(self.alloc); + try state.materializeBoundaries(self.alloc, txn, key); + if (!std.mem.eql(u8, &state.boundaryDigest(), &plan.boundary_digest)) return error.InvalidGraphMetricBuildManifest; + for (state.edge_boundaries.items) |boundary| { + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, boundary, self.index_name)) orelse return error.InvalidGraphMetricBuildManifest; + parsed.deinit(self.alloc); + } + std.mem.swap(std.ArrayListUnmanaged([]u8), &plan.edge_boundaries, &state.edge_boundaries); + std.mem.swap(std.ArrayListUnmanaged([]u8), &plan.node_boundaries, &state.node_boundaries); + for (0..plan.edge_page_count) |page| try plan.edge_page_units.append(self.alloc, graphMetricPartitionSpan(@intCast(plan.edge_count), plan.edge_page_count, page).len); + } + + /// Bounded census checkpoints already wrote the boundary slots. Sealing + /// publishes only this small record in the generation/checkpoint CAS. + fn putGraphMetricPartitionPlanAtKeyInBatch(self: *GraphIndex, batch: anytype, key: []const u8, plan: GraphMetricPartitionPlan) !void { + _ = self; + var encoded: [76]u8 = undefined; + std.mem.writeInt(u32, encoded[0..4], graph_metric_partition_plan_version, .little); + std.mem.writeInt(u64, encoded[4..12], plan.edge_generation, .little); + std.mem.writeInt(u64, encoded[12..20], plan.edge_count, .little); + std.mem.writeInt(u64, encoded[20..28], plan.node_count, .little); + std.mem.writeInt(u32, encoded[28..32], @intCast(plan.edge_page_count), .little); + std.mem.writeInt(u32, encoded[32..36], @intCast(plan.node_page_count), .little); + @memcpy(encoded[36..68], &plan.boundary_digest); + std.mem.writeInt(u64, encoded[68..76], std.hash.Wyhash.hash(graph_metric_partition_plan_checksum_seed, encoded[0..68]), .little); + try batch.put(key, &encoded); + } + + /// Read the immutable partition boundaries before opening the lease write + /// transaction. Snapshot validation below makes this optimistic planning + /// safe while preventing a full edge/node scan from holding the sole + /// reverse-store writer open. + fn prepareGraphMetricPartitionPlan(self: *GraphIndex, cfg: GraphMetricConfig) !GraphMetricPartitionPlan { + // Explicit synchronous callers drain the same bounded checkpoints as + // background maintenance; there is only one production planner. + while (!try self.prepareGraphMetricPartitionForConfigStep(cfg, 4096)) {} + return self.cachedGraphMetricPartitionPlanForConfig(cfg); + } + + fn cachedGraphMetricPartitionPlanForConfig(self: *GraphIndex, cfg: GraphMetricConfig) !GraphMetricPartitionPlan { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var plan = (try self.decodeGraphMetricPartitionPlanAlloc(try self.graphMetricPartitionPlanRaw(&txn, cfg))) orelse return error.GraphMetricBuildSnapshotChanged; + errdefer plan.deinit(self.alloc); + try plan.validateMetricSnapshot(self, &txn, cfg, try self.graphMetricFilterGeneration(&txn, cfg.edge_filter)); + const key = try self.graphMetricPartitionPlanKeyAlloc(cfg.edge_filter); + defer self.alloc.free(key); + try self.materializeGraphMetricPartitionPlan(&txn, key, &plan); + return plan; + } + + fn graphMetricPartitionPlanKeyAlloc(self: *GraphIndex, filter: GraphMetricEdgeFilter) ![]u8 { + if (filter.mode == .all) return self.alloc.dupe(u8, graph_metric_partition_plan_key); + const digest = try topology_owner.filterDigest(self.alloc, filter); + const hex = std.fmt.bytesToHex(digest, .lower); + return std.mem.concat(self.alloc, u8, &.{ graph_metric_filter_plan_prefix, &hex }); + } + + fn graphMetricPartitionPlanRaw(self: *GraphIndex, txn: anytype, cfg: GraphMetricConfig) ![]const u8 { + const key = try self.graphMetricPartitionPlanKeyAlloc(cfg.edge_filter); + defer self.alloc.free(key); + return txn.get(key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildSnapshotChanged, + else => return err, + }; + } + + pub fn prepareGraphMetricPartitionForConfigStep(self: *GraphIndex, cfg: GraphMetricConfig, max_records: usize) !bool { + if (cfg.edge_filter.mode == .all) return self.prepareGraphMetricPartitionStep(max_records); + if (max_records == 0) return error.InvalidGraphMetricBuildOptions; + // Repair/backfill may have invalidated the covering index without + // changing connectivity. A frozen plan never bypasses readiness. + if (!try self.prepareTypedGraphEdgesStep(max_records)) return false; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const raw = self.graphMetricPartitionPlanRaw(&txn, cfg) catch |err| switch (err) { + error.GraphMetricBuildSnapshotChanged => null, + else => return err, + }; + if (raw) |bytes| if (try self.decodeGraphMetricPartitionPlanAlloc(bytes)) |value| { + var plan = value; + defer plan.deinit(self.alloc); + if (plan.edge_generation == try self.graphMetricFilterGeneration(&txn, cfg.edge_filter)) return true; + }; + } + // A filtered census is independent of the global graph generation. + // First count selected edges and distinct endpoints, then choose exact + // partition boundaries. The same key holds either progress or a plan, + // so normal filter-plan GC also reclaims abandoned censuses. + const key = try self.graphMetricPartitionPlanKeyAlloc(cfg.edge_filter); + defer self.alloc.free(key); + var prior: []u8 = &.{}; + defer self.alloc.free(prior); + var state: partition_census.State = undefined; + var phase: u8 = 0; + var complete = false; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const epoch = try self.graphMetricFilterGeneration(&txn, cfg.edge_filter); + state = .{ .generation = epoch, .edge_count = 0, .node_count = 0 }; + errdefer state.deinit(self.alloc); + prior = try self.alloc.dupe(u8, txn.get(key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }); + if (try self.decodeGraphMetricPartitionPlanAlloc(prior)) |value| { + var plan = value; + defer plan.deinit(self.alloc); + if (plan.edge_generation == epoch) return true; } - var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; - defer parsed.deinit(alloc); - if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; - matching_edges += 1; + if (prior.len > 2 and prior[0] == 0xff and prior[1] <= 2) { + if (try partition_census.State.decodeAlloc(self.alloc, prior[2..])) |value| { + var saved = value; + if (saved.generation == epoch and saved.phase == prior[1]) { + state = saved; + phase = prior[1]; + } else saved.deinit(self.alloc); + } + } + complete = try self.advanceFilteredPartitionCensus(&txn, cfg, &state, &phase, max_records); + if (complete) try state.materializeBoundaries(self.alloc, &txn, key); + } + defer state.deinit(self.alloc); + const boundary_digest = if (complete) state.boundaryDigest() else @as([32]u8, @splat(0)); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = batch.get(key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + if (!std.mem.eql(u8, current, prior) or state.generation != try self.graphMetricFilterGeneration(&batch, cfg.edge_filter)) { + batch.abort(); + return false; + } + if (complete) { + var plan = GraphMetricPartitionPlan{ + .edge_generation = state.generation, + .boundary_digest = boundary_digest, + .edge_count = state.edge_count, + .node_count = state.node_count, + .edge_page_count = self.graphMetricDegreeScanPageCount(@intCast(state.edge_count)), + .node_page_count = self.graphMetricDegreeReducePageCount(@intCast(state.node_count)), + .edge_boundaries = state.edge_boundaries, + .node_boundaries = state.node_boundaries, + }; + defer plan.edge_page_units.deinit(self.alloc); + for (0..plan.edge_page_count) |page| try plan.edge_page_units.append(self.alloc, graphMetricPartitionSpan(@intCast(state.edge_count), plan.edge_page_count, page).len); + try state.persistBoundaries(self.alloc, &batch, key); + try self.putGraphMetricPartitionPlanAtKeyInBatch(&batch, key, plan); + } else { + try state.persistBoundaries(self.alloc, &batch, key); + state.phase = phase; + const raw = try state.encodeAlloc(self.alloc); + defer self.alloc.free(raw); + const encoded = try std.mem.concat(self.alloc, u8, &.{ &.{ 0xff, phase }, raw }); + defer self.alloc.free(encoded); + try batch.put(key, encoded); + } + try batch.commit(); + return complete; + } - const rev_key = try reverseEdgeKeyAlloc(alloc, parsed.target, self.index_name, parsed.edge_type, parsed.source); - defer alloc.free(rev_key); - try txn.put(rev_key, pair.value); - rebuilt += 1; - batch_count += 1; + fn advanceFilteredPartitionCensus(self: *GraphIndex, txn: anytype, cfg: GraphMetricConfig, state: *partition_census.State, phase: *u8, max_records: usize) !bool { + var remaining = max_records; + var bytes: usize = 0; + while (true) { + const nodes = phase.* == 1 or (phase.* == 2 and state.edges_done); + const progress = if (nodes) &state.node_cursor else &state.edge_cursor; + var cursor = try typed_edges.MergedCursor.init(self.alloc, txn, cfg.edge_filter.types, nodes, progress.*); + defer cursor.deinit(); + while (try cursor.next()) |key| { + if (remaining == 0 or (remaining < max_records and bytes +| key.len > 1024 * 1024)) return false; + bytes +|= key.len; + remaining -= 1; + if (phase.* < 2) { + const count = if (nodes) &state.node_count else &state.edge_count; + count.* = std.math.add(u64, count.*, 1) catch return error.GraphMetricBuildBudgetExceeded; + } else { + const seen = if (nodes) &state.nodes_seen else &state.edges_seen; + const count = std.math.cast(usize, if (nodes) state.node_count else state.edge_count) orelse return error.GraphMetricBuildBudgetExceeded; + const boundaries = if (nodes) &state.node_boundaries else &state.edge_boundaries; + const pages = if (nodes) self.graphMetricDegreeReducePageCount(count) else self.graphMetricDegreeScanPageCount(count); + const page = state.boundaryCount(nodes); + if (page < pages and seen.* == graphMetricPartitionSpan(count, pages, page).start) { + try boundaries.ensureUnusedCapacity(self.alloc, 1); + boundaries.appendAssumeCapacity(try self.alloc.dupe(u8, key)); + } + seen.* += 1; + if (seen.* > count) return error.InvalidGraphMetricPartitionCensus; + } + try self.replaceOwnedBytes(progress, key); + } + try self.replaceOwnedBytes(progress, ""); + if (phase.* < 2) { + phase.* += 1; + } else if (!nodes) { + if (state.edges_seen != state.edge_count) return error.InvalidGraphMetricPartitionCensus; + state.edges_done = true; + } else { + if (state.nodes_seen != state.node_count) return error.InvalidGraphMetricPartitionCensus; + return true; + } + if (remaining == 0) return false; + } + } - if (batch_count >= reverse_rebuild_batch_size) { - try txn.commit(); - txn_active = false; - if (rebuild_state) |state| try state.updateWithIo(io, pair.key); - flushed_batches += 1; - if (@import("builtin").is_test) { - if (test_abort_reverse_rebuild_after_batches) |limit| { - if (flushed_batches >= limit) return error.TestInjectedBackfillFailure; + /// Read-only benchmark oracle for the former control-plus-boundaries path. + pub fn benchmarkPartitionPlanControl(self: *GraphIndex, reference: bool) !u64 { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var plan = (try self.decodeGraphMetricPartitionPlanAlloc(try txn.get(graph_metric_partition_plan_key))) orelse return error.InvalidGraphMetricBuildManifest; + defer plan.deinit(self.alloc); + if (reference) try self.materializeGraphMetricPartitionPlan(&txn, graph_metric_partition_plan_key, &plan); + return std.mem.readInt(u64, plan.boundary_digest[0..8], .little) ^ plan.edge_count ^ plan.node_count; + } + + fn cachedGraphMetricPartitionPlan(self: *GraphIndex) !GraphMetricPartitionPlan { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const raw = txn.get(graph_metric_partition_plan_key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildSnapshotChanged, + else => return err, + }; + var plan = (try self.decodeGraphMetricPartitionPlanAlloc(raw)) orelse return error.GraphMetricBuildSnapshotChanged; + errdefer plan.deinit(self.alloc); + try plan.validateSnapshot(&txn, try readU64OrZero(&txn, graph_edge_generation_key)); + try self.materializeGraphMetricPartitionPlan(&txn, graph_metric_partition_plan_key, &plan); + return plan; + } + + /// One bounded, durable planning step shared across all metric definitions. + /// The census is optimistic: graph writes never wait for the entire scan. + /// A generation change discards only the obsolete checkpoint, and competing + /// coordinators compare the checkpoint before publishing their next step. + pub fn prepareGraphMetricPartitionStep(self: *GraphIndex, max_records: usize) !bool { + if (max_records == 0) return error.InvalidGraphMetricBuildOptions; + var state: partition_census.State = undefined; + var old_raw: []u8 = &.{}; + defer self.alloc.free(old_raw); + var complete = false; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const generation = try readU64OrZero(&txn, graph_edge_generation_key); + const edges = try readU64OrZero(&txn, graph_edge_count_key); + const nodes = try readU64OrZero(&txn, graph_node_count_key); + if (txn.get(graph_metric_partition_plan_key)) |raw| { + if (try self.decodeGraphMetricPartitionPlanAlloc(raw)) |cached_value| { + var cached = cached_value; + defer cached.deinit(self.alloc); + if (cached.edge_generation == generation and cached.edge_count == edges and cached.node_count == nodes) return true; + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + state = .{ .generation = generation, .edge_count = edges, .node_count = nodes }; + errdefer state.deinit(self.alloc); + if (txn.get(graph_metric_partition_census_key)) |raw| { + old_raw = try self.alloc.dupe(u8, raw); + if (try partition_census.State.decodeAlloc(self.alloc, old_raw)) |prior_value| { + var prior = prior_value; + if (prior.identifies(generation, edges, nodes)) state = prior else prior.deinit(self.alloc); + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + complete = try self.advanceGraphMetricPartitionCensus(&txn, &state, max_records); + if (complete) try state.materializeBoundaries(self.alloc, &txn, graph_metric_partition_plan_key); + } + defer state.deinit(self.alloc); + const boundary_digest = if (complete) state.boundaryDigest() else @as([32]u8, @splat(0)); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = batch.get(graph_metric_partition_census_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + if (!std.mem.eql(u8, current, old_raw) or + !state.identifies(try readU64OrZero(&batch, graph_edge_generation_key), try readU64OrZero(&batch, graph_edge_count_key), try readU64OrZero(&batch, graph_node_count_key))) + { + batch.abort(); + return false; + } + if (complete) { + var plan = GraphMetricPartitionPlan{ + .edge_generation = state.generation, + .boundary_digest = boundary_digest, + .edge_count = state.edge_count, + .node_count = state.node_count, + .edge_page_count = self.graphMetricDegreeScanPageCount(@intCast(state.edge_count)), + .node_page_count = self.graphMetricDegreeReducePageCount(@intCast(state.node_count)), + .edge_boundaries = state.edge_boundaries, + .node_boundaries = state.node_boundaries, + }; + // Boundaries stay owned by the census until this transaction ends. + defer plan.edge_page_units.deinit(self.alloc); + for (0..plan.edge_page_count) |page| try plan.edge_page_units.append(self.alloc, graphMetricPartitionSpan(@intCast(state.edge_count), plan.edge_page_count, page).len); + try state.persistBoundaries(self.alloc, &batch, graph_metric_partition_plan_key); + try self.putGraphMetricPartitionPlanAtKeyInBatch(&batch, graph_metric_partition_plan_key, plan); + batch.delete(graph_metric_partition_census_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } else { + try state.persistBoundaries(self.alloc, &batch, graph_metric_partition_plan_key); + const encoded = try state.encodeAlloc(self.alloc); + defer self.alloc.free(encoded); + try batch.put(graph_metric_partition_census_key, encoded); + } + try batch.commit(); + return complete; + } + + fn prepareTypedGraphEdgesStep(self: *GraphIndex, max_records: usize) !bool { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var postings = std.ArrayListUnmanaged([]const u8).empty; + var prior: []const u8 = ""; + var next_cursor: []const u8 = ""; + var generation: u64 = 0; + var complete = false; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (txn.get(typed_edges.ready_key)) |_| return true else |err| if (err != error.NotFound) return err; + generation = try readU64OrZero(&txn, graph_edge_generation_key); + prior = try temp.dupe(u8, txn.get(typed_edges.cursor_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }); + var cursor = try txn.openCursor(); + defer cursor.close(); + var item = try graphMetricSkipMetadata(&cursor, if (prior.len == 0) try cursor.first() else try cursor.seekAtOrAfter(prior)); + if (item) |entry| if (std.mem.eql(u8, entry.key, prior)) { + item = try graphMetricSkipMetadata(&cursor, try cursor.next()); + }; + var visited: usize = 0; + var retained_key_bytes: usize = 0; + while (item) |entry| : (item = try graphMetricSkipMetadata(&cursor, try cursor.next())) { + if (visited == max_records) break; + // Bound temporary key copies too: identifiers/types can be + // large. Include escaped-component and posting intermediates + // retained by the arena. One oversized record may progress. + const key_envelope = std.math.add(usize, std.math.mul(usize, entry.key.len, 8) catch return error.GraphMetricBuildBudgetExceeded, 128) catch return error.GraphMetricBuildBudgetExceeded; + const next_bytes = std.math.add(usize, retained_key_bytes, key_envelope) catch return error.GraphMetricBuildBudgetExceeded; + if (visited > 0 and next_bytes > 1024 * 1024) break; + retained_key_bytes = next_bytes; + visited += 1; + next_cursor = try temp.dupe(u8, entry.key); + if (try parseMetricReverseEdgeKeyView(temp, entry.key, self.index_name)) |parsed_value| { + var parsed = parsed_value; + defer parsed.deinit(temp); + try postings.append(temp, next_cursor); + } + } + complete = item == null; + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = batch.get(typed_edges.cursor_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + if (!std.mem.eql(u8, current, prior) or generation != try readU64OrZero(&batch, graph_edge_generation_key)) { + batch.abort(); + return false; + } + var typed_updates = typed_edges.Updates.init(self.alloc); + defer typed_updates.deinit(); + for (postings.items) |key| { + var parsed = (try parseMetricReverseEdgeKeyView(temp, key, self.index_name)) orelse return error.InvalidGraphMetricBuildManifest; + defer parsed.deinit(temp); + // Posting scratch is reused; endpoint deltas are coalesced across + // this byte-bounded page before touching durable counts. + try typed_updates.stage(&batch, parsed.edge_type.bytes, key, parsed.source.bytes, parsed.target.bytes, true); + } + try typed_updates.flush(&batch); + try batch.put(typed_edges.active_key, "1"); + if (complete) { + try batch.put(typed_edges.ready_key, "1"); + batch.delete(typed_edges.cursor_key) catch |err| if (err != error.NotFound) return err; + } else try batch.put(typed_edges.cursor_key, next_cursor); + try batch.commit(); + return false; // This call already consumed its bounded work allowance. + } + + fn advanceGraphMetricPartitionCensus(self: *GraphIndex, txn: anytype, state: *partition_census.State, max_records: usize) !bool { + const edge_count = std.math.cast(usize, state.edge_count) orelse return error.GraphMetricBuildBudgetExceeded; + const node_count = std.math.cast(usize, state.node_count) orelse return error.GraphMetricBuildBudgetExceeded; + var remaining = max_records; + var scanned_bytes: usize = 0; + if (!state.edges_done) { + var cur = try txn.openCursor(); + defer cur.close(); + var item = if (state.edge_cursor.len == 0) try cur.first() else try cur.seekAtOrAfter(state.edge_cursor); + if (item) |entry| if (std.mem.eql(u8, entry.key, state.edge_cursor)) { + item = try cur.next(); + }; + while (item) |entry| { + // Metadata contains score vectors and checkpoints, not edges. + // Skip its entire namespace in one seek, regardless of size. + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) { + item = try cur.seekAtOrAfter("meta;"); + continue; + } + if (graphIndexEdgeKeyMatchesIndex(entry.key, self.index_name)) { + const page = state.boundaryCount(false); + const pages = self.graphMetricDegreeScanPageCount(edge_count); + if (page < pages and state.edges_seen == graphMetricPartitionSpan(edge_count, pages, page).start) { + try state.edge_boundaries.ensureUnusedCapacity(self.alloc, 1); + state.edge_boundaries.appendAssumeCapacity(try self.alloc.dupe(u8, entry.key)); } + state.edges_seen += 1; + if (state.edges_seen > state.edge_count) return error.InvalidGraphMetricBuildManifest; } - txn = try self.beginWriteReverseTxn(); - txn_active = true; - batch_count = 0; + remaining -= 1; + scanned_bytes +|= entry.key.len; + if (remaining == 0 or scanned_bytes >= 1024 * 1024) { + try self.replaceOwnedBytes(&state.edge_cursor, entry.key); + return false; + } + item = try cur.next(); + } + if (state.edges_seen != state.edge_count) return error.InvalidGraphMetricBuildManifest; + state.edges_done = true; + } + const prefix = "meta:node_ref:"; + var cur = try txn.openCursor(); + defer cur.close(); + var item = try cur.seekAtOrAfter(if (state.node_cursor.len == 0) prefix else state.node_cursor); + if (item) |entry| if (std.mem.eql(u8, entry.key, state.node_cursor)) { + item = try cur.next(); + }; + while (item) |entry| : (item = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const page = state.boundaryCount(true); + const pages = self.graphMetricDegreeReducePageCount(node_count); + if (page < pages and state.nodes_seen == graphMetricPartitionSpan(node_count, pages, page).start) { + try state.node_boundaries.ensureUnusedCapacity(self.alloc, 1); + state.node_boundaries.appendAssumeCapacity(try self.alloc.dupe(u8, entry.key[prefix.len..])); + } + state.nodes_seen += 1; + if (state.nodes_seen > state.node_count) return error.InvalidGraphMetricBuildManifest; + remaining -= 1; + scanned_bytes +|= entry.key.len; + if (remaining == 0 or scanned_bytes >= 1024 * 1024) { + try self.replaceOwnedBytes(&state.node_cursor, entry.key); + return false; + } + } + if (state.nodes_seen != state.node_count) return error.InvalidGraphMetricBuildManifest; + return true; + } + + fn planGraphMetricBuildManifestInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + partition_plan: GraphMetricPartitionPlan, + ) !void { + // The generation-fenced census has already sealed this plan. + const phases = graphMetricBuildManifestPhases(cfg.kind); + const planned_edge_count = std.math.cast(usize, partition_plan.edge_count) orelse return error.GraphMetricBuildBudgetExceeded; + const planned_node_count = std.math.cast(usize, partition_plan.node_count) orelse return error.GraphMetricBuildBudgetExceeded; + const degree_scan_page_count = if (cfg.kind == .degree or graphMetricKindUsesIterativeBuild(cfg.kind)) + partition_plan.edge_page_count + else + 0; + const degree_reduce_page_count = if (cfg.kind == .degree or graphMetricKindUsesIterativeBuild(cfg.kind)) + partition_plan.node_page_count + else + 0; + const cleanup_page_count = graphMetricBuildCleanupPageCount(cfg.kind); + var planned_page_count: usize = 0; + for (phases) |phase| { + planned_page_count += graphMetricBuildPlannedPageCountForPhase(cfg.kind, phase, degree_scan_page_count, degree_reduce_page_count, cleanup_page_count); + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, phase, planned_node_count)) + planned_page_count += 1 + graphMetricSummaryLeafCount(cfg.kind, planned_node_count, partition_plan.node_page_count); + } + const manifest = GraphMetricBuildManifest{ + .execution_schema_version = graph_metric_build_execution_schema_version, + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .planned_at_ms = if (job.started_at_ms != 0) job.started_at_ms else @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .edge_count = partition_plan.edge_count, + .node_count = partition_plan.node_count, + .phase_count = phases.len, + .page_count = planned_page_count, + }; + if (try self.metricBuildManifest(batch, metric_name, job.job_id)) |existing| { + if (existing.execution_schema_version != manifest.execution_schema_version or + existing.target_generation != manifest.target_generation or + existing.score_generation != manifest.score_generation or + existing.config_fingerprint != manifest.config_fingerprint) + { + return error.InvalidGraphMetricBuildManifest; + } + if (existing.page_count != manifest.page_count) return; + } else { + try self.putGraphMetricBuildManifestInBatch(batch, metric_name, manifest); + } + for (phases, 0..) |phase, i| { + if (cfg.kind == .degree and phase == .scan_edges_and_out_degree) { + const output_prefix = try self.graphMetricBuildDegreePartialPrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(output_prefix); + for (0..degree_scan_page_count) |scan_page_idx| { + const range_lower = if (scan_page_idx < partition_plan.edge_boundaries.items.len) partition_plan.edge_boundaries.items[scan_page_idx] else ""; + const range_upper = if (scan_page_idx + 1 < partition_plan.edge_boundaries.items.len) + partition_plan.edge_boundaries.items[scan_page_idx + 1] + else + ""; + const page_id: u64 = 1 + @as(u64, @intCast(scan_page_idx)); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = .reverse_edges, + .range_lower = range_lower, + .range_upper = range_upper, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = partition_plan.edge_page_units.items[scan_page_idx], + }); + } + continue; + } + if (cfg.kind == .degree and phase == .reduce_ranks) { + const output_prefix = try self.graphMetricScorePrefixAlloc(metric_name, job.score_generation); + defer self.alloc.free(output_prefix); + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, phase, planned_node_count)) { + _ = try self.planGraphMetricSummaryLeaves(batch, metric_name, job, phase, 0, partition_plan, output_prefix); + if ((try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, 0)) == null) { + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = 0, + .state = .pending, + .range_kind = .summary, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = @intCast(planned_node_count), + }); + } + } + for (0..degree_reduce_page_count) |reduce_page_idx| { + const span = graphMetricPartitionSpan(planned_node_count, degree_reduce_page_count, reduce_page_idx); + const range_lower = if (reduce_page_idx < partition_plan.node_boundaries.items.len) partition_plan.node_boundaries.items[reduce_page_idx] else ""; + const range_upper = if (reduce_page_idx + 1 < partition_plan.node_boundaries.items.len) + partition_plan.node_boundaries.items[reduce_page_idx + 1] + else + ""; + const page_id: u64 = 2 + @as(u64, @intCast(reduce_page_idx)); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = .nodes, + .range_lower = range_lower, + .range_upper = range_upper, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = @intCast(span.len), + }); + } + continue; + } + if (graphMetricKindUsesIterativeBuild(cfg.kind)) { + const page_count = graphMetricBuildPlannedPageCountForPhase(cfg.kind, phase, degree_scan_page_count, degree_reduce_page_count, cleanup_page_count); + const range_kind = graphMetricBuildManifestPhaseRangeKind(phase); + if (range_kind == .reverse_edges or range_kind == .nodes) { + const keys = if (range_kind == .reverse_edges) partition_plan.edge_boundaries.items else partition_plan.node_boundaries.items; + const total_units = if (range_kind == .nodes) planned_node_count else planned_edge_count; + const output_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(output_prefix); + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, phase, planned_node_count)) { + _ = try self.planGraphMetricSummaryLeaves(batch, metric_name, job, phase, 0, partition_plan, output_prefix); + if ((try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, 0)) == null) { + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = 0, + .state = .pending, + .range_kind = .summary, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = @intCast(planned_node_count), + }); + } + } + for (0..page_count) |page_idx| { + const span = graphMetricPartitionSpan(total_units, page_count, page_idx); + const page_units: u64 = if (range_kind == .reverse_edges) + partition_plan.edge_page_units.items[page_idx] + else + @intCast(span.len); + const range_lower = if (page_idx < keys.len) keys[page_idx] else ""; + const range_upper = if (page_idx + 1 < keys.len) keys[page_idx + 1] else ""; + const page_id: u64 = @as(u64, @intCast(i)) + @as(u64, @intCast(page_idx)); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = range_kind, + .range_lower = range_lower, + .range_upper = range_upper, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = page_units, + }); + } + continue; + } + } + if (cfg.kind == .degree and phase == .cleanup_old_generations) { + const degree_partial_prefix = try self.graphMetricBuildDegreePartialPrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(degree_partial_prefix); + const job_namespace_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(job_namespace_prefix); + const cleanup_prefixes = [_][]const u8{ + degree_partial_prefix, + job_namespace_prefix, + }; + for (cleanup_prefixes, 0..) |output_prefix, cleanup_page_idx| { + const page_id: u64 = @intCast(cleanup_page_idx); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = .job_control, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = 1, + }); + } + continue; } + if (cfg.kind == .pagerank and phase == .cleanup_old_generations) { + const out_degree_partial_prefix = try self.graphMetricBuildPageRankOutDegreePartialPrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(out_degree_partial_prefix); + const node_partial_prefix = try self.graphMetricBuildPageRankNodePartialPrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(node_partial_prefix); + const job_namespace_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(job_namespace_prefix); + const cleanup_prefixes = [_][]const u8{ + out_degree_partial_prefix, + node_partial_prefix, + job_namespace_prefix, + }; + for (cleanup_prefixes, 0..) |output_prefix, cleanup_page_idx| { + const page_id: u64 = @intCast(cleanup_page_idx); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = .job_control, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = 1, + }); + } + continue; + } + if ((cfg.kind == .hits_authority or cfg.kind == .hits_hub) and phase == .cleanup_old_generations) { + const hub_raw_prefix = try self.graphMetricBuildHitsHubRawNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(hub_raw_prefix); + const hub_raw_summary_prefix = try self.graphMetricBuildHitsHubRawSummaryNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(hub_raw_summary_prefix); + const hits_rank_prefix = try self.graphMetricBuildHitsRankNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(hits_rank_prefix); + const job_namespace_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(job_namespace_prefix); + const cleanup_prefixes = [_][]const u8{ + hub_raw_prefix, + hub_raw_summary_prefix, + hits_rank_prefix, + job_namespace_prefix, + }; + for (cleanup_prefixes, 0..) |output_prefix, cleanup_page_idx| { + const page_id: u64 = @intCast(cleanup_page_idx); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = .job_control, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = 1, + }); + } + continue; + } + const page_id: u64 = @intCast(i); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id)) |_| continue; + const range_kind = graphMetricBuildManifestPhaseRangeKind(phase); + const range_lower = if (range_kind == .reverse_edges) try self.firstReverseEdgeKeyForIndexInBatch(batch) else ""; + defer if (range_kind == .reverse_edges and range_lower.len > 0) self.alloc.free(range_lower); + const output_prefix = if (cfg.kind == .degree and phase == .reduce_ranks) + try self.graphMetricScorePrefixAlloc(metric_name, job.score_generation) + else if (phase == .scan_edges_and_out_degree) + try self.graphMetricScorePrefixAlloc(metric_name, job.score_generation) + else + ""; + defer if (output_prefix.len > 0) self.alloc.free(output_prefix); + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = 0, + .page_id = page_id, + .state = .pending, + .range_kind = range_kind, + .range_lower = range_lower, + .range_upper = "", + .output_prefix = output_prefix, + .worker_id = "", + .total_units = graphMetricBuildManifestPhaseUnits(phase, self.edge_count, self.node_count), + }); } + try self.ensureTopologyBindingInBatch(batch, metric_name, cfg, job); + } - try txn.commit(); - txn_active = false; - if (rebuild_state) |state| try state.clearWithIo(io); - try self.rebuildCounterMetadata(); - try self.checkpointLsmWalAfterDurableBoundary(); - return rebuilt; + /// A bounded, metric-specific work plan. Original leaf IDs remain stable + /// because numeric slots encode those IDs; only scheduling is compacted. + const GraphMetricActivePlan = struct { + count: usize, + counts: [graph_metric_build_max_partition_pages]u64 = @splat(0), + + fn total(self: *const @This()) u64 { + var sum: u64 = 0; + for (self.counts[0..self.count]) |count| sum += count; + return sum; + } + }; + + fn graphMetricActivePlanKey(self: *GraphIndex, metric: []const u8, job_id: u64) ![]u8 { + var job_buf: [20]u8 = undefined; + return self.graphMetricControlKeyAlloc(&.{ metric, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), "active-nodes" }); } - pub fn pruneOwnedRange(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { - var removed: usize = 0; + fn graphMetricActivePlan(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64) !GraphMetricActivePlan { + const key = try self.graphMetricActivePlanKey(metric, job_id); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricBuildManifest, + else => return err, + }; + if (raw.len < 40) return error.InvalidGraphMetricBuildManifest; + const count = std.mem.readInt(u64, raw[0..8], .little); + if (count == 0 or count > graph_metric_build_max_partition_pages or raw.len != 40 + count * 8) return error.InvalidGraphMetricBuildManifest; + var checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(raw[0 .. raw.len - 32], &checksum, .{}); + if (!std.mem.eql(u8, &checksum, raw[raw.len - 32 ..])) return error.InvalidGraphMetricBuildManifest; + const manifest = try self.metricBuildManifest(txn, metric, job_id) orelse return error.InvalidGraphMetricBuildManifest; + if (count != self.graphMetricDegreeReducePageCount(@intCast(manifest.node_count))) return error.InvalidGraphMetricBuildManifest; + var plan = GraphMetricActivePlan{ .count = @intCast(count) }; + var total: u64 = 0; + for (plan.counts[0..plan.count], 0..) |*units, i| { + units.* = std.mem.readInt(u64, raw[8 + i * 8 ..][0..8], .little); + total = std.math.add(u64, total, units.*) catch return error.InvalidGraphMetricBuildManifest; + } + if (total > manifest.node_count) return error.InvalidGraphMetricBuildManifest; + const initialized = try self.metricBuildPage(txn, metric, job_id, .initialize_ranks, 0, 0) orelse return error.InvalidGraphMetricBuildManifest; + if (initialized.state != .complete or initialized.completed_units != total) return error.InvalidGraphMetricBuildManifest; + return plan; + } - const range_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; - defer if (range_lower_owned) |key| alloc.free(key); - const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; - defer if (range_upper_owned) |key| alloc.free(key); - const range_lower = range_lower_owned orelse ""; - const range_upper = range_upper_owned orelse ""; + fn sealGraphMetricActivePlan(self: *GraphIndex, batch: anytype, metric: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob) !void { + const manifest = try self.metricBuildManifest(batch, metric, job.job_id) orelse return error.InvalidGraphMetricBuildManifest; + const count = self.graphMetricDegreeReducePageCount(@intCast(manifest.node_count)); + var encoded: [40 + 8 * graph_metric_build_max_partition_pages]u8 = undefined; + std.mem.writeInt(u64, encoded[0..8], count, .little); + for (0..count) |index| { + const leaf = try self.metricBuildPage(batch, metric, job.job_id, .initialize_ranks, 0, graph_metric_build_summary_leaf_base + index) orelse return error.InvalidGraphMetricBuildManifest; + if (leaf.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + std.mem.writeInt(u64, encoded[8 + index * 8 ..][0..8], leaf.completed_units, .little); + // Iteration zero was planned before membership was known. Finish + // empty node ranges atomically with sealing the active plan. Future + // iterations omit them entirely, including their scalar leaves. + if (leaf.completed_units != 0) continue; + for (graphMetricBuildManifestPhases(cfg.kind)) |phase| { + if (phase == .initialize_ranks or graphMetricBuildManifestPhaseRangeKind(phase) != .nodes) continue; + const data_id = graphMetricBuildPhasePageIdBase(cfg.kind, phase) + index; + for ([_]u64{ data_id, graph_metric_build_summary_leaf_base + index }) |id| { + var empty = try self.metricBuildPage(batch, metric, job.job_id, phase, 0, id) orelse continue; + if (empty.state != .pending) return error.InvalidGraphMetricBuildManifest; + empty.state = .complete; + empty.total_units = 0; + empty.completed_units = 0; + empty.output_fingerprint = 1; + empty.converged = true; + try self.putGraphMetricBuildPageInBatch(batch, metric, empty); + } + } + } + const key = try self.graphMetricActivePlanKey(metric, job.job_id); + defer self.alloc.free(key); + const payload_len = 8 + count * 8; + std.crypto.hash.sha2.Sha256.hash(encoded[0..payload_len], encoded[payload_len..][0..32], .{}); + try batch.put(key, encoded[0 .. payload_len + 32]); + } + + fn planGraphMetricIterationPagesInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + kind: GraphMetricKind, + job: GraphMetricBuildJob, + iteration: u32, + ) !void { + const active = try self.graphMetricActivePlan(batch, metric_name, job.job_id); + const planned_node_count: usize = @intCast(active.total()); + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const raw_partition_plan = try self.graphMetricPartitionPlanRaw(batch, cfg); + var partition_plan = (try self.decodeGraphMetricPartitionPlanAlloc(raw_partition_plan)) orelse + return error.InvalidGraphMetricBuildManifest; + defer partition_plan.deinit(self.alloc); + try partition_plan.validateMetricSnapshot(self, batch, cfg, job.target_generation); + if (partition_plan.node_page_count != active.count) return error.InvalidGraphMetricBuildManifest; + const output_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(output_prefix); + const iterative_planned_phases = [_]GraphMetricBuildPhase{ + .reduce_ranks, + .check_convergence, + .publish_generation, + }; + const hits_planned_phases = [_]GraphMetricBuildPhase{ + .reduce_ranks, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + }; + const planned_phases: []const GraphMetricBuildPhase = switch (kind) { + .hits_authority, .hits_hub => hits_planned_phases[0..], + else => iterative_planned_phases[0..], + }; + var created_pages: usize = 0; + for (planned_phases) |phase| { + const range_kind = graphMetricBuildManifestPhaseRangeKind(phase); + const page_count = switch (range_kind) { + .reverse_edges => partition_plan.edge_page_count, + .nodes => partition_plan.node_page_count, + else => return error.InvalidGraphMetricBuildManifest, + }; + const page_id_base = graphMetricBuildPhasePageIdBase(kind, phase); + if (graphMetricBuildPhaseNeedsSummaryPage(kind, phase, planned_node_count)) { + created_pages += try self.planGraphMetricSummaryLeaves(batch, metric_name, job, phase, iteration, partition_plan, output_prefix); + if ((try self.metricBuildPage(batch, metric_name, job.job_id, phase, iteration, 0)) == null) { + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = iteration, + .page_id = 0, + .state = .pending, + .range_kind = .summary, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = @intCast(planned_node_count), + }); + created_pages += 1; + } + } + for (0..page_count) |page_idx| { + if (range_kind == .nodes and active.counts[page_idx] == 0) continue; + const page_id = page_id_base + @as(u64, @intCast(page_idx)); + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, iteration, page_id)) |_| continue; + // Topology does not change during a fenced metric build. Clone + // the quantile boundaries persisted by iteration zero instead + // of rescanning the complete edge and node stores on every + // iteration. + const template = try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, page_id) orelse + return error.InvalidGraphMetricBuildManifest; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = iteration, + .page_id = page_id, + .state = .pending, + .range_kind = template.range_kind, + .range_lower = template.range_lower, + .range_upper = template.range_upper, + .output_prefix = output_prefix, + .worker_id = "", + .total_units = if (range_kind == .nodes) active.counts[page_idx] else template.total_units, + }); + created_pages += 1; + } + } + if (created_pages != 0) { + var manifest = try self.metricBuildManifest(batch, metric_name, job.job_id) orelse return error.GraphMetricBuildManifestNotFound; + manifest.page_count += created_pages; + try self.putGraphMetricBuildManifestInBatch(batch, metric_name, manifest); + } + } + + fn planPageRankIterationPagesInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job: GraphMetricBuildJob, + iteration: u32, + ) !void { + try self.planGraphMetricIterationPagesInBatch(batch, metric_name, .pagerank, job, iteration); + } + + fn graphMetricDegreeScanPageCount(self: *GraphIndex, edge_key_count: usize) usize { + if (edge_key_count == 0) return 1; + return @min( + graph_metric_build_max_partition_pages, + std.math.divCeil(usize, edge_key_count, self.test_partition_target_units orelse graph_metric_build_checkpoint_scan_units) catch 1, + ); + } + + fn graphMetricDegreeReducePageCount(self: *GraphIndex, node_key_count: usize) usize { + if (node_key_count == 0) return 1; + return @min( + graph_metric_build_max_partition_pages, + std.math.divCeil(usize, node_key_count, self.test_partition_target_units orelse graph_metric_build_checkpoint_reduce_units) catch 1, + ); + } + + fn graphMetricSummaryLeafCount(kind: GraphMetricKind, node_count: usize, partition_count: usize) usize { + return if (kind == .degree and node_count <= graph_metric_build_checkpoint_reduce_units) 0 else partition_count; + } + + /// Reuse the immutable node quantiles for independently leased scalar + /// producers. The root combines at most 256 durable records, never V nodes. + fn planGraphMetricSummaryLeaves(self: *GraphIndex, batch: anytype, metric_name: []const u8, job: GraphMetricBuildJob, phase: GraphMetricBuildPhase, iteration: u32, plan: GraphMetricPartitionPlan, output_prefix: []const u8) !usize { + var created: usize = 0; + const active = if (iteration != 0) try self.graphMetricActivePlan(batch, metric_name, job.job_id) else null; + const kind = (self.metricConfig(metric_name) orelse return error.MetricNotReady).kind; + const count = graphMetricSummaryLeafCount(kind, @intCast(plan.node_count), plan.node_page_count); + for (0..count) |index| { + if (active) |work| if (work.counts[index] == 0) continue; + const id = graph_metric_build_summary_leaf_base + index; + if (try self.metricBuildPage(batch, metric_name, job.job_id, phase, iteration, id)) |_| continue; + const template = if (iteration != 0) + try self.metricBuildPage(batch, metric_name, job.job_id, phase, 0, id) orelse return error.InvalidGraphMetricBuildManifest + else + null; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, .{ + .job_id = job.job_id, + .phase = phase, + .iteration = iteration, + .page_id = id, + .range_kind = .summary, + .range_lower = if (template) |page| page.range_lower else if (index < plan.node_boundaries.items.len) plan.node_boundaries.items[index] else "", + .range_upper = if (template) |page| page.range_upper else if (index + 1 < plan.node_boundaries.items.len) plan.node_boundaries.items[index + 1] else "", + .output_prefix = output_prefix, + .total_units = if (active) |work| work.counts[index] else graphMetricPartitionSpan(@intCast(plan.node_count), count, index).len, + }); + created += 1; + } + return created; + } + + fn graphMetricBuildPhaseNeedsSummaryPage(kind: GraphMetricKind, phase: GraphMetricBuildPhase, node_count: usize) bool { + // Edge density is independent of node count. Every iterative build + // uses bounded ordinal leaves and a scalar-only normalization root. + if (kind != .degree) return phase == .initialize_ranks or phase == .reduce_ranks or + ((kind == .hits_authority or kind == .hits_hub) and phase == .hits_hub_reduce_ranks); + // Degree only counts nodes; its small inline path is cardinality-bound. + return node_count > graph_metric_build_target_reduce_page_units and phase == .reduce_ranks; + } + + pub fn graphMetricPlannedBuildControlRecordEstimate(self: *GraphIndex, cfg: GraphMetricConfig) usize { + const edge_count = std.math.cast(usize, self.edge_count) orelse std.math.maxInt(usize); + const node_count = std.math.cast(usize, self.node_count) orelse std.math.maxInt(usize); + const reverse_edge_page_count = if (cfg.kind == .degree or graphMetricKindUsesIterativeBuild(cfg.kind)) + self.graphMetricDegreeScanPageCount(edge_count) + else + 0; + const node_page_count = if (cfg.kind == .degree or graphMetricKindUsesIterativeBuild(cfg.kind)) + self.graphMetricDegreeReducePageCount(node_count) + else + 0; + const cleanup_page_count = graphMetricBuildCleanupPageCount(cfg.kind); + var planned_page_count: usize = 0; + for (graphMetricBuildManifestPhases(cfg.kind)) |phase| { + planned_page_count +|= graphMetricBuildPlannedPageCountForPhase(cfg.kind, phase, reverse_edge_page_count, node_page_count, cleanup_page_count); + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, phase, node_count)) + planned_page_count +|= 1 + graphMetricSummaryLeafCount(cfg.kind, node_count, node_page_count); + } + return planned_page_count +| 1; + } + + fn graphMetricKindUsesIterativeBuild(kind: GraphMetricKind) bool { + return switch (kind) { + .pagerank, + .eigenvector, + .hits_authority, + .hits_hub, + => true, + .degree => false, + }; + } + + fn graphMetricKindUsesPlannedIterativeRunner(kind: GraphMetricKind) bool { + return switch (kind) { + .pagerank, + .eigenvector, + .hits_authority, + .hits_hub, + => true, + .degree, + => false, + }; + } + + fn graphMetricBuildPlannedPageCountForPhase( + kind: GraphMetricKind, + phase: GraphMetricBuildPhase, + reverse_edge_page_count: usize, + node_page_count: usize, + cleanup_page_count: usize, + ) usize { + return switch (kind) { + .degree => switch (phase) { + .scan_edges_and_out_degree => reverse_edge_page_count, + .reduce_ranks => node_page_count, + .cleanup_old_generations => cleanup_page_count, + else => 1, + }, + .pagerank, + .eigenvector, + .hits_authority, + .hits_hub, + => switch (phase) { + .scan_edges_and_out_degree, + .iterate_contributions, + .hits_hub_contributions, + => reverse_edge_page_count, + .initialize_ranks, + .reduce_ranks, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + => node_page_count, + .cleanup_old_generations => cleanup_page_count, + else => 1, + }, + }; + } + + fn graphMetricBuildPhasePageIdBase(kind: GraphMetricKind, phase: GraphMetricBuildPhase) u64 { + const phases = graphMetricBuildManifestPhases(kind); + for (phases, 0..) |candidate, i| { + if (candidate == phase) return @intCast(i); + } + return 0; + } + + fn graphMetricBuildPhasePublishIteration(kind: GraphMetricKind, phase: GraphMetricBuildPhase, publish_iteration: u32) u32 { + if (!graphMetricKindIsIterative(kind)) return publish_iteration; + return switch (phase) { + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .hits_hub_contributions, + => 0, + .reduce_ranks, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + => publish_iteration, + else => 0, + }; + } + + fn graphMetricBuildCleanupPageCount(kind: GraphMetricKind) usize { + return switch (kind) { + .degree => 2, + .pagerank => 3, + .eigenvector, + => 1, + .hits_authority, + .hits_hub, + => 4, + }; + } + + fn graphMetricBuildCleanupPageIsFinal(kind: GraphMetricKind, page: GraphMetricBuildPage) bool { + if (page.output_prefix.len == 0) return true; + const cleanup_page_count = graphMetricBuildCleanupPageCount(kind); + if (cleanup_page_count == 0) return true; + return page.page_id + 1 >= cleanup_page_count; + } + + fn graphMetricBuildCleanupPagePrefixAlloc( + self: *GraphIndex, + metric_name: []const u8, + kind: GraphMetricKind, + job_id: u64, + page: GraphMetricBuildPage, + ) ![]u8 { + if (graphMetricBuildCleanupPageIsFinal(kind, page)) { + return try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + } + return switch (kind) { + .degree => try self.graphMetricBuildDegreePartialPrefixAlloc(metric_name, job_id), + .pagerank => switch (page.page_id) { + 0 => try self.graphMetricBuildPageRankOutDegreePartialPrefixAlloc(metric_name, job_id), + 1 => try self.graphMetricBuildPageRankNodePartialPrefixAlloc(metric_name, job_id), + else => try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id), + }, + .hits_authority, + .hits_hub, + => switch (page.page_id) { + 0 => try self.graphMetricBuildHitsHubRawNamespacePrefixAlloc(metric_name, job_id), + 1 => try self.graphMetricBuildHitsHubRawSummaryNamespacePrefixAlloc(metric_name, job_id), + 2 => try self.graphMetricBuildHitsRankNamespacePrefixAlloc(metric_name, job_id), + else => try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id), + }, + .eigenvector, + => try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id), + }; + } + + const GraphMetricPartitionSpan = struct { + start: usize, + len: usize, + }; + + fn graphMetricPartitionSpan(total_units: usize, page_count: usize, page_index: usize) GraphMetricPartitionSpan { + std.debug.assert(page_count > 0); + std.debug.assert(page_index < page_count); + const units_per_page = total_units / page_count; + const remainder = total_units % page_count; + const extra_before = @min(page_index, remainder); + const len = units_per_page + @intFromBool(page_index < remainder); + return .{ + .start = page_index * units_per_page + extra_before, + .len = len, + }; + } + + /// All metric edge kernels read balanced pages from the authoritative + /// reverse projection. Their attempt outputs are page-sharded and adopted + /// idempotently, so hubs can span pages without sacrificing correctness or + /// worker parallelism. + fn collectBalancedEdgePartitionBoundariesForIndexInBatch( + self: *GraphIndex, + batch: anytype, + expected_count: usize, + page_count: usize, + out: *std.ArrayListUnmanaged([]u8), + page_units: *std.ArrayListUnmanaged(u64), + ) !void { + if (out.items.len != 0 or page_units.items.len != 0 or page_count == 0) return error.InvalidGraphMetricBuildManifest; + if (expected_count == 0) { + try page_units.append(self.alloc, 0); + return; + } + try out.ensureTotalCapacity(self.alloc, page_count); + try page_units.ensureTotalCapacity(self.alloc, page_count); + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + var ordinal: usize = 0; + var next_page: usize = 0; + var next_boundary = graphMetricPartitionSpan(expected_count, page_count, next_page).start; + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + if (!graphIndexEdgeKeyMatchesIndex(entry.key, self.index_name)) continue; + if (next_page < page_count and ordinal == next_boundary) { + const boundary = try self.alloc.dupe(u8, entry.key); + out.append(self.alloc, boundary) catch |err| { + self.alloc.free(boundary); + return err; + }; + try page_units.append(self.alloc, @intCast(graphMetricPartitionSpan(expected_count, page_count, next_page).len)); + next_page += 1; + if (next_page < page_count) { + next_boundary = graphMetricPartitionSpan(expected_count, page_count, next_page).start; + } + } + ordinal += 1; + } + if (ordinal != expected_count or out.items.len != page_count or page_units.items.len != page_count) { + return error.InvalidGraphMetricBuildManifest; + } + } + + fn collectGraphNodePartitionBoundariesInBatch( + self: *GraphIndex, + batch: anytype, + expected_count: usize, + page_count: usize, + out: *std.ArrayListUnmanaged([]u8), + ) !void { + if (out.items.len != 0 or page_count == 0) return error.InvalidGraphMetricBuildManifest; + if (expected_count == 0) return; + try out.ensureTotalCapacity(self.alloc, page_count); + const prefix = "meta:node_ref:"; + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + var ordinal: usize = 0; + var next_page: usize = 0; + var next_boundary = graphMetricPartitionSpan(expected_count, page_count, next_page).start; + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (next_page < page_count and ordinal == next_boundary) { + const boundary = try self.alloc.dupe(u8, entry.key[prefix.len..]); + out.append(self.alloc, boundary) catch |err| { + self.alloc.free(boundary); + return err; + }; + next_page += 1; + if (next_page < page_count) { + next_boundary = graphMetricPartitionSpan(expected_count, page_count, next_page).start; + } + } + ordinal += 1; + } + if (ordinal != expected_count or out.items.len != page_count) return error.InvalidGraphMetricBuildManifest; + } + + fn firstReverseEdgeKeyForIndexInBatch(self: *GraphIndex, batch: anytype) ![]u8 { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + if (!graphIndexEdgeKeyMatchesIndex(entry.key, self.index_name)) continue; + return try self.alloc.dupe(u8, entry.key); + } + return ""; + } + + fn claimGraphMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + ) !?GraphMetricBuildPage { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.claimGraphMetricBuildPageAt(metric_name, job_id, phase, iteration, page_id, worker_id, now_ms); + } + + fn claimNextGraphMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + worker_id: []const u8, + ) !?GraphMetricBuildPage { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.claimNextGraphMetricBuildPageAt(metric_name, job_id, phase, iteration, worker_id, now_ms); + } + + pub fn claimNextGraphMetricBuildPageAt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + worker_id: []const u8, + now_ms: u64, + ) !?GraphMetricBuildPage { + if (worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + var candidate_page_id: ?u64 = null; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + if (try self.metricBuildPage(&batch, metric_name, job_id, phase, iteration, 0)) |summary_page| { + if (summary_page.range_kind == .summary and summary_page.state != .complete) { + // Scalar leaves may run concurrently, but neither the root nor + // data reducers can observe a partially adopted normalization. + var waiting_for_leaves = false; + const active = if (iteration != 0) try self.graphMetricActivePlan(&batch, metric_name, job_id) else null; + var leaf_index: u64 = 0; + while (leaf_index < graph_metric_build_max_partition_pages) : (leaf_index += 1) { + if (active) |work| { + if (leaf_index >= work.count) break; + if (work.counts[leaf_index] == 0) continue; + } + const leaf = try self.metricBuildPage(&batch, metric_name, job_id, phase, iteration, graph_metric_build_summary_leaf_base + leaf_index) orelse { + if (active != null) return error.InvalidGraphMetricBuildManifest; + break; + }; + if (leaf.range_kind != .summary) return error.InvalidGraphMetricBuildManifest; + if (leaf.state == .complete) continue; + waiting_for_leaves = true; + if (graphMetricBuildPageClaimable(leaf, worker_id, now_ms)) { + candidate_page_id = leaf.page_id; + break; + } + } + if (candidate_page_id != null) { + // Claim the independently fenced producer below. + } else if (!waiting_for_leaves and graphMetricBuildPageClaimable(summary_page, worker_id, now_ms)) { + candidate_page_id = 0; + } else { + // Normalization is a dependency of every data partition; + // do not let another worker race ahead with a partial or + // stale scalar. + try batch.commit(); + return null; + } + } + } + if (candidate_page_id == null) { + const progress = try self.metricBuildPhaseProgress(&batch, metric_name, job_id, phase, iteration); + if (progress) |current| { + if (current.pending_pages == 0 and current.failed_pages == 0 and current.leased_pages == 0) { + try batch.commit(); + return null; + } + } + const prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, job_id, phase, iteration); + defer self.alloc.free(prefix); + // Cleanup pages have ordered destructive dependencies and must + // retain prefix-order claims. Computational phases are independent + // and can safely use the durable round-robin cursor. + const cursor_page_id = if (phase != .cleanup_old_generations) + if (progress) |current| current.next_claim_page_id else 0 + else + 0; + const start_key = try self.graphMetricBuildPageKeyAlloc(metric_name, job_id, phase, iteration, cursor_page_id); + defer self.alloc.free(start_key); + var cur = try batch.openCursor(); + defer cur.close(); + var pass: usize = 0; + while (pass < 2 and candidate_page_id == null) : (pass += 1) { + if (pass == 1 and cursor_page_id == 0) break; + var entry_opt = try cur.seekAtOrAfter(if (pass == 0) start_key else prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (pass == 1 and std.mem.order(u8, entry.key, start_key) != .lt) break; + const page = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + if (page.job_id != job_id or page.phase != phase or page.iteration != iteration) return error.InvalidGraphMetricBuildPage; + if (!graphMetricBuildPageClaimable(page, worker_id, now_ms)) continue; + candidate_page_id = page.page_id; + break; + } + } + } + const page_id = candidate_page_id orelse { + try batch.commit(); + return null; + }; + const claimed = try self.claimGraphMetricBuildPageInBatch(&batch, metric_name, job_id, phase, iteration, page_id, worker_id, now_ms); + if (claimed != null and phase != .cleanup_old_generations) { + if (try self.metricBuildPhaseProgress(&batch, metric_name, job_id, phase, iteration)) |current| { + var next = current; + next.next_claim_page_id = page_id +% 1; + try self.putGraphMetricBuildPhaseProgressInBatch(&batch, metric_name, next); + } + } + try batch.commit(); + return claimed; + } + + fn graphMetricBuildPageClaimable(page: GraphMetricBuildPage, worker_id: []const u8, now_ms: u64) bool { + // Continuing a live lease consumes no attempt and must preserve its + // checkpoint even on the final allowed attempt. Expired leases require + // a fresh attempt, including when the previous owner comes back. + return switch (page.state) { + .complete => false, + .leased => if (page.lease_expires_at_ms > now_ms) + std.mem.eql(u8, page.worker_id, worker_id) + else + page.attempt < graph_metric_build_max_page_attempts, + .pending, .failed => page.attempt < graph_metric_build_max_page_attempts, + }; + } + + fn claimGraphMetricBuildPageAt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + now_ms: u64, + ) !?GraphMetricBuildPage { + if (worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const claimed = try self.claimGraphMetricBuildPageInBatch(&batch, metric_name, job_id, phase, iteration, page_id, worker_id, now_ms); + try batch.commit(); + return claimed; + } + + fn claimGraphMetricBuildPageInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + now_ms: u64, + ) !?GraphMetricBuildPage { + var page = try self.metricBuildPage(batch, metric_name, job_id, phase, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (phase == .cleanup_old_generations) { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (graphMetricBuildCleanupPageIsFinal(cfg.kind, page) and page.cursor.len == 0 and page.completed_units == 0) { + // The final page removes the whole job namespace. It must not + // race a narrower cleanup lease which could otherwise recreate + // progress records after completion. Once bounded final-page + // cleanup has a cursor, its predecessor records may themselves + // have been retired and the durable cursor is the resume fence. + // Most metric kinds assign cleanup page IDs from zero, while a + // single-page cleanup may retain its phase-manifest page ID. + // Dependency cardinality, not that physical ID, defines the + // predecessor set. + const cleanup_page_count = graphMetricBuildCleanupPageCount(cfg.kind); + for (0..cleanup_page_count - 1) |prior_page_id| { + const prior = try self.metricBuildPage( + batch, + metric_name, + job_id, + phase, + iteration, + @intCast(prior_page_id), + ) orelse return error.InvalidGraphMetricBuildManifest; + if (prior.state != .complete) return null; + } + } + } + if (!graphMetricBuildPageClaimable(page, worker_id, now_ms)) return null; + if (page.state == .leased and page.lease_expires_at_ms > now_ms) { + page.worker_id = worker_id; + page.lease_expires_at_ms = now_ms + graph_metric_local_build_lease_ms; + page.last_error = ""; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, page); + return page; + } + page.state = .leased; + page.worker_id = worker_id; + page.lease_expires_at_ms = now_ms + graph_metric_local_build_lease_ms; + page.attempt += 1; + page.cursor = ""; + page.completed_units = 0; + page.last_error = ""; + page.output_fingerprint = 0; + page.max_delta = 0.0; + page.total_delta = 0.0; + page.rank_sum = 0.0; + page.converged = false; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, page); + return page; + } + + fn updateGraphMetricBuildPageProgress( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !GraphMetricBuildPage { + return try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job_id, phase, iteration, page_id, worker_id, 0, cursor, completed_units, total_units); + } + + pub fn updateGraphMetricBuildPageProgressForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const page = try self.updateGraphMetricBuildPageProgressInBatch( + &batch, + metric_name, + job_id, + phase, + iteration, + page_id, + worker_id, + expected_attempt, + cursor, + completed_units, + total_units, + ); + try batch.commit(); + return page; + } + + /// Advances a leased page in the same transaction as the caller's output + /// writes. Additive chunks use page-local cursors so a crash can expose + /// neither output without its cursor nor a cursor without its output. + fn updateGraphMetricBuildPageProgressInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !GraphMetricBuildPage { + var page = try self.metricBuildPage(batch, metric_name, job_id, phase, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state != .leased or !std.mem.eql(u8, page.worker_id, worker_id)) return error.GraphMetricBuildPageNotLeased; + if (expected_attempt != 0 and page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + if (total_units != 0 and completed_units > total_units) return error.InvalidGraphMetricBuildProgress; + page.cursor = cursor; + page.completed_units = completed_units; + page.total_units = if (total_units != 0) total_units else page.total_units; + page.last_error = ""; + try self.putGraphMetricBuildPageInBatch(batch, metric_name, page); + return page; + } + + fn updateGraphMetricBuildSummaryPageProgressForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + cursor: []const u8, + completed_units: u64, + total_units: u64, + rank_sum: f64, + seed_mass: f64, + ) !GraphMetricBuildPage { + if (!std.math.isFinite(rank_sum) or !std.math.isFinite(seed_mass) or seed_mass < 0) return error.InvalidGraphMetricScore; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job_id, phase, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.range_kind != .summary) return error.InvalidGraphMetricBuildPage; + if (page.state != .leased or !std.mem.eql(u8, page.worker_id, worker_id) or page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + if (total_units != 0 and completed_units > total_units) return error.InvalidGraphMetricBuildProgress; + page.cursor = cursor; + page.completed_units = completed_units; + page.total_units = if (total_units != 0) total_units else page.total_units; + page.rank_sum = rank_sum; + page.total_delta = seed_mass; + page.last_error = ""; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn updateGraphMetricBuildConvergencePageProgressForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + cursor: []const u8, + completed_units: u64, + total_units: u64, + max_delta: f64, + total_delta: f64, + rank_sum: f64, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job_id, .check_convergence, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state != .leased or !std.mem.eql(u8, page.worker_id, worker_id)) return error.GraphMetricBuildPageNotLeased; + if (expected_attempt != 0 and page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + if (total_units != 0 and completed_units > total_units) return error.InvalidGraphMetricBuildProgress; + if (!std.math.isFinite(max_delta) or !std.math.isFinite(total_delta) or !std.math.isFinite(rank_sum)) return error.InvalidGraphMetricScore; + page.cursor = cursor; + page.completed_units = completed_units; + page.total_units = if (total_units != 0) total_units else page.total_units; + page.last_error = ""; + page.max_delta = max_delta; + page.total_delta = total_delta; + page.rank_sum = rank_sum; + page.converged = false; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn completeGraphMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + completed_units: u64, + output_fingerprint: u64, + ) !GraphMetricBuildPage { + return try self.completeGraphMetricBuildPageForAttempt(metric_name, job_id, phase, iteration, page_id, worker_id, 0, completed_units, output_fingerprint); + } + + pub fn completeGraphMetricBuildPageForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + completed_units: u64, + output_fingerprint: u64, + ) !GraphMetricBuildPage { + return try self.completeGraphMetricBuildPageWithConvergenceForAttempt(metric_name, job_id, phase, iteration, page_id, worker_id, expected_attempt, completed_units, output_fingerprint, 0.0, 0.0, 0.0, false); + } + + fn completeGraphMetricBuildConvergencePage( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + page_id: u64, + worker_id: []const u8, + completed_units: u64, + output_fingerprint: u64, + max_delta: f64, + total_delta: f64, + rank_sum: f64, + converged: bool, + ) !GraphMetricBuildPage { + return try self.completeGraphMetricBuildConvergencePageForAttempt(metric_name, job_id, iteration, page_id, worker_id, 0, completed_units, output_fingerprint, max_delta, total_delta, rank_sum, converged); + } + + fn completeGraphMetricBuildConvergencePageForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + completed_units: u64, + output_fingerprint: u64, + max_delta: f64, + total_delta: f64, + rank_sum: f64, + converged: bool, + ) !GraphMetricBuildPage { + return try self.completeGraphMetricBuildPageWithConvergenceForAttempt(metric_name, job_id, .check_convergence, iteration, page_id, worker_id, expected_attempt, completed_units, output_fingerprint, max_delta, total_delta, rank_sum, converged); + } + + fn completeGraphMetricBuildPageWithConvergenceForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + completed_units: u64, + output_fingerprint: u64, + max_delta: f64, + total_delta: f64, + rank_sum: f64, + converged: bool, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job_id, phase, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state == .complete) { + if (expected_attempt != 0 and page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + if (page.output_fingerprint != output_fingerprint) return error.GraphMetricBuildPageOutputMismatch; + page.worker_id = ""; + page.cursor = ""; + page.last_error = ""; + try batch.commit(); + return page; + } + if (page.state != .leased or !std.mem.eql(u8, page.worker_id, worker_id)) return error.GraphMetricBuildPageNotLeased; + if (expected_attempt != 0 and page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + page.state = .complete; + page.worker_id = worker_id; + page.lease_expires_at_ms = 0; + page.completed_units = completed_units; + // Summary pages begin with the global graph cardinality as a planning + // estimate. Once their filtered namespace is exhausted, calibrate the + // durable total to the exact number visited so a complete page never + // reports misleading partial progress to operators. + if (page.range_kind == .summary) page.total_units = completed_units; + page.cursor = ""; + page.last_error = ""; + page.output_fingerprint = output_fingerprint; + page.max_delta = max_delta; + page.total_delta = total_delta; + page.rank_sum = rank_sum; + page.converged = converged; + // Target/page contribution shards are the canonical reduce input. They + // remain immutable after page completion so reducers can stream them + // in target then page order, independent of worker completion order. + // The consumer phase barrier reclaims inputs in bounded batches only + // after every output is durable; abandoned jobs use bounded job cleanup. + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn failGraphMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + last_error: []const u8, + ) !GraphMetricBuildPage { + return try self.failGraphMetricBuildPageForAttempt(metric_name, job_id, phase, iteration, page_id, worker_id, 0, last_error); + } + + fn failGraphMetricBuildPageForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + worker_id: []const u8, + expected_attempt: u64, + last_error: []const u8, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job_id, phase, iteration, page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state != .leased or !std.mem.eql(u8, page.worker_id, worker_id)) return error.GraphMetricBuildPageNotLeased; + if (expected_attempt != 0 and page.attempt != expected_attempt) return error.GraphMetricBuildPageNotLeased; + page.state = .failed; + page.worker_id = worker_id; + page.lease_expires_at_ms = 0; + page.last_error = last_error; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn validateGraphMetricBuildPageExecutionLease(_: *GraphIndex, claimed: GraphMetricBuildPage, current: GraphMetricBuildPage) !void { + if (current.state != .leased) return error.GraphMetricBuildPageNotLeased; + if (!std.mem.eql(u8, current.worker_id, claimed.worker_id)) return error.GraphMetricBuildPageNotLeased; + if (current.attempt != claimed.attempt) return error.GraphMetricBuildPageNotLeased; + } + + fn summarizeGraphMetricBuildPhase( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !GraphMetricBuildPhaseSummary { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.job_id != job_id) return error.GraphMetricBuildJobMismatch; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const summary = try self.summarizeGraphMetricBuildPhaseInBatch(&batch, metric_name, cfg, job, phase, iteration); + try batch.commit(); + return summary; + } + + fn summarizeGraphMetricBuildPhaseInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !GraphMetricBuildPhaseSummary { + if (try self.metricBuildPhaseProgress(batch, metric_name, job.job_id, phase, iteration)) |progress| { + if (progress.completed_pages != progress.expected_pages) { + const state: GraphMetricBuildPhaseState = if (progress.failed_pages != 0) .failed else .pending; + const summary = GraphMetricBuildPhaseSummary{ + .job_id = job.job_id, + .phase = phase, + .iteration = iteration, + .state = state, + .expected_pages = progress.expected_pages, + .completed_pages = progress.completed_pages, + .failed_pages = progress.failed_pages, + .completed_units = progress.completed_units, + .total_units = progress.total_units, + }; + try self.putGraphMetricBuildPhaseSummaryInBatch(batch, metric_name, summary); + return summary; + } + if (try self.metricBuildPhaseSummary(batch, metric_name, job.job_id, phase, iteration)) |cached| { + if (cached.state == .complete and cached.expected_pages == progress.expected_pages and + cached.completed_pages == progress.completed_pages and cached.completed_units == progress.completed_units and + cached.total_units == progress.total_units) + { + return cached; + } + } + } + var expected_pages: u64 = 0; + var completed_pages: u64 = 0; + var failed_pages: u64 = 0; + var completed_units: u64 = 0; + var total_units: u64 = 0; + var output_fingerprint: u64 = 0; + var max_delta: f64 = 0.0; + var total_delta: f64 = 0.0; + var rank_sum: f64 = 0.0; + var all_completed_pages_converged = true; + var page_found = false; + const page_prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, job.job_id, phase, iteration); + defer self.alloc.free(page_prefix); + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(page_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, page_prefix)) break; + const page = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + if (page.job_id != job.job_id or page.phase != phase or page.iteration != iteration) return error.InvalidGraphMetricBuildPage; + expected_pages += 1; + page_found = true; + total_units += page.total_units; + completed_units += page.completed_units; + switch (page.state) { + .complete => { + completed_pages += 1; + output_fingerprint ^= page.output_fingerprint; + max_delta = @max(max_delta, page.max_delta); + total_delta += page.total_delta; + rank_sum += page.rank_sum; + all_completed_pages_converged = all_completed_pages_converged and page.converged; + }, + .failed => failed_pages += 1, + .pending, .leased => {}, + } + } + if (!page_found) return error.GraphMetricBuildPhaseNotFound; + const state: GraphMetricBuildPhaseState = if (failed_pages != 0) + .failed + else if (completed_pages == expected_pages) + .complete + else + .pending; + const summary = GraphMetricBuildPhaseSummary{ + .job_id = job.job_id, + .phase = phase, + .iteration = iteration, + .state = state, + .expected_pages = expected_pages, + .completed_pages = completed_pages, + .failed_pages = failed_pages, + .completed_units = completed_units, + .total_units = total_units, + .output_fingerprint = output_fingerprint, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .converged = phase == .check_convergence and state == .complete and all_completed_pages_converged and total_delta <= cfg.tolerance, + }; + try self.putGraphMetricBuildPhaseSummaryInBatch(batch, metric_name, summary); + try self.sealTopologyInBatch(batch, metric_name, cfg, job, summary); + return summary; + } + + fn graphMetricBuildPhaseExhaustedPageAlloc( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + now_ms: u64, + ) !?GraphMetricBuildPageExhaustion { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.metricBuildPhaseProgress(&txn, metric_name, job_id, phase, iteration)) |progress| { + if (progress.max_attempt_pages == 0) return null; + } + const page_prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, job_id, phase, iteration); + defer self.alloc.free(page_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(page_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, page_prefix)) break; + const page = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + if (page.job_id != job_id or page.phase != phase or page.iteration != iteration) return error.InvalidGraphMetricBuildPage; + const exhausted = switch (page.state) { + .failed => page.attempt >= graph_metric_build_max_page_attempts, + .leased => page.attempt >= graph_metric_build_max_page_attempts and page.lease_expires_at_ms <= now_ms, + .pending, .complete => false, + }; + if (!exhausted) continue; + return .{ + .phase = page.phase, + .iteration = page.iteration, + .page_id = page.page_id, + .attempt = page.attempt, + .last_error = if (page.last_error.len > 0) try self.alloc.dupe(u8, page.last_error) else "", + }; + } + return null; + } + + fn advanceGraphMetricBuildPhaseIfReady( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !bool { + return self.advanceGraphMetricBuildPhaseWithRetirement(metric_name, job_id, phase, iteration, null); + } + + fn advanceGraphMetricBuildPhaseWithRetirement(self: *GraphIndex, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, iteration: u32, retired: ?*usize) !bool { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.job_id != job_id) return error.GraphMetricBuildJobMismatch; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + try self.validateGraphMetricBuildExecutionInTxn(&batch, metric_name, job, cfg); + const summary = try self.summarizeGraphMetricBuildPhaseInBatch(&batch, metric_name, cfg, job, phase, iteration); + if (summary.state != .complete) { + try batch.commit(); + return false; + } + // Raw vectors are immutable until every consumer has committed its + // output. Lease takeover may restart a page, so checkpoint-local input + // deletion is unsafe. Drain one bounded batch at the phase barrier; + // persist the last deleted key so LSM tombstones are not revisited. + // A delayed coordinator must never clean a different active phase. + if (job.phase != phase or job.iteration != iteration) { + try batch.commit(); + return false; + } + if (phase == .check_convergence or phase == .cleanup_old_generations) + self.sealed_vectors.retire(self.alloc, sealedVectorScope(metric_name)); + if (phase == .initialize_ranks and graphMetricKindUsesPlannedIterativeRunner(cfg.kind)) + try self.sealGraphMetricActivePlan(&batch, metric_name, cfg, job); + if ((phase == .reduce_ranks and graphMetricKindUsesPlannedIterativeRunner(cfg.kind)) or phase == .hits_hub_reduce_ranks) { + var job_buf: [20]u8 = undefined; + var iteration_buf: [10]u8 = undefined; + const retirement_prefix = try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), "retirement", @tagName(phase), try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}) }); + defer self.alloc.free(retirement_prefix); + const raw_prefix = if (self.topology_preparation_only) try self.alloc.dupe(u8, "") else try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), "vector", if (phase == .hits_hub_reduce_ranks) "raw_hub" else "raw_rank", try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}) }); + defer self.alloc.free(raw_prefix); + // Iteration zero has now packed every consumer's adjacency. + // Retire producer fragments; later iterations use only receipts + // and dense blocks, including after a worker lease takeover. + const staged_prefix = if (iteration == 0) + try self.ordinalAdjacencyPhasePrefixAlloc(metric_name, job_id, if (phase == .hits_hub_reduce_ranks) .hits_hub_contributions else .iterate_contributions) + else + try self.alloc.dupe(u8, ""); + defer self.alloc.free(staged_prefix); + var remaining = graph_metric_build_adoption_page_units; + for ([_][]const u8{ raw_prefix, staged_prefix }) |prefix| { + if (prefix.len == 0) continue; + const cursor_key = try std.fmt.allocPrint(self.alloc, "{s}{s}", .{ retirement_prefix, prefix }); + defer self.alloc.free(cursor_key); + const saved = batch.get(cursor_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + // A zero-byte sentinel marks fully retired namespaces. It + // remains until job cleanup and makes replay an O(1) check. + if (saved.len == 1 and saved[0] == 0) continue; + if (saved.len != 0 and !std.mem.startsWith(u8, saved, prefix)) return error.InvalidGraphMetricBuildManifest; + if (remaining == 0) { + try batch.commit(); + return false; + } + const cleanup = try self.deleteKeysWithPrefixPageInBatch(&batch, prefix, saved, remaining); + defer if (cleanup.cursor.len > 0) self.alloc.free(cleanup.cursor); + try batch.put(cursor_key, if (cleanup.reached_end) "\x00" else cleanup.cursor); + remaining -= cleanup.removed; + if (retired) |count| count.* += cleanup.removed; + if (!cleanup.reached_end) { + try batch.commit(); + return false; + } + } + } + if (graphMetricKindUsesPlannedIterativeRunner(cfg.kind) and phase == .check_convergence and !summary.converged and @as(u64, iteration) + 1 < cfg.max_iterations) { + const iteration_summary = GraphMetricBuildIterationSummary{ + .job_id = job.job_id, + .iteration = iteration, + .expected_pages = summary.expected_pages, + .completed_pages = summary.completed_pages, + .max_delta = summary.max_delta, + .total_delta = summary.total_delta, + .rank_sum = summary.rank_sum, + .converged = false, + .fixed_iteration_limit = false, + .output_fingerprint = summary.output_fingerprint, + }; + try self.putGraphMetricBuildIterationSummaryInBatch(&batch, metric_name, iteration_summary); + const next_iteration = iteration + 1; + try self.planGraphMetricIterationPagesInBatch(&batch, metric_name, cfg.kind, job, next_iteration); + job.phase = .reduce_ranks; + job.iteration = next_iteration; + job.updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + job.completed_units = summary.completed_units; + job.total_units = summary.total_units; + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, job); + + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + if (batch.get(lease_key)) |raw| { + if (decodeGraphMetricBuildLease(raw)) |lease| { + if (lease.job_id == job.job_id) { + var updated_lease = lease; + updated_lease.phase = .reduce_ranks; + updated_lease.iteration = next_iteration; + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(updated_lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(updated_lease, encoded); + try batch.put(lease_key, encoded); + } + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + try batch.commit(); + return true; + } + if (graphMetricKindUsesPlannedIterativeRunner(cfg.kind) and phase == .check_convergence) { + const iteration_summary = GraphMetricBuildIterationSummary{ + .job_id = job.job_id, + .iteration = iteration, + .expected_pages = summary.expected_pages, + .completed_pages = summary.completed_pages, + .max_delta = summary.max_delta, + .total_delta = summary.total_delta, + .rank_sum = summary.rank_sum, + .converged = summary.converged, + .fixed_iteration_limit = !summary.converged and @as(u64, iteration) + 1 >= cfg.max_iterations, + .output_fingerprint = summary.output_fingerprint, + }; + try self.putGraphMetricBuildIterationSummaryInBatch(&batch, metric_name, iteration_summary); + } + const next_phase = (if (iteration != 0 and phase == .reduce_ranks and + (cfg.kind == .hits_authority or cfg.kind == .hits_hub)) + .hits_hub_reduce_ranks + else + graphMetricBuildManifestNextPhase(cfg.kind, phase)) orelse { + try batch.commit(); + return false; + }; + if ((cfg.kind == .hits_authority or cfg.kind == .hits_hub) and next_phase == .hits_hub_contributions) { + const hub_raw_prefix = try self.graphMetricBuildHitsHubRawPrefixAlloc(metric_name, job.job_id, iteration); + defer self.alloc.free(hub_raw_prefix); + _ = try self.deleteKeysWithPrefixInBatch(&batch, hub_raw_prefix); + const hub_raw_summary_key = try self.graphMetricBuildHitsHubRawSummaryKeyAlloc(metric_name, job.job_id, iteration); + defer self.alloc.free(hub_raw_summary_key); + batch.delete(hub_raw_summary_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + job.phase = next_phase; + job.iteration = iteration; + job.updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + job.completed_units = summary.completed_units; + job.total_units = summary.total_units; + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, job); + + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + if (batch.get(lease_key)) |raw| { + if (decodeGraphMetricBuildLease(raw)) |lease| { + if (lease.job_id == job.job_id) { + var updated_lease = lease; + updated_lease.phase = next_phase; + updated_lease.iteration = iteration; + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(updated_lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(updated_lease, encoded); + try batch.put(lease_key, encoded); + } + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + try batch.commit(); + return true; + } + + fn recordGraphMetricBuildIterationSummaryFromCheckPhase( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) !GraphMetricBuildIterationSummary { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.job_id != job_id) return error.GraphMetricBuildJobMismatch; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const phase_summary = try self.summarizeGraphMetricBuildPhaseInBatch(&batch, metric_name, cfg, job, .check_convergence, iteration); + if (phase_summary.state != .complete) return error.GraphMetricBuildIterationNotComplete; + const iteration_summary = GraphMetricBuildIterationSummary{ + .job_id = job.job_id, + .iteration = iteration, + .expected_pages = phase_summary.expected_pages, + .completed_pages = phase_summary.completed_pages, + .max_delta = phase_summary.max_delta, + .total_delta = phase_summary.total_delta, + .rank_sum = phase_summary.rank_sum, + .converged = phase_summary.converged, + .fixed_iteration_limit = !phase_summary.converged and @as(u64, iteration) + 1 >= cfg.max_iterations, + .output_fingerprint = phase_summary.output_fingerprint, + }; + try self.putGraphMetricBuildIterationSummaryInBatch(&batch, metric_name, iteration_summary); + try batch.commit(); + return iteration_summary; + } + + fn validateGraphMetricBuildExecution(manifest: GraphMetricBuildManifest, job: GraphMetricBuildJob, cfg: GraphMetricConfig) !void { + if (manifest.execution_schema_version != graph_metric_build_execution_schema_version or + manifest.job_id != job.job_id or + manifest.target_generation != job.target_generation or + manifest.score_generation != job.score_generation or + manifest.config_fingerprint != graphMetricConfigFingerprint(cfg)) + { + return error.InvalidGraphMetricBuildManifest; + } + } + + fn validateGraphMetricBuildExecutionInTxn(self: *GraphIndex, txn: anytype, metric_name: []const u8, job: GraphMetricBuildJob, cfg: GraphMetricConfig) !void { + // Retirement deletes the manifest by design. No computation or + // publication remains once the durable job enters cleanup. + if (job.phase == .complete or job.phase == .cleanup_old_generations) return; + const manifest = try self.metricBuildManifest(txn, metric_name, job.job_id) orelse return error.GraphMetricBuildManifestNotFound; + try validateGraphMetricBuildExecution(manifest, job, cfg); + } + + fn verifyGraphMetricBuildPublishReady( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) !GraphMetricBuildPublishVerification { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.job_id != job_id) return error.GraphMetricBuildJobMismatch; + if (job.target_generation != try self.graphMetricCurrentGenerationInTxn(&batch, metric_name)) return error.GraphMetricBuildSuperseded; + if (job.phase != .publish_generation) return error.GraphMetricBuildPublishNotReady; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const manifest = try self.metricBuildManifest(&batch, metric_name, job.job_id) orelse return error.GraphMetricBuildManifestNotFound; + try validateGraphMetricBuildExecution(manifest, job, cfg); + const expected_config_fingerprint = graphMetricConfigFingerprint(cfg); + const phases = graphMetricBuildManifestPhases(cfg.kind); + if (manifest.job_id != job.job_id or + manifest.target_generation != job.target_generation or + manifest.score_generation != job.score_generation or + manifest.config_fingerprint != expected_config_fingerprint or + manifest.phase_count != phases.len or + manifest.page_count < phases.len) + { + return error.InvalidGraphMetricBuildManifest; + } + + var expected_phases: u64 = 0; + var completed_phases: u64 = 0; + var expected_pages: u64 = 0; + var completed_pages: u64 = 0; + var output_fingerprint: u64 = 0; + var check_phase_summary: ?GraphMetricBuildPhaseSummary = null; + for (phases) |phase| { + if (phase == .publish_generation) break; + const phase_iteration = graphMetricBuildPhasePublishIteration(cfg.kind, phase, job.iteration); + const summary = try self.summarizeGraphMetricBuildPhaseInBatch(&batch, metric_name, cfg, job, phase, phase_iteration); + expected_phases += 1; + expected_pages += summary.expected_pages; + completed_pages += summary.completed_pages; + output_fingerprint ^= summary.output_fingerprint; + if (summary.state != .complete) return error.GraphMetricBuildPublishNotReady; + completed_phases += 1; + if (phase == .check_convergence) check_phase_summary = summary; + } + + var converged = false; + var fixed_iteration_limit = false; + var max_delta: f64 = 0.0; + var total_delta: f64 = 0.0; + var rank_sum: f64 = 0.0; + if (graphMetricKindIsIterative(cfg.kind)) { + const summary = check_phase_summary orelse return error.GraphMetricBuildPublishNotReady; + const iteration_summary = GraphMetricBuildIterationSummary{ + .job_id = job.job_id, + .iteration = job.iteration, + .expected_pages = summary.expected_pages, + .completed_pages = summary.completed_pages, + .max_delta = summary.max_delta, + .total_delta = summary.total_delta, + .rank_sum = summary.rank_sum, + .converged = summary.converged, + .fixed_iteration_limit = !summary.converged and @as(u64, job.iteration) + 1 >= cfg.max_iterations, + .output_fingerprint = summary.output_fingerprint, + }; + try self.putGraphMetricBuildIterationSummaryInBatch(&batch, metric_name, iteration_summary); + converged = iteration_summary.converged; + fixed_iteration_limit = iteration_summary.fixed_iteration_limit; + max_delta = iteration_summary.max_delta; + total_delta = iteration_summary.total_delta; + rank_sum = iteration_summary.rank_sum; + } + + try batch.commit(); + return .{ + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .config_fingerprint = expected_config_fingerprint, + .iteration = job.iteration, + .expected_phases = expected_phases, + .completed_phases = completed_phases, + .expected_pages = expected_pages, + .completed_pages = completed_pages, + .output_fingerprint = output_fingerprint, + .converged = converged, + .fixed_iteration_limit = fixed_iteration_limit, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + }; + } + + fn releaseGraphMetricBuildLease(self: *GraphIndex, metric_name: []const u8) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(key); + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + try batch.commit(); + } + + fn updateGraphMetricBuildLeaseProgress( + self: *GraphIndex, + metric_name: []const u8, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !void { + try self.updateGraphMetricBuildLeaseProgressWithCursor(metric_name, phase, iteration, "", 0, 0); + } + + fn updateGraphMetricBuildLeaseProgressWithCursor( + self: *GraphIndex, + metric_name: []const u8, + phase: GraphMetricBuildPhase, + iteration: u32, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(key); + const raw = batch.get(key) catch |err| switch (err) { + error.NotFound => { + batch.abort(); + return; + }, + else => return err, + }; + var lease = decodeGraphMetricBuildLease(raw) orelse { + batch.abort(); + return; + }; + lease.phase = phase; + lease.iteration = iteration; + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + const score_generation = if (try self.metricBuildJob(&batch, metric_name)) |job| + if (job.job_id == lease.job_id) job.score_generation else lease.target_generation + else + lease.target_generation; + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(lease, encoded); + try batch.put(key, encoded); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = lease.job_id, + .target_generation = lease.target_generation, + .score_generation = score_generation, + .started_at_ms = lease.started_at_ms, + .updated_at_ms = now_ms, + .lease_expires_at_ms = lease.lease_expires_at_ms, + .phase = phase, + .iteration = iteration, + .worker_id = lease.worker_id, + .cursor = cursor, + .completed_units = completed_units, + .total_units = total_units, + }); + try batch.commit(); + } + + fn completeGraphMetricBuildJob(self: *GraphIndex, metric_name: []const u8) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = try self.metricBuildJob(&batch, metric_name) orelse { + try batch.commit(); + return; + }; + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .started_at_ms = job.started_at_ms, + .updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .lease_expires_at_ms = 0, + .phase = .complete, + .iteration = job.iteration, + .worker_id = job.worker_id, + .cursor = job.cursor, + .completed_units = if (job.total_units != 0) job.total_units else job.completed_units, + .total_units = job.total_units, + }); + try batch.commit(); + } + + fn putGraphMetricBuildJobInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job: GraphMetricBuildJob, + ) !void { + const job_key = try self.graphMetricBuildJobKeyAlloc(metric_name); + defer self.alloc.free(job_key); + const encoded = try self.alloc.alloc(u8, graphMetricBuildJobEncodedLen(job)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildJob(job, encoded); + try batch.put(job_key, encoded); + } + + fn putGraphMetricBuildManifestInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + manifest: GraphMetricBuildManifest, + ) !void { + const manifest_key = try self.graphMetricBuildManifestKeyAlloc(metric_name, manifest.job_id); + defer self.alloc.free(manifest_key); + var encoded: [graph_metric_build_manifest_encoded_len]u8 = undefined; + encodeGraphMetricBuildManifest(manifest, &encoded); + try batch.put(manifest_key, &encoded); + } + + pub fn invalidateGraphMetricBuildManifestConfigFingerprintForTest( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var manifest = try self.metricBuildManifest(&batch, metric_name, job_id) orelse return error.GraphMetricBuildManifestNotFound; + manifest.config_fingerprint +%= 1; + try self.putGraphMetricBuildManifestInBatch(&batch, metric_name, manifest); + try batch.commit(); + } + + fn putGraphMetricBuildPageInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + page: GraphMetricBuildPage, + ) !void { + if (page.phase == .cleanup_old_generations) { + const page_key = try self.graphMetricBuildPageKeyAlloc(metric_name, page.job_id, page.phase, page.iteration, page.page_id); + defer self.alloc.free(page_key); + const encoded = try self.alloc.alloc(u8, graphMetricBuildPageEncodedLen(page)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildPage(page, encoded); + try batch.put(page_key, encoded); + return; + } + const previous = try self.metricBuildPage(batch, metric_name, page.job_id, page.phase, page.iteration, page.page_id); + var progress = try self.metricBuildPhaseProgress(batch, metric_name, page.job_id, page.phase, page.iteration) orelse blk: { + var rebuilt = GraphMetricBuildPhaseProgress{ + .job_id = page.job_id, + .phase = page.phase, + .iteration = page.iteration, + }; + if (previous != null) { + const prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, page.job_id, page.phase, page.iteration); + defer self.alloc.free(prefix); + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const existing = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + try updateGraphMetricBuildPhaseProgress(&rebuilt, existing, true); + } + } + break :blk rebuilt; + }; + if (progress.job_id != page.job_id or progress.phase != page.phase or progress.iteration != page.iteration) + return error.InvalidGraphMetricBuildPage; + if (previous) |old| try updateGraphMetricBuildPhaseProgress(&progress, old, false); + try updateGraphMetricBuildPhaseProgress(&progress, page, true); + + const page_key = try self.graphMetricBuildPageKeyAlloc(metric_name, page.job_id, page.phase, page.iteration, page.page_id); + defer self.alloc.free(page_key); + const encoded = try self.alloc.alloc(u8, graphMetricBuildPageEncodedLen(page)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildPage(page, encoded); + try batch.put(page_key, encoded); + try self.putGraphMetricBuildPhaseProgressInBatch(batch, metric_name, progress); + if (page.phase == .initialize_ranks and page.page_id >= graph_metric_build_summary_leaf_base and page.state == .complete) { + if (try self.topologyBinding(batch, metric_name, page.job_id)) |binding| if (!binding.adopted) { + try self.requireMutableTopology(batch, metric_name, page.job_id); + const key = try self.topologyKey(batch, metric_name, page.job_id, try self.alloc.dupe(u8, page_key)); + defer self.alloc.free(key); + try batch.put(key, encoded); + }; + } + } + + fn updateGraphMetricBuildPhaseProgress(progress: *GraphMetricBuildPhaseProgress, page: GraphMetricBuildPage, add: bool) !void { + const apply = struct { + fn value(target: *u64, amount: u64, should_add: bool) !void { + target.* = if (should_add) + std.math.add(u64, target.*, amount) catch return error.InvalidGraphMetricBuildPage + else + std.math.sub(u64, target.*, amount) catch return error.InvalidGraphMetricBuildPage; + } + }.value; + try apply(&progress.expected_pages, 1, add); + try apply(&progress.completed_units, page.completed_units, add); + try apply(&progress.total_units, page.total_units, add); + switch (page.state) { + .pending => try apply(&progress.pending_pages, 1, add), + .leased => try apply(&progress.leased_pages, 1, add), + .complete => try apply(&progress.completed_pages, 1, add), + .failed => try apply(&progress.failed_pages, 1, add), + } + if (page.attempt >= graph_metric_build_max_page_attempts and (page.state == .leased or page.state == .failed)) + try apply(&progress.max_attempt_pages, 1, add); + } + + fn putGraphMetricBuildPhaseProgressInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + progress: GraphMetricBuildPhaseProgress, + ) !void { + const key = try self.graphMetricBuildPhaseProgressKeyAlloc(metric_name, progress.job_id, progress.phase, progress.iteration); + defer self.alloc.free(key); + var encoded: [graph_metric_build_phase_progress_encoded_len]u8 = undefined; + encodeGraphMetricBuildPhaseProgress(progress, &encoded); + try batch.put(key, &encoded); + } + + fn putGraphMetricBuildPhaseSummaryInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + summary: GraphMetricBuildPhaseSummary, + ) !void { + const summary_key = try self.graphMetricBuildPhaseSummaryKeyAlloc(metric_name, summary.job_id, summary.phase, summary.iteration); + defer self.alloc.free(summary_key); + var encoded: [graph_metric_build_phase_summary_encoded_len]u8 = undefined; + encodeGraphMetricBuildPhaseSummary(summary, &encoded); + try batch.put(summary_key, &encoded); + } + + fn putGraphMetricBuildIterationSummaryInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + summary: GraphMetricBuildIterationSummary, + ) !void { + const summary_key = try self.graphMetricBuildIterationSummaryKeyAlloc(metric_name, summary.job_id, summary.iteration); + defer self.alloc.free(summary_key); + var encoded: [graph_metric_build_iteration_summary_encoded_len]u8 = undefined; + encodeGraphMetricBuildIterationSummary(summary, &encoded); + try batch.put(summary_key, &encoded); + } + + fn graphMetricLastEvent(self: *GraphIndex, txn: anytype, metric_name: []const u8) !?GraphMetricEvent { + const sequence_key = try self.graphMetricEventSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + const sequence = try readU64OrZero(txn, sequence_key); + if (sequence == 0) return null; + const event_key = try self.graphMetricEventKeyAlloc(metric_name, sequence); + defer self.alloc.free(event_key); + const raw = txn.get(event_key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeGraphMetricEvent(sequence, raw); + } + + fn graphMetricRecentEvents( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + limit: usize, + ) ![]GraphMetricEvent { + if (limit == 0) return &.{}; + const sequence_key = try self.graphMetricEventSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + var sequence = try readU64OrZero(txn, sequence_key); + if (sequence == 0) return &.{}; + + const events = try self.alloc.alloc(GraphMetricEvent, @min(limit, sequence)); + var count: usize = 0; + errdefer if (events.len > 0) self.alloc.free(events); + while (sequence > 0 and count < limit) : (sequence -= 1) { + const event_key = try self.graphMetricEventKeyAlloc(metric_name, sequence); + defer self.alloc.free(event_key); + const raw = txn.get(event_key) catch |err| switch (err) { + error.NotFound => continue, + else => return err, + }; + const event = decodeGraphMetricEvent(sequence, raw) orelse continue; + events[count] = event; + count += 1; + } + return try self.alloc.realloc(events, count); + } + + fn graphMetricRecentFailureRecords( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + limit: usize, + ) ![]GraphMetricFailureRecord { + if (limit == 0) return &.{}; + const sequence_key = try self.graphMetricFailureSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + var sequence = try readU64OrZero(txn, sequence_key); + if (sequence == 0) return &.{}; + + const records = try self.alloc.alloc(GraphMetricFailureRecord, @min(limit, sequence)); + var count: usize = 0; + errdefer { + for (records[0..count]) |*record| record.deinit(self.alloc); + if (records.len > 0) self.alloc.free(records); + } + while (sequence > 0 and count < limit) : (sequence -= 1) { + const record_key = try self.graphMetricFailureRecordKeyAlloc(metric_name, sequence); + defer self.alloc.free(record_key); + const raw = txn.get(record_key) catch |err| switch (err) { + error.NotFound => continue, + else => return err, + }; + const record = (try decodeGraphMetricFailureRecordAlloc(self.alloc, sequence, raw)) orelse continue; + records[count] = record; + count += 1; + } + return try self.alloc.realloc(records, count); + } + + const GraphMetricBuildPageStatusList = struct { + pages: []GraphMetricBuildPageStatus = &.{}, + truncated: bool = false, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.pages) |*page| page.deinit(alloc); + if (self.pages.len > 0) alloc.free(self.pages); + self.* = undefined; + } + }; + + const GraphMetricBuildProgressAggregate = struct { + cursor: []const u8 = "", + completed_units: u64 = 0, + total_units: u64 = 0, + + fn deinit(self: *@This(), alloc: Allocator) void { + if (self.cursor.len > 0) alloc.free(self.cursor); + self.* = undefined; + } + }; + + /// Derive operator-visible progress from every page in the active phase. + /// Workers own disjoint page records; keeping the shared job record out of + /// the checkpoint path removes a hot write key and prevents whichever page + /// committed last from making global progress move backwards. + fn graphMetricActiveBuildProgressAggregate( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job: GraphMetricBuildJob, + ) !GraphMetricBuildProgressAggregate { + if (job.phase == .idle or job.phase == .computing or job.phase == .publishing or job.phase == .complete or job.phase == .publish_generation) { + return .{ + .cursor = if (job.cursor.len > 0) try self.alloc.dupe(u8, job.cursor) else "", + .completed_units = job.completed_units, + .total_units = job.total_units, + }; + } + const iteration: u32 = if (job.phase == .cleanup_old_generations) 0 else job.iteration; + const prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, job.job_id, job.phase, iteration); + defer self.alloc.free(prefix); + + var completed_units: u64 = 0; + var total_units: u64 = 0; + var single_cursor: []const u8 = ""; + errdefer if (single_cursor.len > 0) self.alloc.free(single_cursor); + var cursor_count: usize = 0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const page = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + if (page.job_id != job.job_id or page.phase != job.phase or page.iteration != iteration) return error.InvalidGraphMetricBuildPage; + completed_units = std.math.add(u64, completed_units, @min(page.completed_units, page.total_units)) catch + return error.InvalidGraphMetricBuildProgress; + total_units = std.math.add(u64, total_units, page.total_units) catch + return error.InvalidGraphMetricBuildProgress; + if (page.state == .leased and page.cursor.len > 0) { + cursor_count += 1; + if (cursor_count == 1) { + // Cursor values are borrowed only until the next movement. + single_cursor = try self.alloc.dupe(u8, page.cursor); + } else if (single_cursor.len > 0) { + self.alloc.free(single_cursor); + single_cursor = ""; + } + } + } + return .{ + // A single cursor has useful semantics. With concurrent pages the + // per-page status list is authoritative and a global cursor would + // be misleading. + .cursor = single_cursor, + .completed_units = completed_units, + .total_units = total_units, + }; + } + + fn graphMetricActiveBuildPageStatuses( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job: GraphMetricBuildJob, + limit: usize, + ) !GraphMetricBuildPageStatusList { + if (job.phase == .idle or job.phase == .computing or job.phase == .publishing or job.phase == .complete or job.phase == .publish_generation) { + return .{}; + } + const iteration: u32 = if (job.phase == .cleanup_old_generations) 0 else job.iteration; + const prefix = try self.graphMetricBuildPagePrefixAlloc(metric_name, job.job_id, job.phase, iteration); + defer self.alloc.free(prefix); + + var pages = std.ArrayListUnmanaged(GraphMetricBuildPageStatus).empty; + errdefer { + for (pages.items) |*page| page.deinit(self.alloc); + pages.deinit(self.alloc); + } + var truncated = false; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const page = decodeGraphMetricBuildPage(entry.value) orelse return error.InvalidGraphMetricBuildPage; + if (page.job_id != job.job_id or page.phase != job.phase or page.iteration != iteration) return error.InvalidGraphMetricBuildPage; + if (page.state != .leased and page.state != .failed) continue; + if (pages.items.len >= limit) { + truncated = true; + break; + } + const worker_id = if (page.worker_id.len > 0) try self.alloc.dupe(u8, page.worker_id) else ""; + errdefer if (worker_id.len > 0) self.alloc.free(worker_id); + const cursor = if (page.cursor.len > 0) try self.alloc.dupe(u8, page.cursor) else ""; + errdefer if (cursor.len > 0) self.alloc.free(cursor); + const last_error = if (page.last_error.len > 0) try self.alloc.dupe(u8, page.last_error) else ""; + errdefer if (last_error.len > 0) self.alloc.free(last_error); + try pages.append(self.alloc, .{ + .phase = page.phase, + .iteration = page.iteration, + .page_id = page.page_id, + .state = page.state, + .range_kind = page.range_kind, + .worker_id = worker_id, + .lease_expires_at_ms = page.lease_expires_at_ms, + .attempt = page.attempt, + .cursor = cursor, + .completed_units = page.completed_units, + .total_units = page.total_units, + .last_error = last_error, + }); + } + + return .{ + .pages = try pages.toOwnedSlice(self.alloc), + .truncated = truncated, + }; + } + + fn appendGraphMetricFailureRecord( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + record: GraphMetricFailureRecord, + ) !void { + const sequence_key = try self.graphMetricFailureSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + const sequence = (try readU64OrZero(batch, sequence_key)) + 1; + try putU64(batch, sequence_key, sequence); + + const record_key = try self.graphMetricFailureRecordKeyAlloc(metric_name, sequence); + defer self.alloc.free(record_key); + const encoded = try self.alloc.alloc(u8, graph_metric_failure_record_header_len + record.last_error.len); + defer self.alloc.free(encoded); + var sequenced = record; + sequenced.sequence = sequence; + encodeGraphMetricFailureRecord(sequenced, encoded); + try batch.put(record_key, encoded); + if (sequence > graph_metric_recent_event_limit) { + const prune_key = try self.graphMetricFailureRecordKeyAlloc(metric_name, sequence - graph_metric_recent_event_limit); + defer self.alloc.free(prune_key); + batch.delete(prune_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + + fn appendGraphMetricEvent( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + event: GraphMetricEvent, + ) !void { + const sequence_key = try self.graphMetricEventSequenceKeyAlloc(metric_name); + defer self.alloc.free(sequence_key); + const sequence = (try readU64OrZero(batch, sequence_key)) + 1; + try putU64(batch, sequence_key, sequence); + + const event_key = try self.graphMetricEventKeyAlloc(metric_name, sequence); + defer self.alloc.free(event_key); + var encoded: [graph_metric_event_encoded_len]u8 = undefined; + var sequenced = event; + sequenced.sequence = sequence; + encodeGraphMetricEvent(sequenced, &encoded); + try batch.put(event_key, &encoded); + if (sequence > graph_metric_recent_event_limit) { + const prune_key = try self.graphMetricEventKeyAlloc(metric_name, sequence - graph_metric_recent_event_limit); + defer self.alloc.free(prune_key); + batch.delete(prune_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + + fn clearGraphMetricFailureInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8) !void { + const failure_key = try self.graphMetricFailureKeyAlloc(metric_name); + defer self.alloc.free(failure_key); + batch.delete(failure_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + + fn nextGraphMetricFailureRetryCountInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8) !u64 { + const failure_key = try self.graphMetricFailureKeyAlloc(metric_name); + defer self.alloc.free(failure_key); + if (batch.get(failure_key)) |raw| { + if (try decodeGraphMetricFailureDetailAlloc(self.alloc, raw)) |detail| { + defer detail.deinit(self.alloc); + return detail.retry_count + 1; + } + } else |get_err| switch (get_err) { + error.NotFound => {}, + else => return get_err, + } + return 1; + } + + fn putGraphMetricFailureDetailInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, retry_count: u64, err_name: []const u8) !void { + const failure_key = try self.graphMetricFailureKeyAlloc(metric_name); + defer self.alloc.free(failure_key); + const encoded = try self.alloc.alloc(u8, graph_metric_failure_detail_header_len + err_name.len); + defer self.alloc.free(encoded); + encodeGraphMetricFailureDetail(.{ .retry_count = retry_count, .last_error = err_name }, encoded); + try batch.put(failure_key, encoded); + } + + fn recordGraphMetricFailure(self: *GraphIndex, metric_name: []const u8, err: anyerror) !void { + return try self.recordGraphMetricFailureReason(metric_name, @errorName(err)); + } + + fn recordGraphMetricFailureReason(self: *GraphIndex, metric_name: []const u8, failure_reason: []const u8) !void { + return self.recordGraphMetricFailureReasonAtGeneration(metric_name, failure_reason, null, null); + } + + fn recordGraphMetricFailureReasonAtGeneration(self: *GraphIndex, requested_name: []const u8, failure_reason: []const u8, preparation_generation: ?u64, preparation_task: ?[]const u8) !void { + const metric_name = if (preparation_task != null) try self.graphMetricLifecycleOwnerName(requested_name) else requested_name; + const pair_cfg = if (self.metricConfig(metric_name)) |cfg| self.pairedHitsMetricConfig(cfg) else null; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + if (preparation_task) |task_name| { + const task_key = try topologyTaskKeyAlloc(self.alloc, task_name); + defer self.alloc.free(task_key); + const raw = batch.get(task_key) catch |err| switch (err) { + error.NotFound => { + batch.abort(); + return; + }, + else => return err, + }; + if (try topologyTaskIncarnation(raw) != try topologyTaskNameIncarnation(task_name) or raw[8] & 2 != 0) { + batch.abort(); + return; + } + // Deliver a failure once per dependent and incarnation. A manual + // retry accepted after delivery must survive delayed reporters. + const delivered = try self.graphMetricControlKeyAlloc(&.{ task_name, "failed-dependent", metric_name }); + defer self.alloc.free(delivered); + if (batch.get(delivered)) |_| { + batch.abort(); + return; + } else |err| if (err != error.NotFound) return err; + try batch.put(delivered, ""); + } + if (preparation_generation) |generation| { + if (generation != try self.graphMetricCurrentGenerationInTxn(&batch, metric_name) or try self.metricDisabled(&batch, metric_name) or try self.metricMaintenancePaused(&batch, metric_name)) { + batch.abort(); + return; + } + } + const retry_count = try self.nextGraphMetricFailureRetryCountInBatch(&batch, metric_name); + try self.putGraphMetricFailureDetailInBatch(&batch, metric_name, retry_count, failure_reason); + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + var failure_record = GraphMetricFailureRecord{ + .at_ms = now_ms, + .retry_count = retry_count, + .last_error = failure_reason, + .target_generation = try self.graphMetricCurrentGenerationInTxn(&batch, metric_name), + }; + const requested = try self.graphMetricControlKeyAlloc(&.{ metric_name, "requested" }); + defer self.alloc.free(requested); + batch.delete(requested) catch |err| if (err != error.NotFound) return err; + const failed_job = job: { + const candidate = try self.metricBuildJob(&batch, metric_name) orelse break :job null; + if (preparation_generation) |generation| { + // A queued dependency failure is not a failure of the old, + // already-published numerical job or its score namespace. + if (candidate.phase == .complete or candidate.target_generation != generation) break :job null; + } + break :job candidate; + }; + if (failed_job) |job| { + failure_record.job_id = job.job_id; + self.sealed_vectors.retire(self.alloc, sealedVectorScope(metric_name)); + failure_record.target_generation = job.target_generation; + failure_record.score_generation = job.score_generation; + failure_record.phase = job.phase; + failure_record.iteration = job.iteration; + const published_generation = try self.metricPublishedGeneration(&batch, metric_name); + if (job.score_generation != 0 and + job.score_generation != published_generation and + try self.scoreGenerationHasKeysInBatch(&batch, metric_name, job.score_generation)) + { + try self.enqueueRetiredScoreGenerationInBatch(&batch, metric_name, job.score_generation); + } + // Failure recording must remain bounded even when a build + // accumulated millions of intermediate keys. Reclaim one small + // page in the failure transaction (which preserves the historical + // immediate cleanup behavior for small jobs), then persist a + // durable cursor when more work remains. + const failed_cleanup_key = try self.graphMetricFailedJobCleanupKeyAlloc(metric_name, job.job_id); + defer self.alloc.free(failed_cleanup_key); + const failed_job_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(failed_job_prefix); + var failed_cleanup = try self.deleteKeysWithPrefixPageInBatch(&batch, failed_job_prefix, "", graph_metric_build_cleanup_delete_page_units); + defer failed_cleanup.deinit(self.alloc); + if (failed_cleanup.reached_end) { + batch.delete(failed_cleanup_key) catch |cleanup_err| switch (cleanup_err) { + error.NotFound => {}, + else => return cleanup_err, + }; + } else { + try batch.put(failed_cleanup_key, failed_cleanup.cursor); + } + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + batch.delete(lease_key) catch |delete_err| switch (delete_err) { + error.NotFound => {}, + else => return delete_err, + }; + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .started_at_ms = job.started_at_ms, + .updated_at_ms = now_ms, + .lease_expires_at_ms = job.lease_expires_at_ms, + .phase = job.phase, + .iteration = job.iteration, + .retry_count = retry_count, + .worker_id = job.worker_id, + .last_error = failure_reason, + .cursor = job.cursor, + .completed_units = job.completed_units, + .total_units = job.total_units, + }); + } + // Failure identity belongs to the immutable build attempt, not to the + // graph's live generation at the instant the coordinator records it. + // A concurrent graph mutation may already have advanced + // `self.edge_generation`; stamping that newer value here would turn a + // normally superseded build into a terminal failure for the new work + // and prevent background maintenance from requeuing it. + const failure_target_generation = preparation_generation orelse if (failure_record.job_id != 0) + failure_record.target_generation + else + try self.graphMetricCurrentGenerationInTxn(&batch, metric_name); + try self.appendGraphMetricFailureRecord(&batch, metric_name, failure_record); + try self.appendGraphMetricEvent(&batch, metric_name, .{ + .kind = .failed, + .at_ms = now_ms, + .target_edge_generation = failure_target_generation, + .published_generation = try self.metricPublishedEdgeGeneration(&batch, metric_name), + .score_count = 0, + }); + if (pair_cfg) |pair| { + const pair_retry_count = try self.nextGraphMetricFailureRetryCountInBatch(&batch, pair.name); + try self.putGraphMetricFailureDetailInBatch(&batch, pair.name, pair_retry_count, failure_reason); + var pair_failure_record = failure_record; + pair_failure_record.retry_count = pair_retry_count; + if (pair_failure_record.score_generation != 0) { + const pair_published_generation = try self.metricPublishedGeneration(&batch, pair.name); + if (pair_failure_record.score_generation != pair_published_generation and + try self.scoreGenerationHasKeysInBatch(&batch, pair.name, pair_failure_record.score_generation)) + { + try self.enqueueRetiredScoreGenerationInBatch(&batch, pair.name, pair_failure_record.score_generation); + } + } + try self.appendGraphMetricFailureRecord(&batch, pair.name, pair_failure_record); + try self.appendGraphMetricEvent(&batch, pair.name, .{ + .kind = .failed, + .at_ms = now_ms, + .target_edge_generation = failure_target_generation, + .published_generation = try self.metricPublishedEdgeGeneration(&batch, pair.name), + .score_count = 0, + }); + } + try batch.commit(); + } + + fn markMetricDirty(self: *GraphIndex, batch: anytype, changed_types: *const std.StringHashMapUnmanaged(void)) !void { + if (self.metric_configs.len == 0) return; + for (self.metric_configs) |cfg| { + if (cfg.edge_filter.mode != .all) { + var affected = false; + for (cfg.edge_filter.types) |kind| if (changed_types.contains(kind)) { + affected = true; + break; + }; + if (!affected) continue; + } + const key = try self.graphMetricDirtyGenerationKeyAlloc(cfg.name); + defer self.alloc.free(key); + try putU64(batch, key, self.edge_generation); + } + } + + fn rememberNodeRefCount(self: *GraphIndex, counts: *std.StringHashMapUnmanaged(u64), node: []const u8) !void { + const result = try counts.getOrPut(self.alloc, node); + if (result.found_existing) { + result.value_ptr.* += 1; + return; + } + errdefer _ = counts.remove(node); + result.key_ptr.* = try self.alloc.dupe(u8, node); + result.value_ptr.* = 1; + } + + fn rebuildCounterMetadata(self: *GraphIndex) !void { + const prev_edge_count = self.edge_count; + const prev_node_count = self.node_count; + errdefer { + self.edge_count = prev_edge_count; + self.node_count = prev_node_count; + } + + var read_txn = try self.beginReadReverseTxn(); + defer read_txn.abort(); + + var meta_keys = std.ArrayListUnmanaged([]u8).empty; + defer { + for (meta_keys.items) |key| self.alloc.free(key); + meta_keys.deinit(self.alloc); + } + var node_refs = std.StringHashMapUnmanaged(u64).empty; + defer { + var key_it = node_refs.keyIterator(); + while (key_it.next()) |key| self.alloc.free(key.*); + node_refs.deinit(self.alloc); + } + + var edge_count: u64 = 0; + var cur = try read_txn.openCursor(); + defer cur.close(); + var maybe_entry = try cur.first(); + while (maybe_entry) |entry| { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) { + if (!std.mem.startsWith(u8, entry.key, graph_metric_key_prefix)) { + try self.appendOwnedBytes(&meta_keys, entry.key); + } + } else { + edge_count += 1; + if (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) |parsed_owned| { + var parsed = parsed_owned; + defer parsed.deinit(self.alloc); + try self.rememberNodeRefCount(&node_refs, parsed.source.bytes); + try self.rememberNodeRefCount(&node_refs, parsed.target.bytes); + } + } + maybe_entry = try cur.next(); + } + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + for (meta_keys.items) |key| { + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + + var node_count: u64 = 0; + var refs_it = node_refs.iterator(); + while (refs_it.next()) |entry| { + const ref_key = try graphNodeRefKeyAlloc(self.alloc, entry.key_ptr.*); + defer self.alloc.free(ref_key); + try putU64(&batch, ref_key, entry.value_ptr.*); + node_count += 1; + } + + self.edge_count = edge_count; + self.node_count = node_count; + try self.persistGraphCounters(&batch); + try batch.commit(); + } + + fn openEdgeStore(alloc: Allocator, path: [*:0]const u8, opts: GraphIndexOptions) !OpenedReverseStore { + switch (opts.reverse_backend) { + .lmdb => { + if (!supports_native_reverse_lmdb) return error.UnsupportedPlatform; + const backend = try alloc.create(lmdb_backend.Backend); + errdefer alloc.destroy(backend); + backend.* = try lmdb_backend.Backend.open(alloc, path, .{ + .backend = .{ + .durability = if (opts.no_sync) .none else .full, + }, + .env = .{ + .map_size = opts.map_size, + .no_sync = opts.no_sync, + .no_meta_sync = opts.no_meta_sync, + .no_tls = true, + .max_dbs = 1, + }, + }); + errdefer backend.close(); + + var runtime = try backend.runtimeStore(alloc, .{}); + errdefer runtime.deinit(); + return .{ + .store = runtime, + .owner = .{ .lmdb = backend }, + }; + }, + .mem => { + const backend = try alloc.create(mem_backend.Backend); + errdefer alloc.destroy(backend); + backend.* = mem_backend.Backend.init(alloc, .{}); + errdefer backend.close(); + + var runtime = try backend.runtimeStore(alloc, .{}); + runtime.write_gate = &backend.serialized_write_mutex; + errdefer runtime.deinit(); + return .{ + .store = runtime, + .owner = .{ .mem = backend }, + }; + }, + .lsm_memory => { + var handle = try lsm_backend.BackendHandle.init(alloc, resolvedReverseLsmOptions(opts, true)); + errdefer handle.close(); + + var runtime = try handle.backend.runtimeStore(alloc, .{}); + runtime.write_gate = &handle.backend.serialized_write_mutex; + errdefer runtime.deinit(); + return .{ + .store = runtime, + .owner = .{ .lsm = handle }, + }; + }, + .lsm => { + var handle = try lsm_backend.BackendHandle.open(alloc, std.mem.span(path), resolvedReverseLsmOptions(opts, false)); + errdefer handle.close(); + + var runtime = try handle.backend.runtimeStore(alloc, .{}); + runtime.write_gate = &handle.backend.serialized_write_mutex; + errdefer runtime.deinit(); + return .{ + .store = runtime, + .owner = .{ .lsm = handle }, + }; + }, + } + } + + fn openReverseStore(alloc: Allocator, reverse_path: [*:0]const u8, opts: GraphIndexOptions) !OpenedReverseStore { + return try openEdgeStore(alloc, reverse_path, opts); + } + + /// Test/backward-compatible opener. The supplied store is ignored: graph + /// edges live in private forward/reverse stores rooted under reverse_path. + pub fn open(alloc: Allocator, main_store: anytype, reverse_path: [*:0]const u8, index_name: []const u8, opts: GraphIndexOptions) !GraphIndex { + _ = main_store; + const root = std.mem.span(reverse_path); + const outgoing_raw = try std.fmt.allocPrint(alloc, "{s}/forward", .{root}); + defer alloc.free(outgoing_raw); + const outgoing_path = try alloc.dupeZ(u8, outgoing_raw); + defer alloc.free(outgoing_path); + const reverse_raw = try std.fmt.allocPrint(alloc, "{s}/reverse", .{root}); + defer alloc.free(reverse_raw); + const private_reverse_path = try alloc.dupeZ(u8, reverse_raw); + defer alloc.free(private_reverse_path); + return try openWithPrivateStores(alloc, outgoing_path, private_reverse_path, index_name, opts); + } + + pub fn openWithPrivateStores(alloc: Allocator, outgoing_path: [*:0]const u8, reverse_path: [*:0]const u8, index_name: []const u8, opts: GraphIndexOptions) !GraphIndex { + try validateGraphMetricEdgeFilters(opts.edge_type_configs, opts.metric_configs); + + var outgoing_store = try openEdgeStore(alloc, outgoing_path, opts); + errdefer { + outgoing_store.store.deinit(); + outgoing_store.owner.close(alloc); + } + var reverse_store = try openReverseStore(alloc, reverse_path, opts); + errdefer { + reverse_store.store.deinit(); + reverse_store.owner.close(alloc); + } + try outgoing_store.owner.ensureDurableEmptyManifest(); + try reverse_store.owner.ensureDurableEmptyManifest(); + const loaded_stats = try loadGraphCounters(&reverse_store.store); + + return .{ + .alloc = alloc, + .index_name = index_name, + .outgoing_store = outgoing_store.store, + .outgoing_owner = outgoing_store.owner, + .reverse_store = reverse_store.store, + .reverse_owner = reverse_store.owner, + .edge_type_configs = opts.edge_type_configs, + .metric_configs = opts.metric_configs, + .sealed_vectors = if (opts.sealed_vector_budget) |budget| .{ .budget = budget } else .{}, + .rebuild_root_path = if (opts.rebuild_root_path) |path| try alloc.dupe(u8, path) else null, + .rebuild_storage = opts.reverse_lsm_storage, + .rebuild_owner_generation = opts.rebuild_owner_generation, + .algebraic_semiring_traversal = opts.algebraic_semiring_traversal, + .edge_count = loaded_stats.edge_count, + .node_count = loaded_stats.node_count, + .edge_generation = loaded_stats.edge_generation, + .algebraic_traversal_attempt_count = 0, + .algebraic_traversal_proven_count = 0, + .algebraic_traversal_rejected_count = 0, + .algebraic_traversal_fallback_count = 0, + .algebraic_traversal_result_node_count = 0, + }; + } + + pub fn close(self: *GraphIndex) void { + self.sealed_vectors.deinit(self.alloc); + self.outgoing_store.deinit(); + self.outgoing_owner.close(self.alloc); + self.reverse_store.deinit(); + self.reverse_owner.close(self.alloc); + if (self.rebuild_root_path) |path| self.alloc.free(path); + self.* = undefined; + } + + pub fn abandonAfterCrash(self: *GraphIndex) void { + self.sealed_vectors.deinit(self.alloc); + self.outgoing_store.deinit(); + self.outgoing_owner.abandonAfterCrash(self.alloc); + self.reverse_store.deinit(); + self.reverse_owner.abandonAfterCrash(self.alloc); + if (self.rebuild_root_path) |path| self.alloc.free(path); + self.* = undefined; + } + + pub fn sync(self: *GraphIndex, force: bool) !void { + try self.outgoing_owner.sync(force); + try self.reverse_owner.sync(force); + } + + pub fn syncReplayState(self: *GraphIndex) !void { + try self.outgoing_owner.sync(false); + try self.reverse_owner.sync(false); + } + + pub fn supportsAlgebraicSemiringTraversal(self: *const GraphIndex) bool { + return self.algebraic_semiring_traversal; + } + + pub const AlgebraicTraversalRuntimeStats = struct { + attempt_count: u64 = 0, + proven_count: u64 = 0, + rejected_count: u64 = 0, + fallback_count: u64 = 0, + result_node_count: u64 = 0, + }; + + pub fn noteAlgebraicTraversalAttempt(self: *GraphIndex) void { + self.algebraic_traversal_attempt_count += 1; + } + + pub fn noteAlgebraicTraversalProven(self: *GraphIndex, result_node_count: usize) void { + self.algebraic_traversal_proven_count += 1; + self.algebraic_traversal_result_node_count += @intCast(result_node_count); + } + + pub fn noteAlgebraicTraversalRejected(self: *GraphIndex) void { + self.algebraic_traversal_rejected_count += 1; + } + + pub fn noteAlgebraicTraversalFallback(self: *GraphIndex) void { + self.algebraic_traversal_fallback_count += 1; + } + + pub fn algebraicTraversalRuntimeStats(self: *const GraphIndex) AlgebraicTraversalRuntimeStats { + return .{ + .attempt_count = self.algebraic_traversal_attempt_count, + .proven_count = self.algebraic_traversal_proven_count, + .rejected_count = self.algebraic_traversal_rejected_count, + .fallback_count = self.algebraic_traversal_fallback_count, + .result_node_count = self.algebraic_traversal_result_node_count, + }; + } + + pub const Stats = struct { + edge_count: u64 = 0, + node_count: u64 = 0, + edge_generation: u64 = 0, + }; + + pub const GraphMetricStorageFootprint = struct { + score_records: usize = 0, + metric_records: usize = 0, + control_records: usize = 0, + job_namespace_records: usize = 0, + attempt_records: usize = 0, + failure_records: usize = 0, + event_records: usize = 0, + }; + + pub fn stats(self: *GraphIndex, alloc: Allocator) !Stats { + _ = alloc; + if (self.edge_count == 0 and self.node_count == 0) { + const persisted = try loadGraphCounters(&self.reverse_store); + if (persisted.edge_count != 0 or persisted.node_count != 0) { + self.edge_count = persisted.edge_count; + self.node_count = persisted.node_count; + return persisted; + } + } + return .{ + .edge_count = self.edge_count, + .node_count = self.node_count, + }; + } + + pub fn scanStats(self: *GraphIndex, alloc: Allocator) !Stats { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + + var cur = try txn.openCursor(); + defer cur.close(); + + var seen_nodes = std.StringHashMapUnmanaged(void).empty; + defer { + var it = seen_nodes.keyIterator(); + while (it.next()) |key| alloc.free(key.*); + seen_nodes.deinit(alloc); + } + + var first = (try cur.first()) orelse return .{}; + while (std.mem.startsWith(u8, first.key, graph_meta_prefix)) { + first = (try cur.next()) orelse return .{}; + } + var edge_count: u64 = 0; + try rememberStatsNode(alloc, &seen_nodes, first.key); + edge_count += 1; + + while (try cur.next()) |entry| { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + try rememberStatsNode(alloc, &seen_nodes, entry.key); + edge_count += 1; + } + return .{ + .edge_count = edge_count, + .node_count = seen_nodes.count(), + .edge_generation = self.edge_generation, + }; + } + + pub fn graphMetricStorageFootprint(self: *GraphIndex, metric_name: []const u8) !GraphMetricStorageFootprint { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + + const score_prefix = try self.graphMetricKeyAlloc(&.{ metric_name, "score" }); + defer self.alloc.free(score_prefix); + const metric_prefix = try self.graphMetricKeyAlloc(&.{metric_name}); + defer self.alloc.free(metric_prefix); + const control_prefix = try self.graphMetricControlKeyAlloc(&.{metric_name}); + defer self.alloc.free(control_prefix); + const job_prefix = try self.graphMetricControlKeyAlloc(&.{ metric_name, "job" }); + defer self.alloc.free(job_prefix); + const failure_prefix = try self.graphMetricControlKeyAlloc(&.{ metric_name, "failure" }); + defer self.alloc.free(failure_prefix); + const event_prefix = try self.graphMetricKeyAlloc(&.{ metric_name, "event" }); + defer self.alloc.free(event_prefix); + + return .{ + .score_records = try countKeysWithPrefix(&txn, score_prefix), + .metric_records = try countKeysWithPrefix(&txn, metric_prefix), + .control_records = try countKeysWithPrefix(&txn, control_prefix), + .job_namespace_records = try countKeysWithPrefix(&txn, job_prefix), + .attempt_records = try countGraphMetricAttemptRecordsWithJobPrefix(&txn, job_prefix), + .failure_records = try countKeysWithPrefix(&txn, failure_prefix), + .event_records = try countKeysWithPrefix(&txn, event_prefix), + }; + } + + fn countKeysWithPrefix(txn: anytype, prefix: []const u8) !usize { + var cur = try txn.openCursor(); + defer cur.close(); + var count: usize = 0; + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + count += 1; + } + return count; + } + + fn countGraphMetricAttemptRecordsWithJobPrefix(txn: anytype, job_prefix: []const u8) !usize { + var cur = try txn.openCursor(); + defer cur.close(); + var count: usize = 0; + var entry_opt = try cur.seekAtOrAfter(job_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, job_prefix)) break; + const job_id_start = job_prefix.len; + const job_id_term = internal_keys.findComponentTerminator(entry.key, job_id_start) orelse continue; + const attempt_start = job_id_term + 2; + if (attempt_start >= entry.key.len) continue; + if (internal_keys.componentEquals(entry.key, attempt_start, "attempt")) count += 1; + } + return count; + } + + fn rememberStatsNode( + alloc: Allocator, + seen_nodes: *std.StringHashMapUnmanaged(void), + key: []const u8, + ) !void { + var parsed = (try parseReverseEdgeKeyAlloc(alloc, key)) orelse return; + defer parsed.deinit(alloc); + try rememberStatsNodeValue(alloc, seen_nodes, parsed.source); + try rememberStatsNodeValue(alloc, seen_nodes, parsed.target); + } + + fn rememberStatsNodeValue( + alloc: Allocator, + seen_nodes: *std.StringHashMapUnmanaged(void), + key: []const u8, + ) !void { + const result = try seen_nodes.getOrPut(alloc, key); + if (result.found_existing) return; + errdefer _ = seen_nodes.remove(key); + result.key_ptr.* = try alloc.dupe(u8, key); + } + + fn getTopologyMode(self: *const GraphIndex, edge_type: []const u8) TopologyMode { + for (self.edge_type_configs) |cfg| { + if (std.mem.eql(u8, cfg.name, edge_type)) return cfg.topology; + } + return .graph; + } + + /// Add an edge (writes outgoing and reverse edge rows to private graph stores). + /// Returns TreeTopologyViolation if the edge type has tree topology and + /// the source already has an outgoing edge of that type to a different target. + pub fn addEdge( + self: *GraphIndex, + source: []const u8, + target: []const u8, + edge_type: []const u8, + weight: f64, + created_at: u64, + updated_at: u64, + metadata: []const u8, + ) !void { + // Tree topology: source can have at most one outgoing edge of this type + if (self.getTopologyMode(edge_type) == .tree) { + const existing = try self.getEdges(self.alloc, source, edge_type, .out); + defer freeEdges(self.alloc, existing); + for (existing) |e| { + if (!std.mem.eql(u8, e.target, target)) { + return TreeTopologyViolation.TreeTopologyViolation; + } + } + } + + return try self.batchApply(&.{.{ + .source = source, + .target = target, + .edge_type = edge_type, + .weight = weight, + .created_at = created_at, + .updated_at = updated_at, + .metadata_json = metadata, + }}, &.{}); + } + + pub fn batchApply(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete) !void { + return self.batchApplyWithAccounting(writes, deletes, true); + } + + /// Benchmark oracle keeps identical durable writes and topology accounting; + /// Reference uses scalar presence probes and per-edge global incidence + /// maintenance. Both paths commit normally. + pub fn benchmarkBatchApply(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete, reference: bool) !void { + if (reference) return self.batchApplyWithAccounting(writes, deletes, false); + return self.batchApplyWithAccounting(writes, deletes, true); + } + + fn batchApplyWithAccounting(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete, comptime coalesced: bool) !void { + if (writes.len == 0 and deletes.len == 0) return; + + // Validate the complete batch before opening either physical write + // batch, so invalid durable fields cannot partially mutate one + // direction or create records that the public graph wire contract + // cannot represent. + for (writes) |write| { + try edge_type_mod.validateStored(write.edge_type); + try edge_weight.validateStored(write.weight); + } + for (deletes) |delete| try edge_type_mod.validateStored(delete.edge_type); + try self.validateTreeBatchWrites(writes, deletes); + + var main_batch = try self.beginWriteOutgoingBatch(); + errdefer main_batch.abort(); + + var reverse_batch = try self.beginWriteReverseBatch(); + errdefer reverse_batch.abort(); + var topology_changes = std.StringHashMapUnmanaged(TopologyMutation).empty; + defer { + var keys = topology_changes.keyIterator(); + while (keys.next()) |key| self.alloc.free(key.*); + topology_changes.deinit(self.alloc); + } + const typed_demand = for (self.metric_configs) |cfg| { + if (cfg.edge_filter.mode == .types) break true; + } else false; + const typed_state = try typed_edges.maintenanceState(&reverse_batch, typed_demand); + // Compare original and final identity sets, not intermediate delete / + // insert operations. Replacing attributes or replaying an identical + // batch must not retire immutable topology or restart numerical jobs. + for (deletes) |item| try self.rememberTopologyMutation(&topology_changes, item.source, item.target, item.edge_type, false); + for (writes) |item| try self.rememberTopologyMutation(&topology_changes, item.source, item.target, item.edge_type, true); + try self.resolveTopologyMutationPresence(&reverse_batch, &topology_changes, coalesced); + var changed_types = std.StringHashMapUnmanaged(void).empty; + defer changed_types.deinit(self.alloc); + var changes = topology_changes.iterator(); + while (changes.next()) |entry| { + if (entry.value_ptr.before != entry.value_ptr.after) try changed_types.put(self.alloc, entry.value_ptr.kind, {}); + } + const prev_edge_count = self.edge_count; + const prev_node_count = self.node_count; + const prev_edge_generation = self.edge_generation; + errdefer { + self.edge_count = prev_edge_count; + self.node_count = prev_node_count; + self.edge_generation = prev_edge_generation; + } + + for (deletes) |delete| { + const out_key = try edgeKeyAlloc(self.alloc, delete.source, self.index_name, delete.edge_type, delete.target); + defer self.alloc.free(out_key); + main_batch.delete(out_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + + const rev_key = try reverseEdgeKeyAlloc(self.alloc, delete.target, self.index_name, delete.edge_type, delete.source); + defer self.alloc.free(rev_key); + if (!coalesced) try self.accountReverseDelete(&reverse_batch, delete.source, delete.target, rev_key); + reverse_batch.delete(rev_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + + for (writes) |write| { + const edge_val = try encodeEdgeValueAlloc( + self.alloc, + write.weight, + write.created_at, + write.updated_at, + write.metadata_json, + ); + defer self.alloc.free(edge_val); + + const out_key = try edgeKeyAlloc(self.alloc, write.source, self.index_name, write.edge_type, write.target); + defer self.alloc.free(out_key); + try main_batch.put(out_key, edge_val); + + const rev_key = try reverseEdgeKeyAlloc(self.alloc, write.target, self.index_name, write.edge_type, write.source); + defer self.alloc.free(rev_key); + if (!coalesced) try self.accountReverseInsert(&reverse_batch, write.source, write.target, rev_key); + try reverse_batch.put(rev_key, edge_val); + } + + if (coalesced) try self.accountTopologyMutations(&reverse_batch, &topology_changes); + var typed_updates = typed_edges.Updates.init(self.alloc); + defer typed_updates.deinit(); + changes = topology_changes.iterator(); + while (changes.next()) |entry| { + if (!typed_state.active) break; + if (entry.value_ptr.before == entry.value_ptr.after) continue; + const mutation = entry.value_ptr.*; + try typed_updates.stageKnown(&reverse_batch, mutation.kind, entry.key_ptr.*, mutation.source, mutation.target, mutation.after, if (typed_state.ready) mutation.before else null); + } + try typed_updates.flush(&reverse_batch); + if (typed_state.active and prev_edge_count == 0) try reverse_batch.put(typed_edges.ready_key, "1"); + if (changed_types.count() > 0) { + self.edge_generation = std.math.add(u64, self.edge_generation, 1) catch return error.InvalidGraphMetricBuildManifest; + // A migration floor gives previously indexed types a conservative + // dependency epoch without an unbounded writer-side backfill. + if (reverse_batch.get(graph_metric_type_epoch_floor_key)) |_| {} else |err| switch (err) { + error.NotFound => try putU64(&reverse_batch, graph_metric_type_epoch_floor_key, prev_edge_generation), + else => return err, + } + var kinds = changed_types.keyIterator(); + while (kinds.next()) |kind| { + const key = try self.graphMetricTypeEpochKeyAlloc(kind.*); + defer self.alloc.free(key); + try putU64(&reverse_batch, key, self.edge_generation); + } + try self.markMetricDirty(&reverse_batch, &changed_types); + } + try self.persistGraphCounters(&reverse_batch); + try main_batch.commit(); + try reverse_batch.commit(); + } + + const TopologyMutation = struct { before: bool, after: bool, kind: []const u8, source: []const u8, target: []const u8 }; + + /// Count original-to-final connectivity once, independent of duplicate + /// writes, attribute replacement and intermediate delete/reinsert pairs. + /// Only distinct changed endpoints incur a counter read and mutation. + fn accountTopologyMutations(self: *GraphIndex, batch: anytype, mutations: *const std.StringHashMapUnmanaged(TopologyMutation)) !void { + var deltas = std.StringHashMapUnmanaged(i64).empty; + defer deltas.deinit(self.alloc); + var edge_delta: i128 = 0; + var it = mutations.valueIterator(); + while (it.next()) |mutation| { + if (mutation.before == mutation.after) continue; + const delta: i64 = if (mutation.after) 1 else -1; + edge_delta += delta; + for ([_][]const u8{ mutation.source, mutation.target }) |node| { + const entry = try deltas.getOrPut(self.alloc, node); + if (!entry.found_existing) entry.value_ptr.* = 0; + entry.value_ptr.* = try std.math.add(i64, entry.value_ptr.*, delta); + } + } + self.edge_count = std.math.cast(u64, @as(i128, self.edge_count) + edge_delta) orelse return error.InvalidGraphMetricBuildManifest; + const Update = struct { key: []u8, delta: i64 }; + const updates = try self.alloc.alloc(Update, deltas.count()); + defer self.alloc.free(updates); + var count: usize = 0; + defer for (updates[0..count]) |update| self.alloc.free(update.key); + var entries = deltas.iterator(); + while (entries.next()) |entry| { + if (entry.value_ptr.* == 0) continue; + updates[count] = .{ .key = try graphNodeRefKeyAlloc(self.alloc, entry.key_ptr.*), .delta = entry.value_ptr.* }; + count += 1; + } + std.mem.sort(Update, updates[0..count], {}, struct { + fn less(_: void, a: Update, b: Update) bool { + return std.mem.order(u8, a.key, b.key) == .lt; + } + }.less); + var values: [256]?[]const u8 = undefined; + var keys: [256][]const u8 = undefined; + var next_counts: [256]u64 = undefined; + var offset: usize = 0; + while (offset < count) { + const page = updates[offset..@min(count, offset + values.len)]; + for (page, keys[0..page.len]) |update, *key| key.* = update.key; + try batch.getManySorted(keys[0..page.len], values[0..page.len]); + // Decode borrowed batch values before the first mutation. + for (page, values[0..page.len], 0..) |update, value, i| { + const current = if (value) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else 0; + const next = std.math.cast(u64, @as(i128, current) + update.delta) orelse return error.InvalidGraphMetricBuildManifest; + next_counts[i] = next; + if (current == 0 and next != 0) self.node_count += 1; + if (current != 0 and next == 0) self.node_count = std.math.sub(u64, self.node_count, 1) catch return error.InvalidGraphMetricBuildManifest; + } + for (page, next_counts[0..page.len]) |update, next| { + if (next == 0) try batch.delete(update.key) else try putU64(batch, update.key, next); + } + offset += page.len; + } + } + + fn rememberTopologyMutation(self: *GraphIndex, mutations: *std.StringHashMapUnmanaged(TopologyMutation), source: []const u8, target: []const u8, kind: []const u8, after: bool) !void { + const key = try reverseEdgeKeyAlloc(self.alloc, target, self.index_name, kind, source); + errdefer self.alloc.free(key); + if (mutations.getPtr(key)) |existing| { + existing.after = after; + self.alloc.free(key); + return; + } + try mutations.put(self.alloc, key, .{ .before = false, .after = after, .kind = kind, .source = source, .target = target }); + } + + fn resolveTopologyMutationPresence(self: *GraphIndex, batch: anytype, mutations: *std.StringHashMapUnmanaged(TopologyMutation), comptime bulk: bool) !void { + const keys = try self.alloc.alloc([]const u8, mutations.count()); + defer self.alloc.free(keys); + var iterator = mutations.keyIterator(); + for (keys) |*key| key.* = iterator.next().?.*; + std.mem.sort([]const u8, keys, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + var present: [256]bool = undefined; + var offset: usize = 0; + while (offset < keys.len) { + const page = keys[offset..@min(keys.len, offset + present.len)]; + if (bulk) try batch.containsManySorted(page, present[0..page.len]) else { + for (page, present[0..page.len]) |key, *exists| exists.* = if (batch.get(key)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + } + for (page, present[0..page.len]) |key, exists| mutations.getPtr(key).?.before = exists; + offset += page.len; + } + } + + fn graphMetricTypeEpochKeyAlloc(self: *GraphIndex, kind: []const u8) ![]u8 { + var key: std.ArrayListUnmanaged(u8) = .empty; + defer key.deinit(self.alloc); + try key.appendSlice(self.alloc, graph_metric_type_epochs_prefix); + try internal_keys.appendEncodedComponent(&key, self.alloc, kind); + return key.toOwnedSlice(self.alloc); + } + + fn graphMetricFilterGeneration(self: *GraphIndex, txn: anytype, filter: GraphMetricEdgeFilter) !u64 { + const current = try readU64OrZero(txn, graph_edge_generation_key); + if (filter.mode == .all) return current; + const floor = txn.get(graph_metric_type_epoch_floor_key) catch |err| switch (err) { + error.NotFound => return current, + else => return err, + }; + if (floor.len != 8) return error.InvalidGraphMetricBuildManifest; + var generation = std.mem.readInt(u64, floor[0..8], .little); + for (filter.types) |kind| { + const key = try self.graphMetricTypeEpochKeyAlloc(kind); + defer self.alloc.free(key); + generation = @max(generation, try readU64OrZero(txn, key)); + } + // Epoch zero means no source publication yet. An empty selected graph + // in a nonempty index still has a stable publishable dependency epoch. + return @max(generation, @intFromBool(current != 0)); + } + + fn graphMetricCurrentGenerationInTxn(self: *GraphIndex, txn: anytype, metric: []const u8) !u64 { + const cfg = self.metricConfig(metric) orelse return error.MetricNotReady; + return self.graphMetricFilterGeneration(txn, cfg.edge_filter); + } + + pub fn graphMetricCurrentGeneration(self: *GraphIndex, metric: []const u8) !u64 { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + return self.graphMetricCurrentGenerationInTxn(&txn, metric); + } + + pub const MetricEdgeScanBenchmark = struct { visited: usize = 0, matched: usize = 0, checksum: u64 = 0 }; + + /// Reference oracle for endpoint-maintenance cost. Abort leaves the same + /// durable fixture available to every sample; WAL/commit are not measured. + pub fn benchmarkTypedMembershipUpdates(self: *GraphIndex, alloc: Allocator, writes: []const BatchWrite, reference: bool) !usize { + var batch = try self.beginWriteReverseBatch(); + defer batch.abort(); + var updates = typed_edges.Updates.init(alloc); + defer updates.deinit(); + for (writes) |write| { + const key = try reverseEdgeKeyAlloc(alloc, write.target, self.index_name, write.edge_type, write.source); + defer alloc.free(key); + if (reference) try typed_edges.update(alloc, &batch, write.edge_type, key, write.source, write.target, false) else try updates.stageKnown(&batch, write.edge_type, key, write.source, write.target, false, true); + } + if (!reference) try updates.flush(&batch); + return if (reference) writes.len * 2 else updates.deltas.count(); + } + + pub fn benchmarkPartitionCensus(self: *GraphIndex, filter: GraphMetricEdgeFilter) !struct { steps: usize, edges: u64, nodes: u64 } { + const cfg = GraphMetricConfig{ .name = "benchmark", .kind = .degree, .edge_filter = filter }; + const key = try self.graphMetricPartitionPlanKeyAlloc(filter); + defer self.alloc.free(key); + { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + batch.delete(key) catch |err| if (err != error.NotFound) return err; + if (filter.mode == .all) batch.delete(graph_metric_partition_census_key) catch |err| if (err != error.NotFound) return err; + try batch.commit(); + } + var steps: usize = 1; + while (!try self.prepareGraphMetricPartitionForConfigStep(cfg, 4096)) steps += 1; + var plan = try self.cachedGraphMetricPartitionPlanForConfig(cfg); + defer plan.deinit(self.alloc); + return .{ .steps = steps, .edges = plan.edge_count, .nodes = plan.node_count }; + } + + pub fn benchmarkMetricEdgeScan(self: *GraphIndex, filter: GraphMetricEdgeFilter, reference: bool) !MetricEdgeScanBenchmark { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cursor = try typed_edges.Cursor.init(self.alloc, &txn, if (reference) GraphMetricEdgeFilter{} else filter, "", "", ""); + defer cursor.deinit(); + var compiled = try CompiledGraphMetricEdgeFilter.init(self.alloc, filter); + defer compiled.deinit(self.alloc); + var result = MetricEdgeScanBenchmark{}; + while (try cursor.next()) |entry| { + result.visited += 1; + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(self.alloc); + if (!compiled.allows(parsed.edge_type.bytes)) continue; + result.matched += 1; + result.checksum +%= std.hash.Wyhash.hash(0, entry.key); + } + return result; + } + + /// Delete an edge (removes from both private graph stores). + pub fn deleteEdge(self: *GraphIndex, source: []const u8, target: []const u8, edge_type: []const u8) !void { + return try self.batchApply(&.{}, &.{.{ + .source = source, + .target = target, + .edge_type = edge_type, + }}); + } + + /// Get edges connected to a key. Caller owns the returned slice and edge data. + pub fn getEdges(self: *GraphIndex, alloc: Allocator, key: []const u8, edge_type: []const u8, direction: EdgeDirection) ![]Edge { + var results = std.ArrayListUnmanaged(Edge).empty; + errdefer { + for (results.items) |e| freeEdge(alloc, e); + results.deinit(alloc); + } + + if (direction == .out or direction == .both) { + try self.scanOutgoingEdges(alloc, &results, key, edge_type); + } + if (direction == .in or direction == .both) { + try self.scanIncomingEdges(alloc, &results, key, edge_type, direction == .both); + } + + return try results.toOwnedSlice(alloc); + } + + /// Get edges connected to a key, restricting storage scans to the requested + /// relationship types. An empty type list retains the unfiltered behavior. + pub fn getEdgesByTypes(self: *GraphIndex, alloc: Allocator, key: []const u8, edge_types: []const []const u8, direction: EdgeDirection) ![]Edge { + if (edge_types.len == 0) return try self.getEdges(alloc, key, "", direction); + + var results = std.ArrayListUnmanaged(Edge).empty; + errdefer { + for (results.items) |e| freeEdge(alloc, e); + results.deinit(alloc); + } + for (edge_types, 0..) |edge_type, type_index| { + var duplicate = false; + for (edge_types[0..type_index]) |prior| { + if (std.mem.eql(u8, edge_type, prior)) { + duplicate = true; + break; + } + } + if (duplicate) continue; + if (direction == .out or direction == .both) { + try self.scanOutgoingEdges(alloc, &results, key, edge_type); + } + if (direction == .in or direction == .both) { + try self.scanIncomingEdges(alloc, &results, key, edge_type, direction == .both); + } + } + return try results.toOwnedSlice(alloc); + } + + /// Read one bounded page of an adjacency in the same deterministic order + /// as `getEdgesByTypes`. The cursor contains logical edge identity rather + /// than backend state, so a caller can resume against the same pinned DB + /// generation across an internal RPC boundary. + pub fn getEdgesByTypesPage( + self: *GraphIndex, + alloc: Allocator, + key: []const u8, + edge_types: []const []const u8, + direction: EdgeDirection, + scan_cursor: ?EdgeScanCursor, + limits: EdgePageLimits, + ) !EdgePage { + if (limits.max_edges == 0) return error.GraphExploredEdgesBudgetExceeded; + if (limits.max_owned_bytes == 0) return error.GraphExploredEdgeBytesBudgetExceeded; + + var results = std.ArrayListUnmanaged(Edge).empty; + errdefer { + for (results.items) |edge| freeEdge(alloc, edge); + results.deinit(alloc); + } + var owned_bytes: usize = 0; + const type_count: usize = if (edge_types.len == 0) 1 else edge_types.len; + var type_index: usize = if (scan_cursor) |cursor| cursor.type_index else 0; + if (type_index >= type_count) return error.InvalidArgument; + if (scan_cursor) |cursor| { + if (cursor.direction == .both or + (direction == .out and cursor.direction != .out) or + (direction == .in and cursor.direction != .in) or + (edge_types.len > 0 and !std.mem.eql(u8, cursor.edge_type, edge_types[type_index]))) + return error.InvalidArgument; + if (edge_types.len > 0) { + for (edge_types[0..type_index]) |prior| { + if (std.mem.eql(u8, edge_types[type_index], prior)) return error.InvalidArgument; + } + } + } + + while (type_index < type_count) : (type_index += 1) { + if (edge_types.len > 0) { + var duplicate = false; + for (edge_types[0..type_index]) |prior| { + if (std.mem.eql(u8, edge_types[type_index], prior)) { + duplicate = true; + break; + } + } + if (duplicate) continue; + } + const requested_type = if (edge_types.len == 0) "" else edge_types[type_index]; + var phase: EdgeDirection = if (scan_cursor) |cursor| + if (cursor.type_index == type_index) cursor.direction else firstScanDirection(direction) + else + firstScanDirection(direction); + + while (true) { + if (results.items.len >= limits.max_edges or owned_bytes >= limits.max_owned_bytes) { + var next_cursor = try edgeScanStartCursor( + alloc, + phase, + @intCast(type_index), + requested_type, + ); + errdefer next_cursor.deinit(alloc); + return .{ + .edges = try results.toOwnedSlice(alloc), + .next_cursor = next_cursor, + .owned_bytes = owned_bytes, + }; + } + const active_resume = if (scan_cursor) |cursor| + cursor.type_index == type_index and cursor.direction == phase + else + false; + const capped = try self.scanEdgePagePhase( + alloc, + &results, + &owned_bytes, + key, + requested_type, + @intCast(type_index), + phase, + if (active_resume) scan_cursor else null, + direction == .both and phase == .in, + limits, + ); + if (capped) |cursor| { + return .{ + .edges = try results.toOwnedSlice(alloc), + .next_cursor = cursor, + .owned_bytes = owned_bytes, + }; + } + if (direction != .both or phase == .in) break; + phase = .in; + } + } + + return .{ + .edges = try results.toOwnedSlice(alloc), + .owned_bytes = owned_bytes, + }; + } + + /// Materialize an adjacency only while it fits the caller's explicit + /// request budget. Page-sized scans ensure the budget is checked before a + /// high-degree node can force unbounded allocation. + pub fn getEdgesByTypesBounded( + self: *GraphIndex, + alloc: Allocator, + key: []const u8, + edge_types: []const []const u8, + direction: EdgeDirection, + max_edges: usize, + max_owned_bytes: usize, + ) ![]Edge { + const page_edge_cap: usize = 4096; + const page_byte_cap: usize = 4 * 1024 * 1024; + var results = std.ArrayListUnmanaged(Edge).empty; + errdefer { + for (results.items) |edge| freeEdge(alloc, edge); + results.deinit(alloc); + } + var total_bytes: usize = 0; + var cursor: ?EdgeScanCursor = null; + defer if (cursor) |*value| value.deinit(alloc); + + while (true) { + const edge_room = if (results.items.len < max_edges) max_edges - results.items.len else 0; + const byte_room = if (total_bytes < max_owned_bytes) max_owned_bytes - total_bytes else 0; + var page = try self.getEdgesByTypesPage( + alloc, + key, + edge_types, + direction, + cursor, + .{ + .max_edges = @min(page_edge_cap, std.math.add(usize, edge_room, 1) catch std.math.maxInt(usize)), + .max_owned_bytes = @min(page_byte_cap, @max(byte_room, 1)), + }, + ); + if (cursor) |*value| value.deinit(alloc); + cursor = null; + errdefer page.deinit(alloc); + + const next_count = std.math.add(usize, results.items.len, page.edges.len) catch + return error.GraphExploredEdgesBudgetExceeded; + const next_bytes = std.math.add(usize, total_bytes, page.owned_bytes) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + if (next_count > max_edges) return error.GraphExploredEdgesBudgetExceeded; + if (next_bytes > max_owned_bytes) return error.GraphExploredEdgeBytesBudgetExceeded; + + try results.ensureUnusedCapacity(alloc, page.edges.len); + for (page.edges) |edge| results.appendAssumeCapacity(edge); + alloc.free(page.edges); + page.edges = @constCast((&[_]Edge{})[0..]); + total_bytes = next_bytes; + cursor = page.next_cursor; + page.next_cursor = null; + page.deinit(alloc); + if (cursor == null) break; + } + return try results.toOwnedSlice(alloc); + } + + fn firstScanDirection(direction: EdgeDirection) EdgeDirection { + return if (direction == .in) .in else .out; + } + + fn scanEdgePagePhase( + self: *GraphIndex, + alloc: Allocator, + results: *std.ArrayListUnmanaged(Edge), + owned_bytes: *usize, + key: []const u8, + requested_type: []const u8, + type_index: u32, + phase: EdgeDirection, + scan_cursor: ?EdgeScanCursor, + skip_mirrored_self_loops: bool, + limits: EdgePageLimits, + ) !?EdgeScanCursor { + std.debug.assert(phase != .both); + const phase_start_len = results.items.len; + const prefix = if (phase == .out) + try edgePrefixAlloc(alloc, key, self.index_name, requested_type) + else + try reverseEdgePrefixAlloc(alloc, key, self.index_name, requested_type); + defer alloc.free(prefix); + + const resume_key = if (scan_cursor) |cursor| + if (cursor.at_phase_start) + null + else if (phase == .out) + try edgeKeyAlloc(alloc, key, self.index_name, cursor.edge_type, cursor.adjacent_key) + else + try reverseEdgeKeyAlloc(alloc, key, self.index_name, cursor.edge_type, cursor.adjacent_key) + else + null; + defer if (resume_key) |value| alloc.free(value); + + var txn = if (phase == .out) try self.beginReadOutgoingTxn() else try self.beginReadReverseTxn(); + defer txn.abort(); + var cursor = try txn.openCursor(); + defer cursor.close(); + + var entry = (try cursor.seekAtOrAfter(resume_key orelse prefix)) orelse return null; + if (resume_key) |value| { + if (std.mem.eql(u8, entry.key, value)) entry = (try cursor.next()) orelse return null; + } + + while (std.mem.startsWith(u8, entry.key, prefix)) { + if (results.items.len >= limits.max_edges) + return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, results.items[results.items.len - 1]); + + const before = results.items.len; + if (phase == .out) + try appendEdgeFromKV(alloc, results, entry.key, entry.value) + else + try appendReverseEdgeFromKV(alloc, results, entry.key, entry.value, skip_mirrored_self_loops); + if (results.items.len != before) { + const appended = results.items[results.items.len - 1]; + const edge_bytes = edgeOwnedBytes(appended); + const next_bytes = std.math.add(usize, owned_bytes.*, edge_bytes) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + if (next_bytes > limits.max_owned_bytes) { + _ = results.pop(); + freeEdge(alloc, appended); + if (results.items.len == 0) return error.GraphExploredEdgeBytesBudgetExceeded; + if (results.items.len == phase_start_len) + return try edgeScanStartCursor(alloc, phase, type_index, requested_type); + return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, results.items[results.items.len - 1]); + } + owned_bytes.* = next_bytes; + if (results.items.len >= limits.max_edges) + return try edgeScanCursorFromPhysicalKey(alloc, phase, type_index, appended); + } + entry = (try cursor.next()) orelse break; + } + return null; + } + + /// Resolve exact physical relationships with one snapshot and one sorted + /// backend multi-get. Results remain aligned with `probes`; null means the + /// relationship does not exist. Only found edges allocate edge payloads. + pub fn probeEdgesAlloc(self: *GraphIndex, alloc: Allocator, probes: []const EdgeProbe) ![]?Edge { + return try self.probeEdgesAllocBounded(alloc, probes, std.math.maxInt(usize)); + } + + /// Resolve exact physical relationships without allowing decoded edge + /// payloads to exceed the caller's remaining request budget. The limit is + /// checked against values borrowed from the read transaction before any + /// found edge payload is copied into caller-owned memory. + pub fn probeEdgesAllocBounded( + self: *GraphIndex, + alloc: Allocator, + probes: []const EdgeProbe, + max_owned_bytes: usize, + ) ![]?Edge { + const ProbeKey = struct { + encoded: []u8, + result_index: usize, + + fn lessThan(_: void, left: @This(), right: @This()) bool { + return std.mem.order(u8, left.encoded, right.encoded) == .lt; + } + }; + + const results = try alloc.alloc(?Edge, probes.len); + errdefer alloc.free(results); + @memset(results, null); + if (probes.len == 0) return results; + + const keys = try alloc.alloc(ProbeKey, probes.len); + var initialized_keys: usize = 0; + defer { + for (keys[0..initialized_keys]) |item| alloc.free(item.encoded); + alloc.free(keys); + } + for (probes, 0..) |probe, i| { + keys[i] = .{ + .encoded = try edgeKeyAlloc(alloc, probe.source, self.index_name, probe.edge_type, probe.target), + .result_index = i, + }; + initialized_keys += 1; + } + std.mem.sort(ProbeKey, keys, {}, ProbeKey.lessThan); + + const sorted_key_refs = try alloc.alloc([]const u8, keys.len); + defer alloc.free(sorted_key_refs); + for (keys, 0..) |item, i| sorted_key_refs[i] = item.encoded; + const values = try alloc.alloc(?[]const u8, keys.len); + defer alloc.free(values); + + var txn = try self.beginReadOutgoingTxn(); + defer txn.abort(); + try txn.getManySorted(sorted_key_refs, values); + + errdefer { + for (results) |maybe_edge| if (maybe_edge) |edge| freeEdge(alloc, edge); + } + var owned_bytes: usize = 0; + for (keys, values) |item, maybe_value| { + const value = maybe_value orelse continue; + const decoded = try decodeEdgeValue(value); + const probe = probes[item.result_index]; + var edge_bytes: usize = @sizeOf(Edge); + edge_bytes = std.math.add(usize, edge_bytes, probe.source.len) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + edge_bytes = std.math.add(usize, edge_bytes, probe.target.len) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + edge_bytes = std.math.add(usize, edge_bytes, probe.edge_type.len) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + edge_bytes = std.math.add(usize, edge_bytes, decoded.metadata.len) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + owned_bytes = std.math.add(usize, owned_bytes, edge_bytes) catch + return error.GraphExploredEdgeBytesBudgetExceeded; + if (owned_bytes > max_owned_bytes) return error.GraphExploredEdgeBytesBudgetExceeded; + const source = try alloc.dupe(u8, probe.source); + errdefer alloc.free(source); + const target = try alloc.dupe(u8, probe.target); + errdefer alloc.free(target); + const edge_type = try alloc.dupe(u8, probe.edge_type); + errdefer alloc.free(edge_type); + const metadata = if (decoded.metadata.len > 0) + try alloc.dupe(u8, decoded.metadata) + else + ""; + errdefer if (metadata.len > 0) alloc.free(metadata); + results[item.result_index] = .{ + .source = source, + .target = target, + .edge_type = edge_type, + .weight = decoded.weight, + .created_at = decoded.created_at, + .updated_at = decoded.updated_at, + .metadata = metadata, + }; + } + return results; + } + + pub fn freeProbedEdges(alloc: Allocator, edges: []?Edge) void { + for (edges) |maybe_edge| if (maybe_edge) |edge| freeEdge(alloc, edge); + alloc.free(edges); + } + + /// Probe incoming-edge existence for a key batch using one reverse-store + /// snapshot and one cursor. Results are aligned with `keys`. + pub fn hasIncomingEdgesManyAlloc( + self: *GraphIndex, + alloc: Allocator, + keys: []const []const u8, + ) ![]bool { + const result = try alloc.alloc(bool, keys.len); + errdefer alloc.free(result); + @memset(result, false); + if (keys.len == 0) return result; + + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cursor = try txn.openCursor(); + defer cursor.close(); + + for (keys, 0..) |key, i| { + const prefix = try reverseEdgePrefixAlloc(alloc, key, self.index_name, ""); + defer alloc.free(prefix); + const first = (try cursor.seekAtOrAfter(prefix)) orelse continue; + result[i] = std.mem.startsWith(u8, first.key, prefix); + } + return result; + } + + fn scanOutgoingEdges(self: *GraphIndex, alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), key: []const u8, edge_type: []const u8) !void { + const prefix = try edgePrefixAlloc(alloc, key, self.index_name, edge_type); + defer alloc.free(prefix); + + var txn = try self.beginReadOutgoingTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + + const first = (try cur.seekAtOrAfter(prefix)) orelse return; + if (!std.mem.startsWith(u8, first.key, prefix)) return; + try appendEdgeFromKV(alloc, results, first.key, first.value); + while (try cur.next()) |entry| { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + try appendEdgeFromKV(alloc, results, entry.key, entry.value); + } + } + + fn scanIncomingEdges( + self: *GraphIndex, + alloc: Allocator, + results: *std.ArrayListUnmanaged(Edge), + key: []const u8, + edge_type: []const u8, + skip_mirrored_self_loops: bool, + ) !void { + const prefix = try reverseEdgePrefixAlloc(alloc, key, self.index_name, edge_type); + defer alloc.free(prefix); + + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + + var cur = try txn.openCursor(); + defer cur.close(); + + const first = (try cur.seekAtOrAfter(prefix)) orelse return; + + if (std.mem.startsWith(u8, first.key, prefix)) { + try appendReverseEdgeFromKV(alloc, results, first.key, first.value, skip_mirrored_self_loops); + } else { + return; + } + + while (try cur.next()) |entry| { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + try appendReverseEdgeFromKV(alloc, results, entry.key, entry.value, skip_mirrored_self_loops); + } + } + + fn appendEdgeFromKV(alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), key: []const u8, value: []const u8) !void { + var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, key)) orelse return; + defer parsed.deinit(alloc); + try appendParsedEdge(alloc, results, parsed, value); + } + + fn appendReverseEdgeFromKV( + alloc: Allocator, + results: *std.ArrayListUnmanaged(Edge), + key: []const u8, + value: []const u8, + skip_mirrored_self_loops: bool, + ) !void { + var parsed = (try parseReverseEdgeKeyAlloc(alloc, key)) orelse return; + defer parsed.deinit(alloc); + // A physical self-loop is indexed once in each adjacency direction so + // independent `out` and `in` reads remain complete. A `both` read has + // already emitted the outgoing copy, so suppress only its mirrored + // reverse-index representation. Reciprocal non-self edges remain + // distinct because their physical source/target identities differ. + if (skip_mirrored_self_loops and std.mem.eql(u8, parsed.source, parsed.target)) return; + try appendParsedEdge(alloc, results, parsed, value); + } + + fn appendParsedEdge(alloc: Allocator, results: *std.ArrayListUnmanaged(Edge), parsed: ParsedGraphEdgeKey, value: []const u8) !void { + const decoded = try decodeEdgeValue(value); + const source = try alloc.dupe(u8, parsed.source); + errdefer alloc.free(source); + const target = try alloc.dupe(u8, parsed.target); + errdefer alloc.free(target); + const edge_type = try alloc.dupe(u8, parsed.edge_type); + errdefer alloc.free(edge_type); + const metadata = if (decoded.metadata.len > 0) try alloc.dupe(u8, decoded.metadata) else ""; + errdefer if (metadata.len > 0) alloc.free(metadata); + try results.append(alloc, .{ + .source = source, + .target = target, + .edge_type = edge_type, + .weight = decoded.weight, + .created_at = decoded.created_at, + .updated_at = decoded.updated_at, + .metadata = metadata, + }); + } + + /// Delete all outgoing edges for a document (cleanup on doc deletion). + pub fn deleteEdgesForDoc(self: *GraphIndex, doc_key: []const u8) !void { + const edges = try self.getEdges(self.alloc, doc_key, "", .both); + defer freeEdges(self.alloc, edges); + + var deletes = try self.alloc.alloc(BatchDelete, edges.len); + defer self.alloc.free(deletes); + for (edges, 0..) |edge, i| { + deletes[i] = .{ + .source = edge.source, + .target = edge.target, + .edge_type = edge.edge_type, + }; + } + try self.batchApply(&.{}, deletes); + } + + fn validateTreeBatchWrites(self: *GraphIndex, writes: []const BatchWrite, deletes: []const BatchDelete) !void { + for (writes, 0..) |write, i| { + if (self.getTopologyMode(write.edge_type) != .tree) continue; + + const existing = try self.getEdges(self.alloc, write.source, write.edge_type, .out); + defer freeEdges(self.alloc, existing); + + for (existing) |edge| { + if (containsBatchDelete(deletes, edge.source, edge.target, edge.edge_type)) continue; + if (!std.mem.eql(u8, edge.target, write.target)) { + return TreeTopologyViolation.TreeTopologyViolation; + } + } + + for (writes[0..i]) |prior| { + if (!std.mem.eql(u8, prior.source, write.source)) continue; + if (!std.mem.eql(u8, prior.edge_type, write.edge_type)) continue; + if (containsBatchDelete(deletes, prior.source, prior.target, prior.edge_type)) continue; + if (!std.mem.eql(u8, prior.target, write.target)) { + return TreeTopologyViolation.TreeTopologyViolation; + } + } + } + } + + pub fn rebuildReverseFromOwnedOutgoingEdges(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + return try self.rebuildReverseFromOwnedOutgoingEdgesResumeWithIo(alloc, io_impl.io(), lower, upper, null); + } + + pub fn copyOwnedOutgoingEdgesTo(self: *GraphIndex, dest: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { + const range_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; + defer if (range_lower_owned) |key| alloc.free(key); + const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; + defer if (range_upper_owned) |key| alloc.free(key); + const range_lower = range_lower_owned orelse ""; + const range_upper = range_upper_owned orelse ""; + + const pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); + defer backend_scan.freeResults(alloc, pairs); + + var batch = try dest.beginWriteOutgoingBatch(); + errdefer batch.abort(); + var copied: usize = 0; + for (pairs) |pair| { + var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; + defer parsed.deinit(alloc); + if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; + if (!std.mem.eql(u8, dest.index_name, self.index_name)) continue; + try batch.put(pair.key, pair.value); + copied += 1; + } + try batch.commit(); + return copied; + } + + pub fn rebuildReverseFromOwnedOutgoingEdgesResume( + self: *GraphIndex, + alloc: Allocator, + lower: []const u8, + upper: []const u8, + resume_from: ?[]const u8, + ) !usize { + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + return try self.rebuildReverseFromOwnedOutgoingEdgesResumeWithIo(alloc, io_impl.io(), lower, upper, resume_from); + } + + pub fn rebuildReverseFromOwnedOutgoingEdgesResumeWithIo( + self: *GraphIndex, + alloc: Allocator, + io: std.Io, + lower: []const u8, + upper: []const u8, + resume_from: ?[]const u8, + ) !usize { + const base_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; + defer if (base_lower_owned) |key| alloc.free(key); + const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; + defer if (range_upper_owned) |key| alloc.free(key); + const base_lower = base_lower_owned orelse ""; + const range_lower = if (resume_from) |key| + if (key.len > 0 and std.mem.order(u8, key, base_lower) == .gt) key else base_lower + else + base_lower; + const range_upper = range_upper_owned orelse ""; + + const pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); + defer backend_scan.freeResults(alloc, pairs); + + var rebuilt: usize = 0; + var batch_count: usize = 0; + var flushed_batches: usize = 0; + var matching_edges: usize = 0; + var txn = try self.beginWriteReverseTxn(); + var txn_active = true; + errdefer if (txn_active) txn.abort(); + const rebuild_state = if (self.rebuild_root_path) |path| + if (self.rebuild_owner_generation != 0) + backfill_state_mod.RebuildState.initOwned(path, self.rebuild_storage, self.rebuild_owner_generation) + else + backfill_state_mod.RebuildState.initWithStorage(path, self.rebuild_storage) + else + null; + + for (pairs) |pair| { + if (resume_from) |resume_key| { + if (resume_key.len > 0 and std.mem.order(u8, pair.key, resume_key) != .gt) continue; + } + var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; + defer parsed.deinit(alloc); + if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; + matching_edges += 1; + + const rev_key = try reverseEdgeKeyAlloc(alloc, parsed.target, self.index_name, parsed.edge_type, parsed.source); + defer alloc.free(rev_key); + try txn.put(rev_key, pair.value); + rebuilt += 1; + batch_count += 1; + + if (batch_count >= reverse_rebuild_batch_size) { + try txn.commit(); + txn_active = false; + if (rebuild_state) |state| try state.updateWithIo(io, pair.key); + flushed_batches += 1; + if (@import("builtin").is_test) { + if (test_abort_reverse_rebuild_after_batches) |limit| { + if (flushed_batches >= limit) return error.TestInjectedBackfillFailure; + } + } + txn = try self.beginWriteReverseTxn(); + txn_active = true; + batch_count = 0; + } + } + + try txn.commit(); + txn_active = false; + if (rebuild_state) |state| try state.clearWithIo(io); + try self.rebuildCounterMetadata(); + try self.checkpointLsmWalAfterDurableBoundary(); + return rebuilt; + } + + pub fn pruneOwnedRange(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) !usize { + var removed: usize = 0; + + const range_lower_owned = if (lower.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, lower) else null; + defer if (range_lower_owned) |key| alloc.free(key); + const range_upper_owned = if (upper.len > 0) try internal_keys.documentRangeLowerAlloc(alloc, upper) else null; + defer if (range_upper_owned) |key| alloc.free(key); + const range_lower = range_lower_owned orelse ""; + const range_upper = range_upper_owned orelse ""; + + const owned_pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); + defer backend_scan.freeResults(alloc, owned_pairs); + + var outgoing_batch = try self.beginWriteOutgoingBatch(); + errdefer outgoing_batch.abort(); + var reverse_txn = try self.beginWriteReverseTxn(); + errdefer reverse_txn.abort(); + + for (owned_pairs) |pair| { + var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; + defer parsed.deinit(alloc); + if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; + + const rev_key = try reverseEdgeKeyAlloc(alloc, parsed.target, self.index_name, parsed.edge_type, parsed.source); + defer alloc.free(rev_key); + outgoing_batch.delete(pair.key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + reverse_txn.delete(rev_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + removed += 1; + } + + // Reverse rows are projections of source-owned outgoing edges, not + // target-owned records. Keep projections whose target moved to another + // range; distributed incoming reads fan out across source owners. The + // loop above already removes the exact reverse projection for every + // outgoing edge whose source is leaving this range. + // + // Match normal graph batch publication order: make forward ownership + // authoritative first, then retire the corresponding projections. + try outgoing_batch.commit(); + try reverse_txn.commit(); + try self.rebuildCounterMetadata(); + return removed; + } + + fn mainStoreScanPrefix(self: *GraphIndex, alloc: Allocator, prefix: []const u8) ![]backend_scan.OwnedKVPair { + return try backend_scan.scanPrefix(alloc, &self.outgoing_store, prefix); + } + + fn mainStoreScanRange(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) ![]backend_scan.OwnedKVPair { + return try backend_scan.scanRange(alloc, &self.outgoing_store, lower, upper); + } + + fn containsBatchDelete( + deletes: []const BatchDelete, + source: []const u8, + target: []const u8, + edge_type: []const u8, + ) bool { + for (deletes) |delete| { + if (!std.mem.eql(u8, delete.source, source)) continue; + if (!std.mem.eql(u8, delete.target, target)) continue; + if (!std.mem.eql(u8, delete.edge_type, edge_type)) continue; + return true; + } + return false; + } + + pub const GraphMetricState = enum { + disabled, + not_ready, + fresh, + stale, + building, + failed, + }; + + pub const GraphMetricBuildPhase = enum { + idle, + computing, + publishing, + complete, + prepare_generation, + scan_edges_and_out_degree, + initialize_ranks, + iterate_contributions, + reduce_ranks, + hits_hub_contributions, + hits_hub_reduce_ranks, + check_convergence, + publish_generation, + cleanup_old_generations, + }; + + pub const GraphMetricEventKind = enum(u8) { + publish, + delete, + pause, + @"resume", + failed, + }; + + pub const GraphMetricEvent = struct { + sequence: u64 = 0, + kind: GraphMetricEventKind, + at_ms: u64 = 0, + target_edge_generation: u64 = 0, + published_generation: u64 = 0, + score_count: u64 = 0, + }; + + pub const GraphMetricFailureRecord = struct { + sequence: u64 = 0, + at_ms: u64 = 0, + job_id: u64 = 0, + target_generation: u64 = 0, + score_generation: u64 = 0, + phase: GraphMetricBuildPhase = .idle, + iteration: u32 = 0, + retry_count: u64 = 0, + last_error: []const u8 = "", + + pub fn deinit(self: *@This(), alloc: Allocator) void { + if (self.last_error.len > 0) alloc.free(self.last_error); + self.* = undefined; + } + }; + + pub const GraphMetricBuildPageStatus = struct { + phase: GraphMetricBuildPhase = .idle, + iteration: u32 = 0, + page_id: u64 = 0, + state: GraphMetricBuildPageState = .pending, + range_kind: GraphMetricBuildPageRangeKind = .full, + worker_id: []const u8 = "", + lease_expires_at_ms: u64 = 0, + attempt: u64 = 0, + cursor: []const u8 = "", + completed_units: u64 = 0, + total_units: u64 = 0, + last_error: []const u8 = "", + + pub fn deinit(self: *@This(), alloc: Allocator) void { + if (self.worker_id.len > 0) alloc.free(self.worker_id); + if (self.cursor.len > 0) alloc.free(self.cursor); + if (self.last_error.len > 0) alloc.free(self.last_error); + self.* = undefined; + } + }; + + pub const GraphMetricStatus = struct { + name: []const u8, + state: GraphMetricState = .not_ready, + phase: GraphMetricBuildPhase = .idle, + edge_filter: GraphMetricEdgeFilter = .{}, + metadata_version: u32 = 0, + config_fingerprint: u64 = 0, + maintenance_paused: bool = false, + build_queued: bool = false, + published_generation: u64 = 0, + /// Edge snapshot identity represented by the internal published score + /// namespace. API adapters expose this as `published_generation`. + published_edge_generation: u64 = 0, + edge_generation: u64 = 0, + target_edge_generation: u64 = 0, + queued_generation: u64 = 0, + building_generation: u64 = 0, + build_job_id: u64 = 0, + build_started_at_ms: u64 = 0, + build_iteration: u32 = 0, + build_lease_expires_at_ms: u64 = 0, + build_worker_id: []const u8 = "", + build_cursor: []const u8 = "", + build_completed_units: u64 = 0, + build_total_units: u64 = 0, + build_pages: []GraphMetricBuildPageStatus = &.{}, + build_pages_truncated: bool = false, + retry_count: u64 = 0, + last_error: []const u8 = "", + progress: f64 = 0.0, + converged: bool = false, + iterations_completed: u32 = 0, + delta: f64 = 0.0, + computed_at_ms: u64 = 0, + last_event: ?GraphMetricEvent = null, + recent_events: []GraphMetricEvent = &.{}, + recent_failures: []GraphMetricFailureRecord = &.{}, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.name); + self.edge_filter.deinit(alloc); + if (self.build_worker_id.len > 0) alloc.free(self.build_worker_id); + if (self.build_cursor.len > 0) alloc.free(self.build_cursor); + for (self.build_pages) |*page| page.deinit(alloc); + if (self.build_pages.len > 0) alloc.free(self.build_pages); + if (self.last_error.len > 0) alloc.free(self.last_error); + if (self.recent_events.len > 0) alloc.free(self.recent_events); + for (self.recent_failures) |*failure| failure.deinit(alloc); + if (self.recent_failures.len > 0) alloc.free(self.recent_failures); + self.* = undefined; + } + }; + + /// Allocation-light control-plane view used by maintenance schedulers. + /// Operator status intentionally remains rich; hot scheduler polling must + /// not clone filters, histories, worker strings, or page arrays. + pub const GraphMetricSchedulerStatus = struct { + state: GraphMetricState = .not_ready, + phase: GraphMetricBuildPhase = .idle, + maintenance_paused: bool = false, + target_edge_generation: u64 = 0, + failed_target_generation: u64 = 0, + }; + + pub const GraphMetricScore = struct { + node: []const u8, + score: f64, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.node); + self.* = undefined; + } + }; + + // Execution rows carry their immutable topology ordinal through job-local vector + // reads and writes. Never expose these ordinals as public score identity. + const OrdinalMetricScore = struct { node: []const u8, score: f64, slot: u64 }; + + /// Owned query view of one published metric generation. Status and point + /// scores are read under the same storage snapshot so generation cleanup + /// can never turn a concurrent rerank into a mixture of metadata and + /// missing values. + pub const GraphMetricScoreSnapshot = struct { + status: GraphMetricStatus, + scores: []?f64, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + self.status.deinit(alloc); + alloc.free(self.scores); + self.* = undefined; + } + }; + + /// Owned top-K view pinned to the same reverse-store snapshot as its + /// publication status. Public query code must use this instead of issuing + /// independent status and rank reads: publication may advance and retire + /// an old generation between two transactions. + pub const GraphMetricTopKSnapshot = struct { + status: GraphMetricStatus, + scores: []GraphMetricScore, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + self.status.deinit(alloc); + for (self.scores) |*score| score.deinit(alloc); + if (self.scores.len > 0) alloc.free(self.scores); + self.* = undefined; + } + }; + + pub const GraphMetricColumnReadPolicy = struct { + require_published: bool = false, + require_fresh: bool = false, + }; + + /// Multiple published metric columns pinned to one reverse-store snapshot. + /// This is the query boundary used by metric filtering and ordering so a + /// concurrent publication can never mix generations across dependencies. + pub const GraphMetricColumnsSnapshot = struct { + statuses: []GraphMetricStatus, + score_columns: [][]?f64, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + for (self.statuses) |*status| status.deinit(alloc); + if (self.statuses.len > 0) alloc.free(self.statuses); + for (self.score_columns) |column| alloc.free(column); + if (self.score_columns.len > 0) alloc.free(self.score_columns); + self.* = undefined; + } + }; + + pub const GraphMetricPlannedDrainOptions = struct { + worker_ids: []const []const u8, + max_steps: usize = 100000, + }; + + const GraphMetricMeta = struct { + schema_version: u32 = graph_metric_meta_schema_version, + target_edge_generation: u64 = 0, + converged: bool = false, + iterations_completed: u32 = 0, + delta: f64 = 0.0, + computed_at_ms: u64 = 0, + config_fingerprint: u64 = 0, + edge_filter: GraphMetricEdgeFilter = .{}, + }; + + const GraphMetricBuildLease = struct { + job_id: u64 = 0, + target_generation: u64 = 0, + started_at_ms: u64 = 0, + lease_expires_at_ms: u64 = 0, + phase: GraphMetricBuildPhase = .computing, + iteration: u32 = 0, + worker_id: []const u8 = graph_metric_local_build_worker_id, + }; + + const GraphMetricBuildJob = struct { + job_id: u64 = 0, + target_generation: u64 = 0, + score_generation: u64 = 0, + started_at_ms: u64 = 0, + updated_at_ms: u64 = 0, + lease_expires_at_ms: u64 = 0, + phase: GraphMetricBuildPhase = .computing, + iteration: u32 = 0, + retry_count: u64 = 0, + worker_id: []const u8 = graph_metric_local_build_worker_id, + last_error: []const u8 = "", + cursor: []const u8 = "", + completed_units: u64 = 0, + total_units: u64 = 0, + }; + + // v7 uses direct attempt-tagged ordinal shards and bounded summary leaves + // for every iterative build, independent of node count. + // v8 retains attempt-fenced adjacency once and folds input vectors directly. + // v9 packs producer fragments into dense, receipt-selected adjacency tiles. + // v10 bounds partition census work; v11 seals canonical membership blocks. + // v12 stages ordered publication runs; v13 seals metric-specific node work + // plans so later iterations omit empty leaves without changing ordinals. + // v14 removes later producer phases; reducers reuse iteration-zero adjacency. + // v15 separates durable topology ownership from numerical job lifetimes. + // v16 amortizes scheduling over 4096-unit ranges and adds independent, + // numerical-free topology preparation in its own durable task namespace. + // v17 seals checksummed ordinal coverage and resumes numerical node work + // by completed-unit offsets instead of borrowed/string dictionary cursors. + // Older in-flight jobs must restart; published score layout is unchanged. + const graph_metric_build_execution_schema_version: u64 = 20; + + const GraphMetricBuildManifest = struct { + execution_schema_version: u64 = graph_metric_build_execution_schema_version, + job_id: u64 = 0, + target_generation: u64 = 0, + score_generation: u64 = 0, + config_fingerprint: u64 = 0, + planned_at_ms: u64 = 0, + edge_count: u64 = 0, + node_count: u64 = 0, + phase_count: usize = 0, + page_count: usize = 0, + }; + + pub const GraphMetricBuildPageState = enum(u8) { + pending, + leased, + complete, + failed, + }; + + pub const GraphMetricBuildPageRangeKind = enum(u8) { + full, + reverse_edges, + nodes, + scores, + contributions, + job_control, + /// Ordered scalar pre-pass consumed by the remaining pages in a + /// phase. A summary page prevents every partition from rescanning the + /// full graph for a shared normalization value. + summary, + }; + + pub const GraphMetricBuildPageSnapshotForTest = struct { + state: GraphMetricBuildPageState = .pending, + worker_id_hash: u64 = 0, + attempt: u64 = 0, + completed_units: u64 = 0, + total_units: u64 = 0, + output_fingerprint: u64 = 0, + }; + + const GraphMetricBuildPage = struct { + job_id: u64 = 0, + phase: GraphMetricBuildPhase = .prepare_generation, + iteration: u32 = 0, + page_id: u64 = 0, + state: GraphMetricBuildPageState = .pending, + range_kind: GraphMetricBuildPageRangeKind = .full, + range_lower: []const u8 = "", + range_upper: []const u8 = "", + output_prefix: []const u8 = "", + worker_id: []const u8 = "", + lease_expires_at_ms: u64 = 0, + attempt: u64 = 0, + cursor: []const u8 = "", + completed_units: u64 = 0, + total_units: u64 = 0, + last_error: []const u8 = "", + output_fingerprint: u64 = 0, + max_delta: f64 = 0.0, + total_delta: f64 = 0.0, + rank_sum: f64 = 0.0, + converged: bool = false, + }; + + const GraphMetricBuildPhaseState = enum(u8) { + pending, + complete, + failed, + }; + + const GraphMetricBuildPhaseSummary = struct { + job_id: u64 = 0, + phase: GraphMetricBuildPhase = .prepare_generation, + iteration: u32 = 0, + state: GraphMetricBuildPhaseState = .pending, + expected_pages: u64 = 0, + completed_pages: u64 = 0, + failed_pages: u64 = 0, + completed_units: u64 = 0, + total_units: u64 = 0, + max_delta: f64 = 0.0, + total_delta: f64 = 0.0, + rank_sum: f64 = 0.0, + converged: bool = false, + output_fingerprint: u64 = 0, + }; + + /// Transactionally maintained, order-independent phase state. Floating + /// aggregates intentionally remain in the phase summary: they are reduced + /// once in page-key order after this record reports full completion, so + /// distributed worker timing cannot change published results. + const GraphMetricBuildPhaseProgress = struct { + job_id: u64 = 0, + phase: GraphMetricBuildPhase = .prepare_generation, + iteration: u32 = 0, + expected_pages: u64 = 0, + pending_pages: u64 = 0, + leased_pages: u64 = 0, + completed_pages: u64 = 0, + failed_pages: u64 = 0, + max_attempt_pages: u64 = 0, + completed_units: u64 = 0, + total_units: u64 = 0, + next_claim_page_id: u64 = 0, + }; + + const GraphMetricBuildIterationSummary = struct { + job_id: u64 = 0, + iteration: u32 = 0, + expected_pages: u64 = 0, + completed_pages: u64 = 0, + max_delta: f64 = 0.0, + total_delta: f64 = 0.0, + rank_sum: f64 = 0.0, + converged: bool = false, + fixed_iteration_limit: bool = false, + output_fingerprint: u64 = 0, + }; + + const GraphMetricBuildPublishVerification = struct { + job_id: u64 = 0, + target_generation: u64 = 0, + score_generation: u64 = 0, + config_fingerprint: u64 = 0, + iteration: u32 = 0, + expected_phases: u64 = 0, + completed_phases: u64 = 0, + expected_pages: u64 = 0, + completed_pages: u64 = 0, + output_fingerprint: u64 = 0, + converged: bool = false, + fixed_iteration_limit: bool = false, + max_delta: f64 = 0.0, + total_delta: f64 = 0.0, + rank_sum: f64 = 0.0, + }; + + pub const GraphMetricBuildWorkerStepResult = struct { + checkpointed_publication: bool = false, + retired_input_records: usize = 0, + phase: GraphMetricBuildPhase = .idle, + page_id: u64 = 0, + claimed_page: bool = false, + completed_page: bool = false, + advanced_phase: bool = false, + published: bool = false, + completed_build: bool = false, + failed_build: bool = false, + }; + + const GraphMetricBuildPageExecutionResult = struct { + phase: GraphMetricBuildPhase = .idle, + page_id: u64 = 0, + completed_page: bool = false, + completed_units: u64 = 0, + total_units: u64 = 0, + output_fingerprint: u64 = 0, + max_delta: f64 = 0.0, + total_delta: f64 = 0.0, + rank_sum: f64 = 0.0, + score_count: usize = 0, + published: bool = false, + completed_build: bool = false, + }; + + const GraphMetricBuildPageCompletionSnapshot = struct { + state: GraphMetricBuildPageState = .pending, + completed_units: u64 = 0, + total_units: u64 = 0, + output_fingerprint: u64 = 0, + }; + + const GraphMetricBuildPageExhaustion = struct { + phase: GraphMetricBuildPhase, + iteration: u32, + page_id: u64, + attempt: u64, + last_error: []u8, + + fn deinit(self: *@This(), alloc: Allocator) void { + if (self.last_error.len > 0) alloc.free(self.last_error); + self.* = undefined; + } + }; + + const PrefixDeletePageResult = struct { + removed: usize = 0, + reached_end: bool = true, + cursor: []u8 = "", + + fn deinit(self: *PrefixDeletePageResult, alloc: Allocator) void { + if (self.cursor.len > 0) alloc.free(self.cursor); + self.* = .{}; + } + }; + + const GraphMetricAttemptAdoptionResult = struct { + adopted: usize = 0, + reached_end: bool = true, + }; + + const GraphMetricFailureDetail = struct { + retry_count: u64 = 0, + last_error: []const u8 = "", + + fn deinit(self: GraphMetricFailureDetail, alloc: Allocator) void { + if (self.last_error.len > 0) alloc.free(self.last_error); + } + }; + + pub const graph_metric_meta_schema_version: u32 = 4; + const graph_metric_meta_legacy_encoded_len = 8 + 8 + 8 + 8; + const graph_metric_meta_v1_encoded_len = 8 + graph_metric_meta_legacy_encoded_len; + const graph_metric_meta_v3_encoded_len = graph_metric_meta_v1_encoded_len + 8; + const graph_metric_meta_encoded_len = graph_metric_meta_v3_encoded_len + 8; + const graph_metric_edge_filter_header_len = 8 + 8; + const graph_metric_build_lease_legacy_encoded_len = 8 + 8 + 8; + const graph_metric_build_lease_v1_header_len = 8 + graph_metric_build_lease_legacy_encoded_len + 8 + 8; + const graph_metric_build_lease_v2_header_len = graph_metric_build_lease_v1_header_len + 8; + const graph_metric_build_lease_header_len = graph_metric_build_lease_v2_header_len + 8; + const graph_metric_build_job_v1_header_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_build_job_v2_header_len = graph_metric_build_job_v1_header_len + 8 + 8; + const graph_metric_build_job_header_len = graph_metric_build_job_v2_header_len + 8 + 8 + 8; + const graph_metric_build_manifest_encoded_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_build_page_v1_header_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_build_page_v2_header_len = graph_metric_build_page_v1_header_len + 8 + 8 + 8 + 8; + const graph_metric_build_page_header_len = graph_metric_build_page_v2_header_len + 8 + 8 + 8 + 8; + const graph_metric_build_phase_summary_encoded_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_build_phase_progress_encoded_len = 13 * @sizeOf(u64); + const graph_metric_build_iteration_summary_encoded_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_failure_detail_header_len = 8; + const graph_metric_failure_record_header_len = 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8; + const graph_metric_event_encoded_len = 8 + 8 + 8 + 8 + 8; + const graph_metric_iterative_build_phases = [_]GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + .publish_generation, + .cleanup_old_generations, + }; + const graph_metric_hits_build_phases = [_]GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + .cleanup_old_generations, + }; + const graph_metric_degree_build_phases = [_]GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .reduce_ranks, + .publish_generation, + .cleanup_old_generations, + }; + + fn encodeGraphMetricMeta(meta: GraphMetricMeta, out: *[graph_metric_meta_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + @as(u64, meta.schema_version), + meta.target_edge_generation, + if (meta.converged) @as(u64, 1) else @as(u64, 0), + @as(u64, meta.iterations_completed), + @as(u64, @bitCast(meta.delta)), + meta.computed_at_ms, + meta.config_fingerprint, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn decodeGraphMetricMeta(raw: []const u8) ?GraphMetricMeta { + if (raw.len != graph_metric_meta_encoded_len and raw.len != graph_metric_meta_v3_encoded_len and raw.len != graph_metric_meta_v1_encoded_len and raw.len != graph_metric_meta_legacy_encoded_len) return null; + var offset: usize = 0; + const has_schema_version = raw.len == graph_metric_meta_encoded_len or raw.len == graph_metric_meta_v3_encoded_len or raw.len == graph_metric_meta_v1_encoded_len; + const schema_version: u32 = if (has_schema_version) blk: { + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (value > graph_metric_meta_schema_version) return null; + break :blk @intCast(value); + } else 0; + const target_edge_generation = if (raw.len == graph_metric_meta_encoded_len) blk: { + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + break :blk value; + } else 0; + const converged = std.mem.readInt(u64, raw[offset..][0..8], .little) != 0; + offset += 8; + const iterations_completed: u32 = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const delta_bits = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const computed_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const config_fingerprint = if (raw.len == graph_metric_meta_encoded_len or raw.len == graph_metric_meta_v3_encoded_len) + std.mem.readInt(u64, raw[offset..][0..8], .little) + else + 0; + return .{ + .schema_version = schema_version, + .target_edge_generation = target_edge_generation, + .converged = converged, + .iterations_completed = iterations_completed, + .delta = @bitCast(delta_bits), + .computed_at_ms = computed_at_ms, + .config_fingerprint = config_fingerprint, + }; + } + + fn graphMetricEdgeFilterEncodedLen(filter: GraphMetricEdgeFilter) usize { + var len: usize = graph_metric_edge_filter_header_len; + for (filter.types) |edge_type| len += 8 + edge_type.len; + return len; + } + + fn encodeGraphMetricEdgeFilter(filter: GraphMetricEdgeFilter, out: []u8) void { + std.debug.assert(out.len == graphMetricEdgeFilterEncodedLen(filter)); + std.mem.writeInt(u64, out[0..8], @intFromEnum(filter.mode), .little); + std.mem.writeInt(u64, out[8..16], filter.types.len, .little); + var offset: usize = graph_metric_edge_filter_header_len; + for (filter.types) |edge_type| { + std.mem.writeInt(u64, out[offset..][0..8], edge_type.len, .little); + offset += 8; + @memcpy(out[offset..][0..edge_type.len], edge_type); + offset += edge_type.len; + } + } + + fn decodeGraphMetricEdgeFilterAlloc(alloc: Allocator, raw: []const u8) !?GraphMetricEdgeFilter { + if (raw.len < graph_metric_edge_filter_header_len) return null; + const mode_raw = std.mem.readInt(u64, raw[0..8], .little); + const mode: GraphMetricEdgeFilterMode = switch (mode_raw) { + @intFromEnum(GraphMetricEdgeFilterMode.all) => .all, + @intFromEnum(GraphMetricEdgeFilterMode.types) => .types, + else => return null, + }; + const count = std.math.cast(usize, std.mem.readInt(u64, raw[8..16], .little)) orelse return error.InvalidGraphMetricEdgeFilterMetadata; + var offset: usize = graph_metric_edge_filter_header_len; + if (mode == .all) { + if (count != 0 or offset != raw.len) return null; + return .{}; + } + if (count == 0) return null; + if (count > (raw.len - offset) / 9) return error.InvalidGraphMetricEdgeFilterMetadata; + const types = try alloc.alloc([]const u8, count); + var initialized: usize = 0; + errdefer { + for (types[0..initialized]) |edge_type| alloc.free(edge_type); + alloc.free(types); + } + for (types) |*slot| { + if (offset + 8 > raw.len) return error.InvalidGraphMetricEdgeFilterMetadata; + const edge_type_len = std.math.cast(usize, std.mem.readInt(u64, raw[offset..][0..8], .little)) orelse return error.InvalidGraphMetricEdgeFilterMetadata; + offset += 8; + if (edge_type_len == 0 or edge_type_len > raw.len - offset) return error.InvalidGraphMetricEdgeFilterMetadata; + slot.* = try alloc.dupe(u8, raw[offset..][0..edge_type_len]); + initialized += 1; + offset += edge_type_len; + } + if (offset != raw.len) return error.InvalidGraphMetricEdgeFilterMetadata; + return .{ .mode = .types, .types = types }; + } + + fn graphMetricBuildLeaseEncodedLen(lease: GraphMetricBuildLease) usize { + return graph_metric_build_lease_header_len + lease.worker_id.len; + } + + fn encodeGraphMetricBuildLease(lease: GraphMetricBuildLease, out: []u8) void { + std.debug.assert(out.len == graphMetricBuildLeaseEncodedLen(lease)); + var offset: usize = 0; + inline for (.{ + @as(u64, 3), + lease.target_generation, + lease.started_at_ms, + lease.lease_expires_at_ms, + @as(u64, @intFromEnum(lease.phase)), + @as(u64, lease.iteration), + lease.job_id, + @as(u64, lease.worker_id.len), + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + @memcpy(out[offset..][0..lease.worker_id.len], lease.worker_id); + } + + fn decodeGraphMetricBuildLease(raw: []const u8) ?GraphMetricBuildLease { + if (raw.len == graph_metric_build_lease_legacy_encoded_len) { + var offset: usize = 0; + const target_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const started_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + return .{ + .job_id = started_at_ms, + .target_generation = target_generation, + .started_at_ms = started_at_ms, + .lease_expires_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little), + .phase = .computing, + .worker_id = graph_metric_local_build_worker_id, + }; + } + if (raw.len < graph_metric_build_lease_v1_header_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1 and version != 2 and version != 3) return null; + const target_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const started_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const lease_expires_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase: GraphMetricBuildPhase = switch (phase_raw) { + @intFromEnum(GraphMetricBuildPhase.idle) => .idle, + @intFromEnum(GraphMetricBuildPhase.computing) => .computing, + @intFromEnum(GraphMetricBuildPhase.publishing) => .publishing, + @intFromEnum(GraphMetricBuildPhase.complete) => .complete, + @intFromEnum(GraphMetricBuildPhase.prepare_generation) => .prepare_generation, + @intFromEnum(GraphMetricBuildPhase.scan_edges_and_out_degree) => .scan_edges_and_out_degree, + @intFromEnum(GraphMetricBuildPhase.initialize_ranks) => .initialize_ranks, + @intFromEnum(GraphMetricBuildPhase.iterate_contributions) => .iterate_contributions, + @intFromEnum(GraphMetricBuildPhase.reduce_ranks) => .reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.hits_hub_contributions) => .hits_hub_contributions, + @intFromEnum(GraphMetricBuildPhase.hits_hub_reduce_ranks) => .hits_hub_reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.check_convergence) => .check_convergence, + @intFromEnum(GraphMetricBuildPhase.publish_generation) => .publish_generation, + @intFromEnum(GraphMetricBuildPhase.cleanup_old_generations) => .cleanup_old_generations, + else => return null, + }; + const iteration: u32 = if (version == 2 or version == 3) blk: { + if (raw.len < graph_metric_build_lease_v2_header_len) return null; + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (value > std.math.maxInt(u32)) return null; + break :blk @intCast(value); + } else 0; + const job_id: u64 = if (version == 3) blk: { + if (raw.len < graph_metric_build_lease_header_len) return null; + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + break :blk value; + } else started_at_ms; + const worker_id_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + if (worker_id_len == 0 or offset + worker_id_len != raw.len) return null; + return .{ + .job_id = job_id, + .target_generation = target_generation, + .started_at_ms = started_at_ms, + .lease_expires_at_ms = lease_expires_at_ms, + .phase = phase, + .iteration = iteration, + .worker_id = raw[offset..][0..worker_id_len], + }; + } + + fn graphMetricBuildJobEncodedLen(job: GraphMetricBuildJob) usize { + return graph_metric_build_job_header_len + job.worker_id.len + job.last_error.len + job.cursor.len; + } + + fn encodeGraphMetricBuildJob(job: GraphMetricBuildJob, out: []u8) void { + std.debug.assert(out.len == graphMetricBuildJobEncodedLen(job)); + var offset: usize = 0; + inline for (.{ + @as(u64, 3), + job.job_id, + job.target_generation, + job.score_generation, + job.started_at_ms, + job.updated_at_ms, + job.lease_expires_at_ms, + @as(u64, @intFromEnum(job.phase)), + @as(u64, job.iteration), + job.retry_count, + job.completed_units, + job.total_units, + @as(u64, job.worker_id.len), + @as(u64, job.last_error.len), + @as(u64, job.cursor.len), + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + @memcpy(out[offset..][0..job.worker_id.len], job.worker_id); + offset += job.worker_id.len; + @memcpy(out[offset..][0..job.last_error.len], job.last_error); + offset += job.last_error.len; + @memcpy(out[offset..][0..job.cursor.len], job.cursor); + } + + fn decodeGraphMetricBuildJob(raw: []const u8) ?GraphMetricBuildJob { + if (raw.len < graph_metric_build_job_v1_header_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1 and version != 2 and version != 3) return null; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const target_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const score_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const started_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const updated_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const lease_expires_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase: GraphMetricBuildPhase = switch (phase_raw) { + @intFromEnum(GraphMetricBuildPhase.idle) => .idle, + @intFromEnum(GraphMetricBuildPhase.computing) => .computing, + @intFromEnum(GraphMetricBuildPhase.publishing) => .publishing, + @intFromEnum(GraphMetricBuildPhase.complete) => .complete, + @intFromEnum(GraphMetricBuildPhase.prepare_generation) => .prepare_generation, + @intFromEnum(GraphMetricBuildPhase.scan_edges_and_out_degree) => .scan_edges_and_out_degree, + @intFromEnum(GraphMetricBuildPhase.initialize_ranks) => .initialize_ranks, + @intFromEnum(GraphMetricBuildPhase.iterate_contributions) => .iterate_contributions, + @intFromEnum(GraphMetricBuildPhase.reduce_ranks) => .reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.hits_hub_contributions) => .hits_hub_contributions, + @intFromEnum(GraphMetricBuildPhase.hits_hub_reduce_ranks) => .hits_hub_reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.check_convergence) => .check_convergence, + @intFromEnum(GraphMetricBuildPhase.publish_generation) => .publish_generation, + @intFromEnum(GraphMetricBuildPhase.cleanup_old_generations) => .cleanup_old_generations, + else => return null, + }; + const iteration_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (iteration_raw > std.math.maxInt(u32)) return null; + const retry_count = if (version >= 2) blk: { + if (raw.len < graph_metric_build_job_v2_header_len) return null; + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + break :blk value; + } else 0; + const completed_units = if (version >= 3) blk: { + if (raw.len < graph_metric_build_job_header_len) return null; + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + break :blk value; + } else 0; + const total_units = if (version >= 3) blk: { + const value = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + break :blk value; + } else 0; + const worker_id_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const last_error_len: usize = if (version >= 2) blk: { + const value: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + break :blk value; + } else 0; + const cursor_len: usize = if (version >= 3) blk: { + const value: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + break :blk value; + } else 0; + if (worker_id_len == 0 or offset + worker_id_len + last_error_len + cursor_len != raw.len) return null; + const worker_id = raw[offset..][0..worker_id_len]; + offset += worker_id_len; + const last_error = raw[offset..][0..last_error_len]; + offset += last_error_len; + return .{ + .job_id = job_id, + .target_generation = target_generation, + .score_generation = score_generation, + .started_at_ms = started_at_ms, + .updated_at_ms = updated_at_ms, + .lease_expires_at_ms = lease_expires_at_ms, + .phase = phase, + .iteration = @intCast(iteration_raw), + .retry_count = retry_count, + .worker_id = worker_id, + .last_error = last_error, + .cursor = raw[offset..][0..cursor_len], + .completed_units = completed_units, + .total_units = total_units, + }; + } + + fn encodeGraphMetricBuildManifest(manifest: GraphMetricBuildManifest, out: *[graph_metric_build_manifest_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + manifest.execution_schema_version, + manifest.job_id, + manifest.target_generation, + manifest.score_generation, + manifest.config_fingerprint, + manifest.planned_at_ms, + manifest.edge_count, + manifest.node_count, + @as(u64, manifest.phase_count), + @as(u64, manifest.page_count), + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn decodeGraphMetricBuildManifest(raw: []const u8) ?GraphMetricBuildManifest { + if (raw.len != graph_metric_build_manifest_encoded_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version == 0 or version > graph_metric_build_execution_schema_version) return null; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const target_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const score_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const config_fingerprint = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const planned_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const edge_count = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const node_count = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_count: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + return .{ + .execution_schema_version = version, + .job_id = job_id, + .target_generation = target_generation, + .score_generation = score_generation, + .config_fingerprint = config_fingerprint, + .planned_at_ms = planned_at_ms, + .edge_count = edge_count, + .node_count = node_count, + .phase_count = phase_count, + .page_count = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)), + }; + } + + fn graphMetricBuildPageEncodedLen(page: GraphMetricBuildPage) usize { + return graph_metric_build_page_header_len + page.worker_id.len + page.cursor.len + page.last_error.len + page.range_lower.len + page.range_upper.len + page.output_prefix.len; + } + + fn encodeGraphMetricBuildPage(page: GraphMetricBuildPage, out: []u8) void { + std.debug.assert(out.len == graphMetricBuildPageEncodedLen(page)); + var offset: usize = 0; + inline for (.{ + @as(u64, 3), + page.job_id, + @as(u64, @intFromEnum(page.phase)), + @as(u64, page.iteration), + page.page_id, + @as(u64, @intFromEnum(page.state)), + page.lease_expires_at_ms, + page.attempt, + page.completed_units, + page.total_units, + page.output_fingerprint, + @as(u64, page.worker_id.len), + @as(u64, page.cursor.len), + @as(u64, page.last_error.len), + @as(u64, @bitCast(page.max_delta)), + @as(u64, @bitCast(page.total_delta)), + @as(u64, @bitCast(page.rank_sum)), + if (page.converged) @as(u64, 1) else @as(u64, 0), + @as(u64, @intFromEnum(page.range_kind)), + @as(u64, page.range_lower.len), + @as(u64, page.range_upper.len), + @as(u64, page.output_prefix.len), + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + @memcpy(out[offset..][0..page.worker_id.len], page.worker_id); + offset += page.worker_id.len; + @memcpy(out[offset..][0..page.cursor.len], page.cursor); + offset += page.cursor.len; + @memcpy(out[offset..][0..page.last_error.len], page.last_error); + offset += page.last_error.len; + @memcpy(out[offset..][0..page.range_lower.len], page.range_lower); + offset += page.range_lower.len; + @memcpy(out[offset..][0..page.range_upper.len], page.range_upper); + offset += page.range_upper.len; + @memcpy(out[offset..][0..page.output_prefix.len], page.output_prefix); + } + + fn decodeGraphMetricBuildPage(raw: []const u8) ?GraphMetricBuildPage { + if (raw.len < graph_metric_build_page_v1_header_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1 and version != 2 and version != 3) return null; + if (version == 2 and raw.len < graph_metric_build_page_v2_header_len) return null; + if (version == 3 and raw.len < graph_metric_build_page_header_len) return null; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase = graphMetricBuildPhaseFromRaw(phase_raw) orelse return null; + const iteration_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (iteration_raw > std.math.maxInt(u32)) return null; + const page_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const state_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const state: GraphMetricBuildPageState = switch (state_raw) { + @intFromEnum(GraphMetricBuildPageState.pending) => .pending, + @intFromEnum(GraphMetricBuildPageState.leased) => .leased, + @intFromEnum(GraphMetricBuildPageState.complete) => .complete, + @intFromEnum(GraphMetricBuildPageState.failed) => .failed, + else => return null, + }; + const lease_expires_at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const attempt = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const completed_units = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const total_units = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const output_fingerprint = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const worker_id_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const cursor_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const last_error_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const max_delta = if (version >= 2) blk: { + const value = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + break :blk value; + } else 0.0; + const total_delta = if (version >= 2) blk: { + const value = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + break :blk value; + } else 0.0; + const rank_sum = if (version >= 2) blk: { + const value = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + break :blk value; + } else 0.0; + const converged = if (version >= 2) blk: { + const value = std.mem.readInt(u64, raw[offset..][0..8], .little) != 0; + offset += 8; + break :blk value; + } else false; + const range_fields = if (version >= 3) blk: { + const range_kind_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const decoded_range_kind: GraphMetricBuildPageRangeKind = switch (range_kind_raw) { + @intFromEnum(GraphMetricBuildPageRangeKind.full) => .full, + @intFromEnum(GraphMetricBuildPageRangeKind.reverse_edges) => .reverse_edges, + @intFromEnum(GraphMetricBuildPageRangeKind.nodes) => .nodes, + @intFromEnum(GraphMetricBuildPageRangeKind.scores) => .scores, + @intFromEnum(GraphMetricBuildPageRangeKind.contributions) => .contributions, + @intFromEnum(GraphMetricBuildPageRangeKind.job_control) => .job_control, + @intFromEnum(GraphMetricBuildPageRangeKind.summary) => .summary, + else => return null, + }; + const decoded_range_lower_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const decoded_range_upper_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + const decoded_output_prefix_len: usize = @intCast(std.mem.readInt(u64, raw[offset..][0..8], .little)); + offset += 8; + break :blk .{ decoded_range_kind, decoded_range_lower_len, decoded_range_upper_len, decoded_output_prefix_len }; + } else .{ GraphMetricBuildPageRangeKind.full, @as(usize, 0), @as(usize, 0), @as(usize, 0) }; + const range_kind = range_fields[0]; + const range_lower_len = range_fields[1]; + const range_upper_len = range_fields[2]; + const output_prefix_len = range_fields[3]; + if (offset + worker_id_len + cursor_len + last_error_len + range_lower_len + range_upper_len + output_prefix_len != raw.len) return null; + const worker_id = raw[offset..][0..worker_id_len]; + offset += worker_id_len; + const cursor = raw[offset..][0..cursor_len]; + offset += cursor_len; + const last_error = raw[offset..][0..last_error_len]; + offset += last_error_len; + const range_lower = raw[offset..][0..range_lower_len]; + offset += range_lower_len; + const range_upper = raw[offset..][0..range_upper_len]; + offset += range_upper_len; + const output_prefix = raw[offset..][0..output_prefix_len]; + return .{ + .job_id = job_id, + .phase = phase, + .iteration = @intCast(iteration_raw), + .page_id = page_id, + .state = state, + .range_kind = range_kind, + .range_lower = range_lower, + .range_upper = range_upper, + .output_prefix = output_prefix, + .worker_id = worker_id, + .lease_expires_at_ms = lease_expires_at_ms, + .attempt = attempt, + .cursor = cursor, + .completed_units = completed_units, + .total_units = total_units, + .last_error = last_error, + .output_fingerprint = output_fingerprint, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .converged = converged, + }; + } + + fn encodeGraphMetricBuildPhaseSummary(summary: GraphMetricBuildPhaseSummary, out: *[graph_metric_build_phase_summary_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + @as(u64, 1), + summary.job_id, + @as(u64, @intFromEnum(summary.phase)), + @as(u64, summary.iteration), + @as(u64, @intFromEnum(summary.state)), + summary.expected_pages, + summary.completed_pages, + summary.failed_pages, + summary.completed_units, + summary.total_units, + @as(u64, @bitCast(summary.max_delta)), + @as(u64, @bitCast(summary.total_delta)), + @as(u64, @bitCast(summary.rank_sum)), + if (summary.converged) @as(u64, 1) else @as(u64, 0), + summary.output_fingerprint, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn encodeGraphMetricBuildPhaseProgress(progress: GraphMetricBuildPhaseProgress, out: *[graph_metric_build_phase_progress_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + @as(u64, 1), + progress.job_id, + @as(u64, @intFromEnum(progress.phase)), + @as(u64, progress.iteration), + progress.expected_pages, + progress.pending_pages, + progress.leased_pages, + progress.completed_pages, + progress.failed_pages, + progress.max_attempt_pages, + progress.completed_units, + progress.total_units, + progress.next_claim_page_id, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn decodeGraphMetricBuildPhaseProgress(raw: []const u8) ?GraphMetricBuildPhaseProgress { + if (raw.len != graph_metric_build_phase_progress_encoded_len) return null; + var values: [13]u64 = undefined; + for (&values, 0..) |*value, i| value.* = std.mem.readInt(u64, raw[i * 8 ..][0..8], .little); + if (values[0] != 1 or values[3] > std.math.maxInt(u32)) return null; + const phase = graphMetricBuildPhaseFromRaw(values[2]) orelse return null; + const state_pages = std.math.add(u64, values[5], values[6]) catch return null; + const non_terminal_pages = std.math.add(u64, state_pages, values[8]) catch return null; + const all_pages = std.math.add(u64, non_terminal_pages, values[7]) catch return null; + if (all_pages != values[4] or values[7] > values[4] or values[9] > values[4] or values[10] > values[11]) return null; + return .{ + .job_id = values[1], + .phase = phase, + .iteration = @intCast(values[3]), + .expected_pages = values[4], + .pending_pages = values[5], + .leased_pages = values[6], + .completed_pages = values[7], + .failed_pages = values[8], + .max_attempt_pages = values[9], + .completed_units = values[10], + .total_units = values[11], + .next_claim_page_id = values[12], + }; + } + + fn decodeGraphMetricBuildPhaseSummary(raw: []const u8) ?GraphMetricBuildPhaseSummary { + if (raw.len != graph_metric_build_phase_summary_encoded_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1) return null; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase = graphMetricBuildPhaseFromRaw(phase_raw) orelse return null; + const iteration_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (iteration_raw > std.math.maxInt(u32)) return null; + const state_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const state: GraphMetricBuildPhaseState = switch (state_raw) { + @intFromEnum(GraphMetricBuildPhaseState.pending) => .pending, + @intFromEnum(GraphMetricBuildPhaseState.complete) => .complete, + @intFromEnum(GraphMetricBuildPhaseState.failed) => .failed, + else => return null, + }; + const expected_pages = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const completed_pages = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const failed_pages = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const completed_units = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const total_units = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const max_delta = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const total_delta = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const rank_sum = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const converged = std.mem.readInt(u64, raw[offset..][0..8], .little) != 0; + offset += 8; + return .{ + .job_id = job_id, + .phase = phase, + .iteration = @intCast(iteration_raw), + .state = state, + .expected_pages = expected_pages, + .completed_pages = completed_pages, + .failed_pages = failed_pages, + .completed_units = completed_units, + .total_units = total_units, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .converged = converged, + .output_fingerprint = std.mem.readInt(u64, raw[offset..][0..8], .little), + }; + } + + fn encodeGraphMetricBuildIterationSummary(summary: GraphMetricBuildIterationSummary, out: *[graph_metric_build_iteration_summary_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + @as(u64, 1), + summary.job_id, + @as(u64, summary.iteration), + summary.expected_pages, + summary.completed_pages, + @as(u64, @bitCast(summary.max_delta)), + @as(u64, @bitCast(summary.total_delta)), + @as(u64, @bitCast(summary.rank_sum)), + if (summary.converged) @as(u64, 1) else @as(u64, 0), + if (summary.fixed_iteration_limit) @as(u64, 1) else @as(u64, 0), + summary.output_fingerprint, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn decodeGraphMetricBuildIterationSummary(raw: []const u8) ?GraphMetricBuildIterationSummary { + if (raw.len != graph_metric_build_iteration_summary_encoded_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1) return null; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const iteration_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (iteration_raw > std.math.maxInt(u32)) return null; + const expected_pages = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const completed_pages = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const max_delta = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const total_delta = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const rank_sum = @as(f64, @bitCast(std.mem.readInt(u64, raw[offset..][0..8], .little))); + offset += 8; + const converged = std.mem.readInt(u64, raw[offset..][0..8], .little) != 0; + offset += 8; + const fixed_iteration_limit = std.mem.readInt(u64, raw[offset..][0..8], .little) != 0; + offset += 8; + return .{ + .job_id = job_id, + .iteration = @intCast(iteration_raw), + .expected_pages = expected_pages, + .completed_pages = completed_pages, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .converged = converged, + .fixed_iteration_limit = fixed_iteration_limit, + .output_fingerprint = std.mem.readInt(u64, raw[offset..][0..8], .little), + }; + } + + fn encodeGraphMetricFailureDetail(detail: GraphMetricFailureDetail, out: []u8) void { + std.debug.assert(out.len == graph_metric_failure_detail_header_len + detail.last_error.len); + std.mem.writeInt(u64, out[0..8], detail.retry_count, .little); + @memcpy(out[graph_metric_failure_detail_header_len..], detail.last_error); + } + + fn decodeGraphMetricFailureDetailAlloc(alloc: Allocator, raw: []const u8) !?GraphMetricFailureDetail { + if (raw.len < graph_metric_failure_detail_header_len) return null; + const retry_count = std.mem.readInt(u64, raw[0..8], .little); + const last_error = try alloc.dupe(u8, raw[graph_metric_failure_detail_header_len..]); + return .{ .retry_count = retry_count, .last_error = last_error }; + } + + fn encodeGraphMetricFailureRecord(record: GraphMetricFailureRecord, out: []u8) void { + std.debug.assert(out.len == graph_metric_failure_record_header_len + record.last_error.len); + var offset: usize = 0; + inline for (.{ + @as(u64, 1), + record.at_ms, + record.job_id, + record.target_generation, + record.score_generation, + @as(u64, @intFromEnum(record.phase)), + @as(u64, record.iteration), + record.retry_count, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + @memcpy(out[offset..], record.last_error); + } + + fn decodeGraphMetricFailureRecordAlloc(alloc: Allocator, sequence: u64, raw: []const u8) !?GraphMetricFailureRecord { + if (raw.len < graph_metric_failure_record_header_len) return null; + var offset: usize = 0; + const version = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (version != 1) return null; + const at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const job_id = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const target_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const score_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const phase = graphMetricBuildPhaseFromRaw(phase_raw) orelse return null; + const iteration_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + if (iteration_raw > std.math.maxInt(u32)) return null; + const retry_count = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const last_error = try alloc.dupe(u8, raw[offset..]); + return .{ + .sequence = sequence, + .at_ms = at_ms, + .job_id = job_id, + .target_generation = target_generation, + .score_generation = score_generation, + .phase = phase, + .iteration = @intCast(iteration_raw), + .retry_count = retry_count, + .last_error = last_error, + }; + } + + fn encodeGraphMetricEvent(event: GraphMetricEvent, out: *[graph_metric_event_encoded_len]u8) void { + var offset: usize = 0; + inline for (.{ + @as(u64, @intFromEnum(event.kind)), + event.at_ms, + event.target_edge_generation, + event.published_generation, + event.score_count, + }) |value| { + std.mem.writeInt(u64, out[offset..][0..8], value, .little); + offset += 8; + } + } + + fn decodeGraphMetricEvent(sequence: u64, raw: []const u8) ?GraphMetricEvent { + if (raw.len != graph_metric_event_encoded_len) return null; + var offset: usize = 0; + const kind_raw = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const kind: GraphMetricEventKind = switch (kind_raw) { + @intFromEnum(GraphMetricEventKind.publish) => .publish, + @intFromEnum(GraphMetricEventKind.delete) => .delete, + @intFromEnum(GraphMetricEventKind.pause) => .pause, + @intFromEnum(GraphMetricEventKind.@"resume") => .@"resume", + @intFromEnum(GraphMetricEventKind.failed) => .failed, + else => return null, + }; + const at_ms = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const target_edge_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + const published_generation = std.mem.readInt(u64, raw[offset..][0..8], .little); + offset += 8; + return .{ + .sequence = sequence, + .kind = kind, + .at_ms = at_ms, + .target_edge_generation = target_edge_generation, + .published_generation = published_generation, + .score_count = std.mem.readInt(u64, raw[offset..][0..8], .little), + }; + } + + fn putF64(batch: anytype, key: []const u8, value: f64) !void { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, @bitCast(value), .little); + try batch.put(key, &buf); + } + + fn decodeF64(raw: []const u8) ?f64 { + if (raw.len != 8) return null; + const bits = std.mem.readInt(u64, raw[0..8], .little); + return @bitCast(bits); + } + + fn readF64OrZero(txn: anytype, key: []const u8) !f64 { + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return 0.0, + else => return err, + }; + return decodeF64(raw) orelse error.InvalidGraphMetricScore; + } + + fn readOptionalF64(txn: anytype, key: []const u8) !?f64 { + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeF64(raw) orelse error.InvalidGraphMetricScore; + } + + /// Reads a materialized score that must exist before publication. Missing + /// accumulator inputs legitimately mean zero while a build is in flight; + /// a missing final rank instead means the generation is incomplete and + /// must never be made visible to readers. + fn readRequiredFiniteF64(txn: anytype, key: []const u8) !f64 { + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricScore, + else => return err, + }; + const value = decodeF64(raw) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + return value; + } + + fn metricConfig(self: *const GraphIndex, metric_name: []const u8) ?GraphMetricConfig { + for (self.metric_configs) |cfg| { + if (std.mem.eql(u8, cfg.name, metric_name)) return cfg; + } + return null; + } + + fn graphMetricEdgeAllowed(filter: GraphMetricEdgeFilter, edge_type: []const u8) bool { + return switch (filter.mode) { + .all => true, + .types => filter.includesType(edge_type), + }; + } + + const CompiledGraphMetricEdgeFilter = struct { + all: bool, + small_types: []const []const u8 = &.{}, + types: std.StringHashMapUnmanaged(void) = .empty, + + fn init(alloc: Allocator, filter: GraphMetricEdgeFilter) !@This() { + var result = @This(){ .all = filter.mode == .all }; + errdefer result.deinit(alloc); + if (!result.all) { + // Tiny filters dominate normal schemas and are cheaper as a + // handful of byte comparisons than a full string hash. Larger + // filters pay one page-local compilation and get O(1) probes. + if (filter.types.len <= 4) { + result.small_types = filter.types; + } else { + try result.types.ensureTotalCapacity(alloc, @intCast(filter.types.len)); + for (filter.types) |edge_type| result.types.putAssumeCapacity(edge_type, {}); + } + } + return result; + } + + fn deinit(self: *@This(), alloc: Allocator) void { + self.types.deinit(alloc); + self.* = undefined; + } + + fn allows(self: @This(), edge_type: []const u8) bool { + if (self.all) return true; + for (self.small_types) |allowed| if (std.mem.eql(u8, allowed, edge_type)) return true; + return self.types.contains(edge_type); + } + }; + + fn graphMetricEdgeFiltersEqual(a: GraphMetricEdgeFilter, b: GraphMetricEdgeFilter) bool { + return a.equivalent(b); + } + + fn graphMetricConfigFingerprintHashU64(hasher: *std.hash.Wyhash, value: u64) void { + var raw = value; + hasher.update(std.mem.asBytes(&raw)); + } + + fn graphMetricConfigFingerprint(cfg: GraphMetricConfig) u64 { + var hasher = std.hash.Wyhash.init(0); + graphMetricConfigFingerprintHashU64(&hasher, @intFromEnum(cfg.kind)); + graphMetricConfigFingerprintHashU64(&hasher, @as(u64, @bitCast(cfg.damping))); + graphMetricConfigFingerprintHashU64(&hasher, @as(u64, @bitCast(cfg.tolerance))); + graphMetricConfigFingerprintHashU64(&hasher, cfg.max_iterations); + graphMetricConfigFingerprintHashU64(&hasher, @intFromEnum(cfg.edge_filter.mode)); + graphMetricConfigFingerprintHashU64(&hasher, cfg.edge_filter.types.len); + var last: ?[]const u8 = null; + var emitted: usize = 0; + while (emitted < cfg.edge_filter.types.len) { + const edge_type = nextGraphMetricEdgeFilterType(cfg.edge_filter.types, last) orelse break; + graphMetricConfigFingerprintHashU64(&hasher, edge_type.len); + hasher.update(edge_type); + last = edge_type; + emitted += 1; + } + // Public OpenAPI and remote-wire status uses signed 64-bit integers. + // Keep the durable identity in the positive i64 domain so adapters do + // not truncate, saturate, or reinterpret an otherwise valid hash. + const fingerprint = hasher.final() & std.math.maxInt(i64); + return if (fingerprint == 0) 1 else fingerprint; + } + + fn nextGraphMetricEdgeFilterType(types: []const []const u8, last: ?[]const u8) ?[]const u8 { + var next: ?[]const u8 = null; + for (types) |edge_type| { + if (last) |prior| { + if (std.mem.order(u8, edge_type, prior) != .gt) continue; + } + if (next == null or std.mem.lessThan(u8, edge_type, next.?)) { + next = edge_type; + } + } + return next; + } + + fn graphMetricScoresFingerprint(scores: []const GraphMetricScore) u64 { + var hasher = std.hash.Wyhash.init(0); + graphMetricConfigFingerprintHashU64(&hasher, scores.len); + for (scores) |score| { + graphMetricConfigFingerprintHashU64(&hasher, score.node.len); + hasher.update(score.node); + graphMetricConfigFingerprintHashU64(&hasher, @as(u64, @bitCast(score.score))); + } + return hasher.final(); + } + + fn graphMetricBuildPhaseFromRaw(phase_raw: u64) ?GraphMetricBuildPhase { + return switch (phase_raw) { + @intFromEnum(GraphMetricBuildPhase.idle) => .idle, + @intFromEnum(GraphMetricBuildPhase.computing) => .computing, + @intFromEnum(GraphMetricBuildPhase.publishing) => .publishing, + @intFromEnum(GraphMetricBuildPhase.complete) => .complete, + @intFromEnum(GraphMetricBuildPhase.prepare_generation) => .prepare_generation, + @intFromEnum(GraphMetricBuildPhase.scan_edges_and_out_degree) => .scan_edges_and_out_degree, + @intFromEnum(GraphMetricBuildPhase.initialize_ranks) => .initialize_ranks, + @intFromEnum(GraphMetricBuildPhase.iterate_contributions) => .iterate_contributions, + @intFromEnum(GraphMetricBuildPhase.reduce_ranks) => .reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.hits_hub_contributions) => .hits_hub_contributions, + @intFromEnum(GraphMetricBuildPhase.hits_hub_reduce_ranks) => .hits_hub_reduce_ranks, + @intFromEnum(GraphMetricBuildPhase.check_convergence) => .check_convergence, + @intFromEnum(GraphMetricBuildPhase.publish_generation) => .publish_generation, + @intFromEnum(GraphMetricBuildPhase.cleanup_old_generations) => .cleanup_old_generations, + else => null, + }; + } + + fn graphMetricBuildManifestPhases(kind: GraphMetricKind) []const GraphMetricBuildPhase { + return switch (kind) { + .degree => graph_metric_degree_build_phases[0..], + .pagerank, .eigenvector => graph_metric_iterative_build_phases[0..], + .hits_authority, .hits_hub => graph_metric_hits_build_phases[0..], + }; + } + + fn graphMetricKindIsIterative(kind: GraphMetricKind) bool { + return switch (kind) { + .pagerank, .eigenvector, .hits_authority, .hits_hub => true, + .degree => false, + }; + } + + fn graphMetricBuildManifestNextPhase(kind: GraphMetricKind, phase: GraphMetricBuildPhase) ?GraphMetricBuildPhase { + const phases = graphMetricBuildManifestPhases(kind); + for (phases, 0..) |candidate, i| { + if (candidate != phase) continue; + if (i + 1 >= phases.len) return null; + return phases[i + 1]; + } + return null; + } + + fn graphMetricBuildManifestPhaseUnits(phase: GraphMetricBuildPhase, edge_count: u64, node_count: u64) u64 { + return switch (phase) { + .scan_edges_and_out_degree, .iterate_contributions, .hits_hub_contributions => edge_count, + .initialize_ranks, .reduce_ranks, .hits_hub_reduce_ranks, .check_convergence, .publish_generation => node_count, + .prepare_generation, .cleanup_old_generations => 1, + .idle, .computing, .publishing, .complete => 0, + }; + } + + fn graphMetricBuildManifestPhaseRangeKind(phase: GraphMetricBuildPhase) GraphMetricBuildPageRangeKind { + return switch (phase) { + .scan_edges_and_out_degree, .iterate_contributions, .hits_hub_contributions => .reverse_edges, + .initialize_ranks, .reduce_ranks, .hits_hub_reduce_ranks, .check_convergence, .publish_generation => .nodes, + .cleanup_old_generations => .job_control, + .prepare_generation, .idle, .computing, .publishing, .complete => .full, + }; + } + + fn graphMetricBuildJobId(metric_name: []const u8, target_generation: u64, started_at_ms: u64) u64 { + var hasher = std.hash.Wyhash.init(0xA17F_6D3B_2C91_5E44); + hasher.update(metric_name); + graphMetricConfigFingerprintHashU64(&hasher, target_generation); + graphMetricConfigFingerprintHashU64(&hasher, started_at_ms); + const id = hasher.final(); + return if (id == 0) 1 else id; + } + + fn graphMetricActiveBuildProgress(cfg: GraphMetricConfig, phase: GraphMetricBuildPhase, iteration: u32, phase_fraction_raw: f64) f64 { + const phase_fraction = @min(1.0, @max(0.0, phase_fraction_raw)); + if (cfg.kind == .degree) return switch (phase) { + .idle => 0.0, + .prepare_generation => 0.01, + .scan_edges_and_out_degree => 0.01 + 0.44 * phase_fraction, + .initialize_ranks => 0.45, + .reduce_ranks => 0.45 + 0.5 * phase_fraction, + .computing => blk: { + if (iteration == 0 or cfg.max_iterations == 0) break :blk 0.01; + const progress = @as(f64, @floatFromInt(iteration)) / @as(f64, @floatFromInt(cfg.max_iterations)); + break :blk @min(0.95, @max(0.01, progress)); + }, + .iterate_contributions, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence => 0.95, + .publishing, .publish_generation => 0.95 + 0.03 * phase_fraction, + .cleanup_old_generations => 0.98 + 0.02 * phase_fraction, + .complete => 1.0, + }; + return switch (phase) { + .idle => 0.0, + .prepare_generation => 0.01, + .scan_edges_and_out_degree => 0.01 + 0.07 * phase_fraction, + .initialize_ranks => 0.08 + 0.02 * phase_fraction, + .computing => blk: { + if (iteration == 0 or cfg.max_iterations == 0) break :blk 0.01; + const progress = @as(f64, @floatFromInt(iteration)) / @as(f64, @floatFromInt(cfg.max_iterations)); + break :blk @min(0.96, @max(0.1, 0.1 + 0.86 * progress)); + }, + .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence => blk: { + if (cfg.max_iterations == 0) break :blk 0.96; + const hits = cfg.kind == .hits_authority or cfg.kind == .hits_hub; + const step_count: f64 = if (iteration == 0) (if (hits) @as(f64, 5.0) else 3.0) else (if (hits) @as(f64, 3.0) else 2.0); + const step: f64 = switch (phase) { + .iterate_contributions => 0.0, + .reduce_ranks => if (iteration == 0) 1.0 else 0.0, + .hits_hub_contributions => 2.0, + .hits_hub_reduce_ranks => if (iteration == 0) 3.0 else 1.0, + .check_convergence => step_count - 1.0, + else => unreachable, + }; + const iteration_progress = @as(f64, @floatFromInt(iteration)) + (step + phase_fraction) / step_count; + const all_iterations = @as(f64, @floatFromInt(cfg.max_iterations)); + break :blk @min(0.96, 0.1 + 0.86 * iteration_progress / all_iterations); + }, + .publishing, .publish_generation => 0.96 + 0.02 * phase_fraction, + .cleanup_old_generations => 0.98 + 0.02 * phase_fraction, + .complete => 1.0, + }; + } + + const PageRankNode = struct { + key: []u8, + }; + + const PageRankEdge = metric_kernels.Edge; + + const PageRankInitializeNode = struct { + node: []const u8, + slot: u64 = 0, + out_degree: u64 = 0, + initial_rank: ?f64 = null, + secondary_rank: ?f64 = null, + }; + + const DegreeNode = struct { + key: []u8, + degree: u64 = 0, + }; + + fn getOrPutPageRankNode( + self: *GraphIndex, + map: *std.StringHashMapUnmanaged(usize), + nodes: *std.ArrayListUnmanaged(PageRankNode), + key: []const u8, + ) !usize { + if (map.get(key)) |idx| return idx; + const owned = try self.alloc.dupe(u8, key); + errdefer self.alloc.free(owned); + const idx = nodes.items.len; + try nodes.append(self.alloc, .{ .key = owned }); + errdefer _ = nodes.pop(); + try map.put(self.alloc, owned, idx); + return idx; + } + + fn getOrPutDegreeNode( + self: *GraphIndex, + map: *std.StringHashMapUnmanaged(usize), + nodes: *std.ArrayListUnmanaged(DegreeNode), + key: []const u8, + ) !usize { + if (map.get(key)) |idx| return idx; + const owned = try self.alloc.dupe(u8, key); + errdefer self.alloc.free(owned); + const idx = nodes.items.len; + try nodes.append(self.alloc, .{ .key = owned }); + errdefer _ = nodes.pop(); + try map.put(self.alloc, owned, idx); + return idx; + } + + fn freePageRankNodes(self: *GraphIndex, nodes: []PageRankNode) void { + for (nodes) |node| self.alloc.free(node.key); + } + + fn freeDegreeNodes(self: *GraphIndex, nodes: []DegreeNode) void { + for (nodes) |node| self.alloc.free(node.key); + } + + fn collectPageRankGraph( + self: *GraphIndex, + filter: GraphMetricEdgeFilter, + nodes: *std.ArrayListUnmanaged(PageRankNode), + edges: *std.ArrayListUnmanaged(PageRankEdge), + ) !void { + var map = std.StringHashMapUnmanaged(usize).empty; + defer map.deinit(self.alloc); + + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(self.alloc); + if (!graphMetricEdgeAllowed(filter, parsed.edge_type.bytes)) continue; + const source_idx = try self.getOrPutPageRankNode(&map, nodes, parsed.source.bytes); + const target_idx = try self.getOrPutPageRankNode(&map, nodes, parsed.target.bytes); + if (source_idx > std.math.maxInt(u32) or target_idx > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + try edges.append(self.alloc, .{ .source = @intCast(source_idx), .target = @intCast(target_idx) }); + } + } + + fn collectDegreeGraph( + self: *GraphIndex, + filter: GraphMetricEdgeFilter, + nodes: *std.ArrayListUnmanaged(DegreeNode), + ) !void { + var map = std.StringHashMapUnmanaged(usize).empty; + defer map.deinit(self.alloc); + + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(self.alloc); + if (!graphMetricEdgeAllowed(filter, parsed.edge_type.bytes)) continue; + const source_idx = try self.getOrPutDegreeNode(&map, nodes, parsed.source.bytes); + const target_idx = try self.getOrPutDegreeNode(&map, nodes, parsed.target.bytes); + nodes.items[source_idx].degree += 1; + nodes.items[target_idx].degree += 1; + } + } + + fn putPageRankScanNode( + self: *GraphIndex, + nodes: *std.StringHashMapUnmanaged(void), + node: []const u8, + ) !void { + const result = try self.getOrPutOwnedStringMap(void, nodes, node); + if (result.found_existing) return; + result.value_ptr.* = {}; + } + + fn getOrPutOwnedStringMap( + self: *GraphIndex, + comptime Value: type, + map: *std.StringHashMapUnmanaged(Value), + key: []const u8, + ) !struct { key: []const u8, value_ptr: *Value, found_existing: bool } { + if (map.getPtr(key)) |value_ptr| return .{ + .key = map.getKey(key).?, + .value_ptr = value_ptr, + .found_existing = true, + }; + + const owned_key = try self.alloc.dupe(u8, key); + errdefer self.alloc.free(owned_key); + try map.putNoClobber(self.alloc, owned_key, undefined); + return .{ + .key = map.getKey(owned_key).?, + .value_ptr = map.getPtr(owned_key).?, + .found_existing = false, + }; + } + + fn replaceOwnedBytes(self: *GraphIndex, target: *[]u8, value: []const u8) !void { + const replacement = try self.alloc.dupe(u8, value); + if (target.*.len > 0) self.alloc.free(target.*); + target.* = replacement; + } + + fn appendOwnedBytes(self: *GraphIndex, list: *std.ArrayListUnmanaged([]u8), value: []const u8) !void { + const owned = try self.alloc.dupe(u8, value); + errdefer self.alloc.free(owned); + try list.append(self.alloc, owned); + } + + fn freeStringHashMapKeys(self: *GraphIndex, comptime V: type, map: *std.StringHashMapUnmanaged(V)) void { + var key_it = map.keyIterator(); + while (key_it.next()) |key_ptr| self.alloc.free(key_ptr.*); + map.deinit(self.alloc); + } + + fn graphMetricPageRankScanFingerprint(page: GraphMetricBuildPage, scanned_units: u64, node_count: usize, out_degree_total: u64) u64 { + var hasher = std.hash.Wyhash.init(0xD392_8A6D_7C51_10EF); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, scanned_units); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(node_count)); + graphMetricConfigFingerprintHashU64(&hasher, out_degree_total); + return hasher.final(); + } + + fn graphMetricPageRankInitializeFingerprint(page: GraphMetricBuildPage, node_count: usize, out_degree_total: u64, rank_sum: f64) u64 { + var hasher = std.hash.Wyhash.init(0x69D0_CAA5_410C_F81B); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(node_count)); + graphMetricConfigFingerprintHashU64(&hasher, out_degree_total); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(rank_sum)); + return hasher.final(); + } + + fn graphMetricPageRankContributionFingerprint(page: GraphMetricBuildPage, scanned_units: u64, target_count: usize, contribution_sum: f64) u64 { + var hasher = std.hash.Wyhash.init(0xA755_7F0A_3E41_C2BD); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, scanned_units); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(target_count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(contribution_sum)); + return hasher.final(); + } + + fn graphMetricPageRankReduceFingerprint(page: GraphMetricBuildPage, node_count: usize, contribution_sum: f64, rank_sum: f64) u64 { + var hasher = std.hash.Wyhash.init(0xE50F_0C79_924B_A365); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(node_count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(contribution_sum)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(rank_sum)); + return hasher.final(); + } + + fn graphMetricPageRankConvergenceFingerprint(page: GraphMetricBuildPage, node_count: usize, max_delta: f64, total_delta: f64, rank_sum: f64) u64 { + var hasher = std.hash.Wyhash.init(0xC7AE_4085_6E22_A91F); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(node_count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(max_delta)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(total_delta)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(rank_sum)); + return hasher.final(); + } + + fn graphMetricHitsHubRawEntryFingerprint(node: []const u8, raw_hub: f64) u64 { + var hasher = std.hash.Wyhash.init(0x4E43_C711_D637_AE19); + graphMetricConfigFingerprintHashU64(&hasher, node.len); + hasher.update(node); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(raw_hub)); + const fingerprint = hasher.final(); + return if (fingerprint == 0) 1 else fingerprint; + } + + fn graphMetricHitsHubRawSummaryFingerprint(metric_name: []const u8, cfg: GraphMetricConfig, job_id: u64, iteration: u32, authority_norm: f64, count: usize, hub_norm: f64, raw_fingerprint: u64) u64 { + var hasher = std.hash.Wyhash.init(0xB56A_0FA2_C4DD_7189); + graphMetricConfigFingerprintHashU64(&hasher, metric_name.len); + hasher.update(metric_name); + graphMetricConfigFingerprintHashU64(&hasher, graphMetricConfigFingerprint(cfg)); + graphMetricConfigFingerprintHashU64(&hasher, job_id); + graphMetricConfigFingerprintHashU64(&hasher, iteration); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(authority_norm)); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(hub_norm)); + graphMetricConfigFingerprintHashU64(&hasher, raw_fingerprint); + const fingerprint = hasher.final(); + return if (fingerprint == 0) 1 else fingerprint; + } + + fn graphMetricHitsReduceFingerprint(page: GraphMetricBuildPage, node_count: usize, contribution_sum: f64, rank_sum: f64, hub_summary: HitsHubRawSummary) u64 { + var hasher = std.hash.Wyhash.init(0x8ECA_74E5_5BB7_0D31); + graphMetricConfigFingerprintHashU64(&hasher, page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(node_count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(contribution_sum)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(rank_sum)); + graphMetricConfigFingerprintHashU64(&hasher, @intCast(hub_summary.count)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(hub_summary.norm)); + graphMetricConfigFingerprintHashU64(&hasher, hub_summary.fingerprint); + const fingerprint = hasher.final(); + return if (fingerprint == 0) 1 else fingerprint; + } + + fn collectPageRankScannedNodes( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + out: *std.ArrayListUnmanaged([]u8), + ) !void { + _ = try self.collectPageRankScannedNodesInRange(txn, metric_name, job_id, "", "", "", null, out); + } + + /// Enumerates the canonical node-partial index directly. The returned + /// boolean says whether the requested range was exhausted; false means a + /// caller-supplied limit stopped the scan and the last returned node is a + /// durable resume cursor. + fn collectPageRankScannedNodesInRange( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + range_lower: []const u8, + range_upper: []const u8, + resume_cursor: []const u8, + max_nodes: ?usize, + out: *std.ArrayListUnmanaged([]u8), + ) !bool { + _ = try self.metricBuildManifest(txn, metric_name, job_id) orelse return error.GraphMetricBuildManifestNotFound; + const prefix = try self.graphMetricBuildPageRankNodePartialPrefixAlloc(metric_name, job_id); + defer self.alloc.free(prefix); + const seek_node = if (resume_cursor.len > 0) resume_cursor else range_lower; + const seek_key = if (seek_node.len > 0) + try self.graphMetricBuildPageRankNodePartialKeyAlloc(metric_name, job_id, seek_node, 0) + else + try self.alloc.dupe(u8, prefix); + defer self.alloc.free(seek_key); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(seek_key); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (entry.value.len != 8 or std.mem.readInt(u64, entry.value[0..8], .little) != 1) { + return error.InvalidGraphMetricBuildManifest; + } + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(node); + if (range_lower.len > 0 and std.mem.order(u8, node, range_lower) == .lt) { + self.alloc.free(node); + continue; + } + if (resume_cursor.len > 0 and std.mem.order(u8, node, resume_cursor) != .gt) { + self.alloc.free(node); + continue; + } + if (range_upper.len > 0 and std.mem.order(u8, node, range_upper) != .lt) { + self.alloc.free(node); + return true; + } + if (out.items.len > 0 and std.mem.eql(u8, out.items[out.items.len - 1], node)) { + self.alloc.free(node); + continue; + } + if (max_nodes) |limit| { + if (out.items.len >= limit) { + self.alloc.free(node); + return false; + } + } + try out.append(self.alloc, node); + } + return true; + } + + /// Enumerates unique nodes in the canonical degree-partial namespace. + /// The namespace is ordered by node and then source page, so duplicate + /// partials for the same node remain adjacent and can be counted without + /// retaining a graph-sized set in memory. + fn collectDegreePartialNodesInRange( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + range_lower: []const u8, + range_upper: []const u8, + resume_cursor: []const u8, + max_nodes: ?usize, + out: *std.ArrayListUnmanaged([]u8), + ) !bool { + _ = try self.metricBuildManifest(txn, metric_name, job_id) orelse return error.GraphMetricBuildManifestNotFound; + const prefix = try self.graphMetricBuildDegreePartialPrefixAlloc(metric_name, job_id); + defer self.alloc.free(prefix); + const start = if (resume_cursor.len > 0) resume_cursor else range_lower; + const seek_key = if (start.len > 0) + try self.graphMetricBuildDegreePartialKeyAlloc(metric_name, job_id, start, 0) + else + try self.alloc.dupe(u8, prefix); + defer self.alloc.free(seek_key); + + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(seek_key); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (entry.value.len != 8 or std.mem.readInt(u64, entry.value[0..8], .little) == 0) { + return error.InvalidGraphMetricBuildManifest; + } + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(node); + if (range_upper.len > 0 and std.mem.order(u8, node, range_upper) != .lt) { + self.alloc.free(node); + break; + } + if (resume_cursor.len > 0 and std.mem.order(u8, node, resume_cursor) != .gt) { + self.alloc.free(node); + continue; + } + if (out.items.len > 0 and std.mem.eql(u8, out.items[out.items.len - 1], node)) { + self.alloc.free(node); + continue; + } + if (max_nodes) |limit| { + if (out.items.len >= limit) { + self.alloc.free(node); + return false; + } + } + try out.append(self.alloc, node); + } + return true; + } + + fn aggregatePageRankOutDegreeForNode( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + node: []const u8, + ) !u64 { + const total_key = try self.topologyKey(txn, metric_name, job_id, try self.graphMetricBuildPageRankOutDegreeKeyAlloc(metric_name, job_id, node)); + defer self.alloc.free(total_key); + return try readU64OrZero(txn, total_key); + } + + fn aggregatePageRankContributionForNode( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) !f64 { + const prefix = try self.graphMetricBuildPageRankContributionNodePrefixAlloc(metric_name, job_id, iteration, node); + defer self.alloc.free(prefix); + var sum: f64 = 0.0; + var correction: f64 = 0.0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const shard = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(shard)) return error.InvalidGraphMetricScore; + const next = sum + shard; + correction += if (@abs(sum) >= @abs(shard)) (sum - next) + shard else (shard - next) + sum; + sum = next; + } + const value = sum + correction; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + return value; + } + + fn pageRankContributionsForNodesAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + nodes: []const []const u8, + ) ![]f64 { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + return self.readGraphMetricVectorAlloc(txn, metric_name, job_id, "raw_rank", iteration, nodes, true); + } + + fn hitsHubRawForNodesAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + nodes: []const []const u8, + ) ![]f64 { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + return self.readGraphMetricVectorAlloc(txn, metric_name, job_id, "raw_hub", iteration, nodes, true); + } + + fn aggregateHitsHubRawForNode( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + node: []const u8, + ) !f64 { + const prefix = try self.graphMetricBuildHitsHubRawNodePrefixAlloc(metric_name, job_id, iteration, node); + defer self.alloc.free(prefix); + var sum: f64 = 0.0; + var correction: f64 = 0.0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const value = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + const next = sum + value; + correction += if (@abs(sum) >= @abs(value)) (sum - next) + value else (value - next) + sum; + sum = next; + } + const value = sum + correction; + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + return value; + } + + fn validateGraphMetricVectorManifest(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64) !void { + const manifest = try self.metricBuildManifest(txn, metric_name, job_id) orelse return error.GraphMetricBuildManifestNotFound; + if (manifest.execution_schema_version != graph_metric_build_execution_schema_version) return error.InvalidGraphMetricBuildManifest; + if (try self.metricBuildJob(txn, metric_name)) |job| { + if (job.job_id == job_id and try self.topologyBinding(txn, metric_name, job_id) == null) + return error.InvalidGraphMetricBuildManifest; + } + } + + fn graphMetricNodeSlotKey(alloc: Allocator, metric_name: []const u8, job_id: u64, node: []const u8) ![]u8 { + var job_buf: [20]u8 = undefined; + return graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), "node_slot", node }); + } + + fn graphMetricVectorChunkKey(alloc: Allocator, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, chunk: u64) ![]u8 { + var job_buf: [20]u8 = undefined; + var iteration_buf: [10]u8 = undefined; + var chunk_buf: [20]u8 = undefined; + return graphMetricControlKeyWithAllocator(alloc, &.{ metric_name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), "vector", lane, try std.fmt.bufPrint(&iteration_buf, "{d}", .{iteration}), try std.fmt.bufPrint(&chunk_buf, "{d}", .{chunk}) }); + } + + fn graphMetricNodeSlotsAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, nodes: []const []const u8) ![]u64 { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const prefix = try self.topologyComponentPrefix(txn, metric_name, job_id, "node_slot"); + defer self.alloc.free(prefix); + const keys = try arena.allocator().alloc([]const u8, nodes.len); + for (nodes, 0..) |node, i| { + keys[i] = try topologyComponentKey(arena.allocator(), prefix, node); + } + const slots = try self.readU64KeysAlloc(txn, keys); + errdefer self.alloc.free(slots); + for (slots) |slot| if (slot == 0) return error.InvalidGraphMetricBuildManifest; + return slots; + } + + fn collectGraphMetricOrdinalNodesInRange(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, lower: []const u8, upper: []const u8, resume_node: []const u8, limit: ?usize, nodes: *std.ArrayListUnmanaged([]u8), slots: *std.ArrayListUnmanaged(u64)) !bool { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + // Initialization seals canonical blocks once. Iterations never revisit + // per-producer membership partials, even when a fold resumes many times. + const complete = try self.collectSealedGraphMetricMembership(txn, metric_name, job_id, lower, upper, resume_node, limit, null, nodes, slots); + try self.validateGraphMetricOrdinalDictionary(txn, metric_name, job_id, nodes.items, slots.items); + return complete; + } + + /// Numerical phases address a sealed, dense ordinal interval. The durable + /// completed-unit count is the cursor; folds additionally retain their + /// bounded in-window position. No node strings or dictionary joins occur. + /// Coverage is checksummed and bound to initialization's exact root/leaf + /// counts. Missing vectors fail closed; publication separately verifies + /// the immutable node dictionary before exposing any scores. + fn collectGraphMetricPageSlots(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, page: GraphMetricBuildPage, prior: u64, limit: ?usize, total: *u64, slots: *std.ArrayListUnmanaged(u64)) !bool { + try self.validateGraphMetricVectorManifest(txn, metric, job_id); + const cfg = self.metricConfig(metric) orelse return error.MetricNotReady; + const active = try self.graphMetricActivePlan(txn, metric, job_id); + // A claim carries scalar identity/attempt fields across transactions, + // not ownership of the storage-backed range strings. Reload those + // boundaries in this snapshot before validating the ordinal interval. + const current = try self.metricBuildPage(txn, metric, job_id, page.phase, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + const base = if (page.page_id >= graph_metric_build_summary_leaf_base) graph_metric_build_summary_leaf_base else graphMetricBuildPhasePageIdBase(cfg.kind, page.phase); + if (page.page_id < base or page.page_id - base >= active.count) return error.InvalidGraphMetricBuildManifest; + const index: usize = @intCast(page.page_id - base); + const leaf = try self.topologyMembershipLeaf(txn, metric, job_id, graph_metric_build_summary_leaf_base + index) orelse return error.InvalidGraphMetricBuildManifest; + if (leaf.state != .complete or leaf.completed_units != active.counts[index] or + !std.mem.eql(u8, leaf.range_lower, current.range_lower) or !std.mem.eql(u8, leaf.range_upper, current.range_upper)) return error.InvalidGraphMetricBuildManifest; + const count = active.counts[index]; + if (count > std.math.maxInt(u32) or prior > count) return error.InvalidGraphMetricBuildProgress; + total.* = count; + const length: usize = @intCast(@min(count - prior, limit orelse graph_metric_build_checkpoint_reduce_units)); + try slots.ensureUnusedCapacity(self.alloc, length); + for (0..length) |offset| slots.appendAssumeCapacity((@as(u64, index + 1) << 32) | (prior + offset)); + return prior + length == count; + } + + fn validateGraphMetricOrdinalDictionary(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, nodes: []const []const u8, slots: []const u64) !void { + if (nodes.len == 0) return; + const start = try self.topologyKey(txn, metric_name, job_id, try graphMetricNodeSlotKey(self.alloc, metric_name, job_id, nodes[0])); + defer self.alloc.free(start); + const prefix_len = start.len - internal_keys.encodedComponentLen(nodes[0]); + var component = std.ArrayListUnmanaged(u8).empty; + defer component.deinit(self.alloc); + var cur = try txn.openCursor(); + defer cur.close(); + var item = try cur.seekAtOrAfter(start); + for (nodes, slots) |node, expected_slot| { + const entry = item orelse return error.InvalidGraphMetricBuildManifest; + component.clearRetainingCapacity(); + try internal_keys.appendEncodedComponent(&component, self.alloc, node); + if (!std.mem.startsWith(u8, entry.key, start[0..prefix_len]) or + !std.mem.eql(u8, entry.key[prefix_len..], component.items) or entry.value.len != 8) + return error.InvalidGraphMetricBuildManifest; + const slot = std.mem.readInt(u64, entry.value[0..8], .little); + if (slot != expected_slot) return error.InvalidGraphMetricBuildManifest; + item = try cur.next(); + } + } + + fn graphMetricMembershipKey(self: *GraphIndex, metric: []const u8, job_id: u64, leaf: u64, block: u64) ![]u8 { + const prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric, job_id); + defer self.alloc.free(prefix); + return std.fmt.allocPrint(self.alloc, "{s}membership/{d}/{d}", .{ prefix, leaf, block }); + } + + /// Benchmark fixture only: one canonical leaf with maximum producer fan-in. + pub fn benchmarkMembershipFixture(self: *GraphIndex, nodes: []const []const u8) !void { + if (nodes.len > graph_metric_build_target_reduce_page_units) return error.InvalidBenchmarkResult; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + try self.putGraphMetricBuildManifestInBatch(&batch, "bench-membership", .{ .job_id = 1, .node_count = nodes.len }); + try self.putGraphMetricBuildPageInBatch(&batch, "bench-membership", .{ .job_id = 1, .phase = .initialize_ranks, .page_id = graph_metric_build_summary_leaf_base, .range_kind = .summary, .state = .complete, .completed_units = nodes.len, .total_units = nodes.len }); + try self.writeGraphMetricMembership(&batch, "bench-membership", 1, 0, 0, nodes); + for (nodes, 0..) |node, i| { + const key = try graphMetricNodeSlotKey(self.alloc, "bench-membership", 1, node); + defer self.alloc.free(key); + try putU64(&batch, key, (@as(u64, 1) << 32) | i); + for (0..256) |producer| { + const partial = try self.graphMetricBuildPageRankNodePartialKeyAlloc("bench-membership", 1, node, producer); + defer self.alloc.free(partial); + try putU64(&batch, partial, 1); + } + } + try batch.commit(); + } + + pub fn benchmarkPrepareOrdinalCursor(self: *GraphIndex, metric: []const u8) !void { + var status = try self.ensureGraphMetricPlannedBuild(metric, try self.graphMetricCurrentGeneration(metric)); + status.deinit(self.alloc); + for (0..100_000) |_| { + const phase = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk (try self.metricBuildJob(&txn, metric)).?.phase; + }; + if (phase == .reduce_ranks) return; + _ = try self.runGraphMetricPlannedWorkerStep(metric, self.metricConfig(metric).?, "ordinal-benchmark"); + } + return error.InvalidBenchmarkResult; + } + + /// Compare string/dictionary traversal with the production ordinal-only + /// cursor over exactly the same sealed initialization. + pub fn benchmarkOrdinalCursorRead(self: *GraphIndex, metric: []const u8, reference: bool) !u64 { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const job = (try self.metricBuildJob(&txn, metric)).?; + const plan = try self.graphMetricActivePlan(&txn, metric, job.job_id); + var checksum: u64 = 0; + for (0..plan.count) |i| { + const page = (try self.metricBuildPage(&txn, metric, job.job_id, .reduce_ranks, 0, graph_metric_build_summary_leaf_base + i)).?; + var prior: u64 = 0; + while (prior < plan.counts[i]) { + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + if (reference) { + if (prior != 0) return error.InvalidBenchmarkResult; + _ = try self.collectGraphMetricOrdinalNodesInRange(&txn, metric, job.job_id, page.range_lower, page.range_upper, "", null, &nodes, &slots); + } else { + var total: u64 = 0; + _ = try self.collectGraphMetricPageSlots(&txn, metric, job.job_id, page, prior, null, &total, &slots); + } + for (slots.items) |slot| checksum +%= slot; + if (slots.items.len == 0) return error.InvalidBenchmarkResult; + prior += slots.items.len; + } + } + return checksum; + } + + /// Both paths validate the same dictionary; only canonical discovery differs. + pub fn benchmarkMembershipRead(self: *GraphIndex, reference: bool) !usize { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + if (reference) { + _ = try self.collectPageRankScannedNodesInRange(&txn, "bench-membership", 1, "", "", "", null, &nodes); + const slots = try self.alloc.alloc(u64, nodes.items.len); + defer self.alloc.free(slots); + for (slots, 0..) |*slot, i| slot.* = (@as(u64, 1) << 32) | i; + try self.validateGraphMetricOrdinalDictionary(&txn, "bench-membership", 1, nodes.items, slots); + } else { + _ = try self.collectGraphMetricInitializedNodesInRange(&txn, "bench-membership", 1, "", "", "", null, &nodes); + } + return nodes.items.len; + } + + fn collectGraphMetricInitializedNodesInRange(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, lower: []const u8, upper: []const u8, resume_node: []const u8, limit: ?usize, nodes: *std.ArrayListUnmanaged([]u8)) !bool { + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + return self.collectGraphMetricOrdinalNodesInRange(txn, metric, job_id, lower, upper, resume_node, limit, nodes, &slots); + } + + fn writeGraphMetricMembership(self: *GraphIndex, batch: anytype, metric: []const u8, job_id: u64, leaf: u64, start: u64, nodes: []const []const u8) !void { + try self.requireMutableTopology(batch, metric, job_id); + var offset: usize = 0; + while (offset < nodes.len) { + const row = start + offset; + const block_id = row / membership.capacity; + const within: usize = @intCast(row % membership.capacity); + const count = @min(nodes.len - offset, membership.capacity - within); + const key = try self.topologyKey(batch, metric, job_id, try self.graphMetricMembershipKey(metric, job_id, leaf, block_id)); + defer self.alloc.free(key); + var block = if (batch.get(key)) |raw| try membership.decode(raw) else |err| switch (err) { + error.NotFound => membership.Block{}, + else => return err, + }; + if (within > block.len) return error.InvalidGraphMetricBuildManifest; + for (nodes[offset..][0..count], 0..) |node, i| { + const slot = ((leaf + 1) << 32) | (row + i); + if (within + i < block.len and (!std.mem.eql(u8, block.rows[within + i].node, node) or block.rows[within + i].slot != slot)) + return error.InvalidGraphMetricBuildManifest; + block.rows[within + i] = .{ .node = node, .slot = slot }; + } + block.len = @max(block.len, within + count); + const raw = try membership.encodeAlloc(self.alloc, block.rows[0..block.len]); + defer self.alloc.free(raw); + try batch.put(key, raw); + offset += count; + } + } + + fn collectSealedGraphMetricMembership(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, lower: []const u8, upper: []const u8, resume_node: []const u8, limit: ?usize, max_node_bytes: ?usize, nodes: *std.ArrayListUnmanaged([]u8), slots: *std.ArrayListUnmanaged(u64)) !bool { + var node_bytes: usize = 0; + const manifest = try self.metricBuildManifest(txn, metric, job_id) orelse return error.GraphMetricBuildManifestNotFound; + const leaf_count = self.graphMetricDegreeReducePageCount(@intCast(manifest.node_count)); + const seek = if (resume_node.len != 0) resume_node else lower; + // Locate one immutable leaf by range, not by enumerating every leaf or + // probing absent filtered boundary nodes in the ordinal dictionary. + var first: usize = 0; + var end = leaf_count; + while (first < end) { + const mid = first + (end - first) / 2; + const leaf = try self.topologyMembershipLeaf(txn, metric, job_id, graph_metric_build_summary_leaf_base + mid) orelse return error.InvalidGraphMetricBuildManifest; + if (leaf.range_upper.len != 0 and std.mem.order(u8, leaf.range_upper, seek) != .gt) first = mid + 1 else end = mid; + } + for (first..leaf_count) |leaf_index| { + const leaf = try self.topologyMembershipLeaf(txn, metric, job_id, graph_metric_build_summary_leaf_base + leaf_index) orelse return error.InvalidGraphMetricBuildManifest; + if (leaf.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + if (upper.len != 0 and std.mem.order(u8, leaf.range_lower, upper) != .lt) return true; + var position: u64 = 0; + if (leaf_index == first and resume_node.len != 0) { + const key = try self.topologyKey(txn, metric, job_id, try graphMetricNodeSlotKey(self.alloc, metric, job_id, resume_node)); + defer self.alloc.free(key); + const slot = try readU64OrZero(txn, key); + if (slot == 0 or slot >> 32 != leaf_index + 1) return error.InvalidGraphMetricBuildManifest; + position = (slot & std.math.maxInt(u32)) + 1; + if (position > leaf.completed_units) return error.InvalidGraphMetricBuildManifest; + const resume_key = try self.topologyKey(txn, metric, job_id, try self.graphMetricMembershipKey(metric, job_id, leaf_index, (position - 1) / membership.capacity)); + defer self.alloc.free(resume_key); + const resume_raw = txn.get(resume_key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricBuildManifest, + else => return err, + }; + const resume_block = try membership.decodeSealed(resume_raw, leaf_index, (position - 1) / membership.capacity, leaf.completed_units, leaf.range_lower, leaf.range_upper); + const within: usize = @intCast((position - 1) % membership.capacity); + if (within >= resume_block.len or resume_block.rows[within].slot != slot or !std.mem.eql(u8, resume_block.rows[within].node, resume_node)) + return error.InvalidGraphMetricBuildManifest; + } + while (position < leaf.completed_units) { + const block_id = position / membership.capacity; + const key = try self.topologyKey(txn, metric, job_id, try self.graphMetricMembershipKey(metric, job_id, leaf_index, block_id)); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricBuildManifest, + else => return err, + }; + const block = try membership.decodeSealed(raw, leaf_index, block_id, leaf.completed_units, leaf.range_lower, leaf.range_upper); + for (block.rows[@intCast(position % membership.capacity)..block.len]) |row| { + if (upper.len != 0 and std.mem.order(u8, row.node, upper) != .lt) return true; + if (lower.len == 0 or std.mem.order(u8, row.node, lower) != .lt) { + if (limit) |cap| if (nodes.items.len == cap) return false; + // Permit a single oversized ID so progress never stalls. + // Admission happens before copying, not after allocation. + if (max_node_bytes) |cap| if (nodes.items.len != 0 and row.node.len > cap -| node_bytes) return false; + node_bytes +|= row.node.len; + try nodes.ensureUnusedCapacity(self.alloc, 1); + try slots.ensureUnusedCapacity(self.alloc, 1); + nodes.appendAssumeCapacity(try self.alloc.dupe(u8, row.node)); + slots.appendAssumeCapacity(row.slot); + } + position += 1; + } + } + } + return true; + } + + fn readGraphMetricVectorAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, nodes: []const []const u8, required: bool) ![]f64 { + const slots = try self.graphMetricNodeSlotsAlloc(txn, metric_name, job_id, nodes); + defer self.alloc.free(slots); + return self.readGraphMetricVectorSlotsAlloc(txn, metric_name, job_id, lane, iteration, slots, required); + } + + // Cached bytes belong to one read transaction and one vector lane/epoch. + // A production fold reads at most 4096 edges, bounding this map as well. + const VectorReadCache = struct { + const Map = std.AutoHashMapUnmanaged(u64, ?[]const u8); + pub const empty: @This() = .{}; + map: Map = .empty, + owned: std.ArrayListUnmanaged([]u8) = .empty, + sealed: ?*@import("sealed_vector_cache.zig").Cache = null, + admission_ticket: ?u64 = null, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.owned.items) |bytes| alloc.free(bytes); + self.owned.deinit(alloc); + self.map.deinit(alloc); + } + fn contains(self: *@This(), chunk: u64) bool { + return self.map.contains(chunk); + } + fn count(self: *@This()) u32 { + return self.map.count(); + } + fn get(self: *@This(), chunk: u64) ??[]const u8 { + return self.map.get(chunk); + } + fn put(self: *@This(), alloc: Allocator, chunk: u64, bytes: ?[]const u8) !void { + try self.map.put(alloc, chunk, bytes); + } + }; + + fn sealedVectorScope(metric: []const u8) [32]u8 { + return @import("sealed_vector_cache.zig").Cache.key(metric); + } + + fn readGraphMetricVectorSlotsAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, slots: []const u64, required: bool) ![]f64 { + var cache = VectorReadCache.empty; + defer cache.deinit(self.alloc); + return self.readGraphMetricVectorSlotsCachedAlloc(txn, metric_name, job_id, lane, iteration, slots, required, &cache); + } + + fn readGraphMetricVectorSlotsCachedAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, slots: []const u64, required: bool, cache: *VectorReadCache) ![]f64 { + return self.readGraphMetricVectorSlotsTypedAlloc(f64, txn, metric_name, job_id, lane, iteration, slots, required, cache); + } + + fn graphMetricDegreeSlotsAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, slots: []const u64) ![]u64 { + var cache = VectorReadCache.empty; + defer cache.deinit(self.alloc); + return self.readGraphMetricVectorSlotsTypedAlloc(u64, txn, metric_name, job_id, "degree", 0, slots, true, &cache); + } + + fn readGraphMetricVectorSlotsTypedAlloc(self: *GraphIndex, comptime T: type, txn: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, slots: []const u64, required: bool, cache: *VectorReadCache) ![]T { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const values = try self.alloc.alloc(T, slots.len); + errdefer self.alloc.free(values); + try self.readGraphMetricVectorSlotsTypedInto(T, txn, metric_name, job_id, lane, iteration, slots, required, cache, values, arena.allocator()); + return values; + } + + fn readGraphMetricVectorSlotsTypedInto(self: *GraphIndex, comptime T: type, txn: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, slots: []const u64, required: bool, cache: *VectorReadCache, values: []T, temp: Allocator) !void { + if (values.len != slots.len) return error.InvalidGraphMetricScore; + if (cache.sealed) |sealed| if (cache.admission_ticket == null) { + cache.admission_ticket = sealed.ticket(); + }; + var missing = std.AutoHashMapUnmanaged(u64, void).empty; + var chunk_ids = std.ArrayListUnmanaged(u64).empty; + var keys = std.ArrayListUnmanaged([]const u8).empty; + for (slots) |slot| { + const chunk = slot / vector_chunk.entries; + if (cache.contains(chunk) or (try missing.getOrPut(temp, chunk)).found_existing) continue; + const key = try graphMetricVectorChunkKey(temp, metric_name, job_id, lane, iteration, chunk); + if (cache.sealed) |sealed| { + var copied: vector_chunk.Chunk = undefined; + if (sealed.copy(@import("sealed_vector_cache.zig").Cache.key(key), &copied)) { + const owned = try self.alloc.dupe(u8, &copied); + cache.owned.append(self.alloc, owned) catch |err| { + self.alloc.free(owned); + return err; + }; + try cache.put(self.alloc, chunk, owned); + continue; + } + } + try chunk_ids.append(temp, chunk); + try keys.append(temp, key); + } + if (keys.items.len != 0) { + const chunks = try self.getManyValuesAlloc(txn, keys.items); + defer self.alloc.free(chunks); + for (chunk_ids.items, keys.items, chunks) |chunk, key, raw| { + try cache.put(self.alloc, chunk, raw); + if (cache.sealed) |sealed| if (raw) |bytes| { + sealed.putAt(self.alloc, @import("sealed_vector_cache.zig").Cache.key(key), bytes, sealedVectorScope(metric_name), cache.admission_ticket.?); + }; + } + } + for (slots, 0..) |slot, i| { + const raw = cache.get(slot / vector_chunk.entries).?; + values[i] = if (raw) |chunk| if (T == u64) try vector_chunk.getU64(chunk, @intCast(slot % vector_chunk.entries), required) else try vector_chunk.get(chunk, @intCast(slot % vector_chunk.entries), required) else if (required) return error.InvalidGraphMetricScore else 0; + } + } + + const OrdinalFoldScratch = struct { + arena: std.heap.ArenaAllocator, + slots: [vector_chunk.entries]u64 = undefined, + ranks: [vector_chunk.entries]f64 = undefined, + targets: [vector_chunk.entries]u16 = undefined, + target_chunk: ?u64 = null, + + fn prepareTargets(self: *@This(), slots: []const u64, chunk: u64) void { + if (self.target_chunk == chunk) return; + @memset(&self.targets, std.math.maxInt(u16)); + for (slots, 0..) |slot, i| if (slot / vector_chunk.entries == chunk) { + self.targets[slot % vector_chunk.entries] = @intCast(i); + }; + self.target_chunk = chunk; + } + }; + + fn foldOrdinalAdjacency(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, lane: []const u8, iteration: u32, damping: f64, raw: []const u8, cache: *VectorReadCache, scratch: *OrdinalFoldScratch, fold: *ordinal_blocks.Fold, chunk: u64, expected_count: u64) !void { + std.debug.assert(scratch.target_chunk == chunk); + // Point reads do not advance the source cursor; its tile stays valid + // through the gather and fold on both native storage and LMDB. + const view = try ordinal_blocks.decodeTopologyView(raw); + if (!view.complete or view.cursor.len != 0 or view.len() > vector_chunk.entries or view.scanned != view.len() or view.len() != expected_count) return error.InvalidGraphMetricBuildManifest; + for (scratch.slots[0..view.len()], 0..) |*slot, i| slot.* = view.edge(i).source; + _ = scratch.arena.reset(.retain_capacity); + try self.readGraphMetricVectorSlotsTypedInto(f64, txn, metric, job_id, lane, iteration, scratch.slots[0..view.len()], true, cache, scratch.ranks[0..view.len()], scratch.arena.allocator()); + for (scratch.ranks[0..view.len()], 0..) |rank, i| { + const target = view.edge(i).target; + const value = rank * damping; + if (!std.math.isFinite(value) or value < 0) return error.InvalidGraphMetricScore; + if (target / vector_chunk.entries != chunk) return error.InvalidGraphMetricBuildManifest; + const index = scratch.targets[target % vector_chunk.entries]; + if (index == std.math.maxInt(u16)) continue; + const sum = fold.sums[index] + value; + fold.corrections[index] += if (@abs(fold.sums[index]) >= @abs(value)) (fold.sums[index] - sum) + value else (value - sum) + fold.sums[index]; + fold.sums[index] = sum; + } + } + + /// Isolated warm-gather benchmark. Fixture ownership is independent of + /// the allocator measuring the old/new numerical tile paths. + pub fn benchmarkOrdinalFold(alloc: Allocator, reference: bool, tiles: usize) !f64 { + const fixture = std.heap.smp_allocator; + var edges: [vector_chunk.entries]ordinal_blocks.Edge = undefined; + var vector: vector_chunk.Chunk = @splat(0); + for (0..vector_chunk.entries) |i| try vector_chunk.put(&vector, @intCast(i), @as(f64, @floatFromInt(i + 1)) / 257); + for (&edges, 0..) |*edge, i| edge.* = .{ .source = 1 + i % 255, .target = 1 }; + const raw = try ordinal_blocks.encodeTopology(fixture, .{ .edges = &edges, .cursor = @constCast(""), .scanned = edges.len, .complete = true }); + defer fixture.free(raw); + var cache = VectorReadCache.empty; + defer cache.deinit(fixture); + try cache.put(fixture, 0, &vector); + var indexes = std.AutoHashMapUnmanaged(u64, usize).empty; + defer indexes.deinit(fixture); + try indexes.put(fixture, 1, 0); + var graph: GraphIndex = undefined; + graph.alloc = alloc; + graph.metric_configs = &.{.{ .name = "rank", .kind = .pagerank, .damping = 0.85 }}; + var txn = struct { + pub fn getManySorted(_: *@This(), _: []const []const u8, _: []?[]const u8) !void { + return error.UnexpectedStorageRead; + } + }{}; + var scratch = OrdinalFoldScratch{ .arena = std.heap.ArenaAllocator.init(alloc) }; + defer scratch.arena.deinit(); + scratch.prepareTargets(&.{1}, 0); + var fold = ordinal_blocks.Fold{}; + for (0..tiles) |_| { + if (reference) { + const values = try graph.ordinalAdjacencyValuesCachedAlloc(&txn, "rank", 1, .iterate_contributions, 0, raw, &cache); + defer alloc.free(values); + for (values) |value| { + const i = indexes.get(value.ordinal).?; + const sum = fold.sums[i] + value.value; + fold.corrections[i] += if (@abs(fold.sums[i]) >= @abs(value.value)) (fold.sums[i] - sum) + value.value else (value.value - sum) + fold.sums[i]; + fold.sums[i] = sum; + } + } else try graph.foldOrdinalAdjacency(&txn, "rank", 1, "factor", 0, 0.85, raw, &cache, &scratch, &fold, 0, edges.len); + } + return fold.sums[0] + fold.corrections[0]; + } + + pub fn prepareSealedVectorBenchmark(self: *GraphIndex) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var chunk: vector_chunk.Chunk = @splat(0); + for (0..vector_chunk.entries) |slot| try vector_chunk.put(&chunk, slot, 0.5); + for (0..128) |i| { + const key = try graphMetricVectorChunkKey(self.alloc, "rank", 1, "rank", 0, i); + defer self.alloc.free(key); + try batch.put(key, &chunk); + } + try batch.commit(); + } + + pub fn benchmarkSealedVectorGather(self: *GraphIndex, checkpoints: usize, reference: bool) !usize { + defer self.sealed_vectors.deinit(self.alloc); + var seed: u32 = 123456789; + var reads: usize = 0; + const Txn = struct { + inner: *backend_erased.ReadTxn, + reads: *usize, + pub fn getManySorted(ctx: *@This(), keys: []const []const u8, values: []?[]const u8) !void { + ctx.reads.* += keys.len; + try ctx.inner.getManySorted(keys, values); + } + }; + for (0..checkpoints) |_| { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var counted = Txn{ .inner = &txn, .reads = &reads }; + var cache = VectorReadCache{ .sealed = if (reference) null else &self.sealed_vectors }; + defer cache.deinit(self.alloc); + var slots: [2048]u64 = undefined; + for (&slots) |*slot| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + slot.* = seed % (128 * vector_chunk.entries); + } + const values = try self.readGraphMetricVectorSlotsCachedAlloc(&counted, "rank", 1, "rank", 0, &slots, true, &cache); + defer self.alloc.free(values); + for (values) |value| if (value != 0.5) return error.InvalidBenchmarkResult; + } + return reads; + } + + fn ordinalContributionPrefixAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, iteration: u32, chunk: u64) ![]u8 { + const namespace = if (phase == .hits_hub_contributions) + try self.graphMetricBuildHitsHubRawPrefixAlloc(metric_name, job_id, iteration) + else + try self.graphMetricBuildPageRankContributionPrefixAlloc(metric_name, job_id, iteration); + defer self.alloc.free(namespace); + return std.fmt.allocPrint(self.alloc, "{s}ordinal/{d:0>20}/", .{ namespace, chunk }); + } + + /// Compile each bounded edge checkpoint once per cold topology producer. Subsequent power + /// iterations read numeric topology and vector blocks without parsing edge + /// keys, hashing document IDs, or resolving the ordinal dictionary again. + fn ordinalTopologyAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, cfg: GraphMetricConfig, job_id: u64, page: GraphMetricBuildPage, limit: usize) !ordinal_blocks.Topology { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var nodes = std.ArrayListUnmanaged([]const u8).empty; + // A cursor is progress, not input history. Keep one reusable buffer + // outside the arena so long typed keys do not accumulate per edge. + var cursor = std.ArrayListUnmanaged(u8).empty; + defer cursor.deinit(self.alloc); + var retained_input_bytes: usize = 0; + var scanned: u64 = 0; + var visited: usize = 0; + var complete = true; + var filter = try CompiledGraphMetricEdgeFilter.init(temp, cfg.edge_filter); + defer filter.deinit(temp); + var cur = try typed_edges.Cursor.init(temp, txn, cfg.edge_filter, page.range_lower, page.range_upper, page.cursor); + defer cur.deinit(); + while (try cur.next()) |entry| { + // Covers decoded/escaped endpoint copies and ordinal lookup + // scratch. Permit one oversized record to make forward progress. + const envelope = std.math.add(usize, std.math.mul(usize, entry.key.len, 8) catch return error.GraphMetricBuildBudgetExceeded, 256) catch return error.GraphMetricBuildBudgetExceeded; + const next_bytes = std.math.add(usize, retained_input_bytes, envelope) catch return error.GraphMetricBuildBudgetExceeded; + if (visited == limit or (visited > 0 and next_bytes > 1024 * 1024)) { + complete = false; + break; + } + retained_input_bytes = next_bytes; + visited += 1; + cursor.clearRetainingCapacity(); + try cursor.appendSlice(self.alloc, entry.cursor); + var parsed = (try parseMetricReverseEdgeKeyView(temp, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(temp); + scanned += 1; + if (!filter.allows(parsed.edge_type.bytes)) continue; + try nodes.append(temp, try temp.dupe(u8, parsed.source.bytes)); + try nodes.append(temp, try temp.dupe(u8, parsed.target.bytes)); + } + const slots = try self.graphMetricNodeSlotsAlloc(txn, metric_name, job_id, nodes.items); + defer self.alloc.free(slots); + const edges = try self.alloc.alloc(ordinal_blocks.Edge, slots.len / 2); + errdefer self.alloc.free(edges); + for (edges, 0..) |*edge, i| edge.* = .{ .source = slots[i * 2], .target = slots[i * 2 + 1] }; + return .{ .edges = edges, .cursor = try cursor.toOwnedSlice(self.alloc), .scanned = scanned, .complete = complete }; + } + + // Build row-oriented immutable adjacency once. Later iterations schedule + // only reducers, which pull from the sealed topology and current vector. + fn executeOrdinalContributionPage(self: *GraphIndex, metric_name: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob, page: GraphMetricBuildPage, max_scan_units: ?u64) !usize { + const limit: usize = @intCast(@min(max_scan_units orelse ordinal_blocks.max_edges, ordinal_blocks.max_edges)); + if (limit == 0) return error.InvalidGraphMetricBuildProgress; + if (page.iteration != 0) return error.InvalidGraphMetricBuildManifest; + var topology: ordinal_blocks.Topology = undefined; + var prior: u64 = 0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric_name, job.job_id, page.phase, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current); + prior = current.completed_units; + topology = try self.ordinalTopologyAlloc(&txn, metric_name, cfg, job.job_id, current, limit); + } + defer topology.deinit(self.alloc); + // Target is always the output row; source is the neighbor ordinal. + if (page.phase == .hits_hub_contributions) for (topology.edges) |*edge| { + std.mem.swap(u64, &edge.source, &edge.target); + }; + std.mem.sort(ordinal_blocks.Edge, topology.edges, {}, struct { + fn lessThan(_: void, a: ordinal_blocks.Edge, b: ordinal_blocks.Edge) bool { + return a.target < b.target or (a.target == b.target and a.source < b.source); + } + }.lessThan); + const completed = std.math.add(u64, prior, topology.scanned) catch return error.InvalidGraphMetricBuildProgress; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = try self.metricBuildPage(&batch, metric_name, job.job_id, page.phase, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current); + if (current.completed_units != prior) return error.GraphMetricBuildPageOutputMismatch; + var start: usize = 0; + while (start < topology.edges.len) { + const chunk = topology.edges[start].target / vector_chunk.entries; + var end = start + 1; + while (end < topology.edges.len and end - start < vector_chunk.entries and topology.edges[end].target / vector_chunk.entries == chunk) : (end += 1) {} + const prefix = try self.ordinalAdjacencyPrefixAlloc(metric_name, job.job_id, page.phase, chunk); + defer self.alloc.free(prefix); + const key = try std.fmt.allocPrint(self.alloc, "{s}{d:0>20}:{d:0>20}:{d:0>20}:{d:0>4}", .{ prefix, page.page_id, page.attempt, prior, start }); + defer self.alloc.free(key); + const encoded = try ordinal_blocks.encodeTopology(self.alloc, .{ + .edges = topology.edges[start..end], + .cursor = @constCast(""), + .scanned = end - start, + .complete = true, + }); + defer self.alloc.free(encoded); + try batch.put(key, encoded); + start = end; + } + var updated = try self.updateGraphMetricBuildPageProgressInBatch(&batch, metric_name, job.job_id, page.phase, page.iteration, page.page_id, page.worker_id, page.attempt, topology.cursor, completed, current.total_units); + if (topology.complete) { + updated.state = .complete; + updated.cursor = ""; + updated.lease_expires_at_ms = 0; + updated.output_fingerprint = completed; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, updated); + } + try batch.commit(); + return topology.edges.len; + } + + fn ordinalAdjacencyPrefixAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, chunk: u64) ![]u8 { + const prefix = try self.ordinalAdjacencyPhasePrefixAlloc(metric_name, job_id, phase); + defer self.alloc.free(prefix); + return std.fmt.allocPrint(self.alloc, "{s}{d:0>20}/", .{ prefix, chunk }); + } + + fn ordinalAdjacencyPhasePrefixAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase) ![]u8 { + const namespace = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + defer self.alloc.free(namespace); + return std.fmt.allocPrint(self.alloc, "{s}adjacency/{s}/", .{ namespace, @tagName(phase) }); + } + + fn packedAdjacencyBaseAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, chunk: u64) ![]u8 { + const namespace = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + defer self.alloc.free(namespace); + return std.fmt.allocPrint(self.alloc, "{s}adjacency-packed/{s}/{d:0>20}/", .{ namespace, @tagName(phase), chunk }); + } + + /// Merge winning producer fragments once into dense, immutable tiles. + /// Cursor, partial tile and output blocks share the summary-page attempt + /// fence. A receipt exposes only a completely packed attempt to readers. + fn compactOrdinalAdjacencyChunk(self: *GraphIndex, metric_name: []const u8, job: GraphMetricBuildJob, claimed: GraphMetricBuildPage, producer_phase: GraphMetricBuildPhase, chunk: u64, max_records: usize) !usize { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const base = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.topologyKey(&txn, metric_name, job.job_id, try self.packedAdjacencyBaseAlloc(metric_name, job.job_id, producer_phase, chunk)); + }; + defer self.alloc.free(base); + const receipt_key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + const state_key = try std.fmt.allocPrint(temp, "{s}state", .{base}); + const input_prefix = try self.ordinalAdjacencyPrefixAlloc(metric_name, job.job_id, producer_phase, chunk); + defer self.alloc.free(input_prefix); + var state = adjacency_blocks.State{ .attempt = claimed.attempt }; + var retired_attempt: ?u64 = null; + var old_state: []const u8 = ""; + var outputs = std.ArrayListUnmanaged([]const u8).empty; + var first_block: u64 = 0; + var prior_units: u64 = 0; + var records: usize = 0; + var edges: usize = 0; + var complete = false; + var receipt: adjacency_blocks.Receipt = undefined; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric_name, job.job_id, claimed.phase, claimed.iteration, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + prior_units = current.completed_units; + if (txn.get(receipt_key)) |raw| { + _ = try adjacency_blocks.Receipt.decode(raw); + return 0; + } else |err| if (err != error.NotFound) return err; + if (txn.get(state_key)) |raw| { + old_state = try temp.dupe(u8, raw); + const saved = try adjacency_blocks.State.decode(old_state); + if (saved.attempt == claimed.attempt) state = saved else retired_attempt = saved.attempt; + } else |err| if (err != error.NotFound) return err; + first_block = state.blocks; + if (state.cursor.len != 0 and !std.mem.startsWith(u8, state.cursor, input_prefix)) return error.InvalidGraphMetricBuildManifest; + for (state.pending[0..state.count]) |edge| if (edge.target / vector_chunk.entries != chunk) return error.InvalidGraphMetricBuildManifest; + var winners = std.AutoHashMapUnmanaged(u64, u64).empty; + var cursor = try txn.openCursor(); + defer cursor.close(); + var next = try cursor.seekAtOrAfter(if (state.cursor.len == 0) input_prefix else state.cursor); + if (next) |entry| if (std.mem.eql(u8, entry.key, state.cursor)) { + next = try cursor.next(); + }; + while (next) |entry| : (next = try cursor.next()) { + if (!std.mem.startsWith(u8, entry.key, input_prefix) or records == max_records) break; + const suffix = entry.key[input_prefix.len..]; + if (suffix.len < 42 or suffix[20] != ':' or suffix[41] != ':') return error.InvalidGraphMetricBuildManifest; + const producer = std.fmt.parseInt(u64, suffix[0..20], 10) catch return error.InvalidGraphMetricBuildManifest; + const attempt = std.fmt.parseInt(u64, suffix[21..41], 10) catch return error.InvalidGraphMetricBuildManifest; + const winner = try winners.getOrPut(temp, producer); + if (!winner.found_existing) { + if (winners.count() > graph_metric_build_max_partition_pages) return error.InvalidGraphMetricBuildManifest; + const page = try self.metricBuildPage(&txn, metric_name, job.job_id, producer_phase, 0, producer) orelse return error.InvalidGraphMetricBuildManifest; + if (page.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + winner.value_ptr.* = page.attempt; + } + if (attempt == winner.value_ptr.*) { + var topology = try ordinal_blocks.decodeTopology(self.alloc, entry.value); + defer topology.deinit(self.alloc); + if (topology.edges.len > adjacency_blocks.tile_entries or !topology.complete or topology.cursor.len != 0 or topology.scanned != topology.edges.len) return error.InvalidGraphMetricBuildManifest; + if (topology.edges.len > ordinal_blocks.max_edges - edges) break; + edges += topology.edges.len; + for (topology.edges) |edge| { + if (edge.target / vector_chunk.entries != chunk) return error.InvalidGraphMetricBuildManifest; + state.pending[state.count] = edge; + state.count += 1; + if (state.count == adjacency_blocks.tile_entries) { + try outputs.append(temp, try ordinal_blocks.encodeTopology(temp, .{ .edges = &state.pending, .cursor = @constCast(""), .scanned = state.count, .complete = true })); + state.blocks = std.math.add(u64, state.blocks, 1) catch return error.InvalidGraphMetricBuildManifest; + state.count = 0; + } + } + } + records += 1; + state.cursor = try temp.dupe(u8, entry.key); + } + complete = next == null or !std.mem.startsWith(u8, next.?.key, input_prefix); + if (complete) { + const full_edges = std.math.mul(u64, state.blocks, adjacency_blocks.tile_entries) catch return error.InvalidGraphMetricBuildManifest; + receipt = .{ .attempt = state.attempt, .edges = std.math.add(u64, full_edges, state.count) catch return error.InvalidGraphMetricBuildManifest }; + if (state.count != 0) try outputs.append(temp, try ordinal_blocks.encodeTopology(temp, .{ .edges = state.pending[0..state.count], .cursor = @constCast(""), .scanned = state.count, .complete = true })); + } + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = try self.metricBuildPage(&batch, metric_name, job.job_id, claimed.phase, claimed.iteration, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + if (current.completed_units != prior_units) return error.GraphMetricBuildPageOutputMismatch; + if (batch.get(receipt_key)) |raw| { + _ = try adjacency_blocks.Receipt.decode(raw); + batch.abort(); + return 0; + } else |err| if (err != error.NotFound) return err; + const latest = batch.get(state_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + if (!std.mem.eql(u8, latest, old_state)) return error.GraphMetricBuildPageOutputMismatch; + try self.requireMutableTopology(&batch, metric_name, job.job_id); + if (retired_attempt) |attempt| { + if (try self.topologyBinding(&batch, metric_name, job.job_id)) |binding| { + const data = try topology_owner.dataPrefix(temp, binding.id); + const retired = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/", .{ base, attempt }); + var digest: topology_owner.Digest = undefined; + std.crypto.hash.sha2.Sha256.hash(retired, &digest, .{}); + const task = try std.fmt.allocPrint(temp, "{s}retired/{s}", .{ data, std.fmt.bytesToHex(digest, .lower) }); + try batch.put(task, retired); + } + } + for (outputs.items, 0..) |output, i| { + const key = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/{d:0>20}", .{ base, state.attempt, first_block + i }); + try batch.put(key, output); + } + if (complete) { + const encoded = receipt.encode(); + try batch.put(receipt_key, &encoded); + if (old_state.len != 0) try batch.delete(state_key); + } else try batch.put(state_key, try state.encode(temp)); + try batch.commit(); + return records; + } + + fn ordinalAdjacencyValuesAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, iteration: u32, raw: []const u8) ![]ordinal_blocks.Value { + var cache = VectorReadCache.empty; + defer cache.deinit(self.alloc); + return self.ordinalAdjacencyValuesCachedAlloc(txn, metric_name, job_id, phase, iteration, raw, &cache); + } + + fn ordinalAdjacencyValuesCachedAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, iteration: u32, raw: []const u8, cache: *VectorReadCache) ![]ordinal_blocks.Value { + var topology = try ordinal_blocks.decodeTopology(self.alloc, raw); + defer topology.deinit(self.alloc); + if (!topology.complete or topology.cursor.len != 0 or topology.edges.len > vector_chunk.entries or topology.scanned != topology.edges.len) return error.InvalidGraphMetricBuildManifest; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const hub = phase == .hits_hub_contributions; + const lane: []const u8 = if (hub) "authority" else switch (cfg.kind) { + .pagerank => "factor", + .eigenvector => "rank", + .hits_authority, .hits_hub => "hub", + else => return error.UnsupportedGraphMetric, + }; + const slots = try self.alloc.alloc(u64, topology.edges.len); + defer self.alloc.free(slots); + for (topology.edges, slots) |edge, *slot| slot.* = edge.source; + const ranks = try self.readGraphMetricVectorSlotsCachedAlloc(txn, metric_name, job_id, lane, iteration + @as(u32, @intFromBool(hub)), slots, true, cache); + defer self.alloc.free(ranks); + const values = try self.alloc.alloc(ordinal_blocks.Value, ranks.len); + errdefer self.alloc.free(values); + for (values, topology.edges, ranks) |*value, edge, rank| { + value.* = .{ .ordinal = edge.target, .value = rank * (if (cfg.kind == .pagerank) cfg.damping else @as(f64, 1)) }; + if (!std.math.isFinite(value.value) or value.value < 0) return error.InvalidGraphMetricScore; + } + return values; + } + + /// Write immutable attempt-tagged shards directly in reducer order. The + /// checkpoint and final producer completion share this transaction. Readers + /// wait for the producer barrier and select only its completed attempt. + fn writeOrdinalContributionsInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, job_id: u64, page: GraphMetricBuildPage, checkpoint: u64, values: []const ordinal_blocks.Value) !void { + const current = try self.metricBuildPage(batch, metric_name, job_id, page.phase, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current); + if (current.completed_units != checkpoint) return error.GraphMetricBuildPageOutputMismatch; + if (values.len > ordinal_blocks.max_edges) return error.InvalidGraphMetricBuildProgress; + for (values, 0..) |value, i| { + if (i > 0 and value.ordinal <= values[i - 1].ordinal) return error.InvalidGraphMetricBuildManifest; + } + var start: usize = 0; + while (start < values.len) { + const chunk = values[start].ordinal / vector_chunk.entries; + var end = start + 1; + while (end < values.len and values[end].ordinal / vector_chunk.entries == chunk) : (end += 1) {} + const prefix = try self.ordinalContributionPrefixAlloc(metric_name, job_id, page.phase, page.iteration, chunk); + defer self.alloc.free(prefix); + const key = try std.fmt.allocPrint(self.alloc, "{s}{d:0>20}:{d:0>20}:{d:0>20}", .{ prefix, page.page_id, page.attempt, checkpoint }); + defer self.alloc.free(key); + const encoded = try ordinal_blocks.encodeValues(self.alloc, values[start..end]); + defer self.alloc.free(encoded); + try batch.put(key, encoded); + start = end; + } + } + + // Uncheckpointed oracle for small test fixtures only. Production folds + // below always bound both node cardinality and physical shuffle records. + fn ordinalContributionsForNodesAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, phase: GraphMetricBuildPhase, iteration: u32, nodes: []const []const u8) ![]f64 { + std.debug.assert(builtin.is_test); + for (nodes, 0..) |node, i| { + if (i > 0 and std.mem.order(u8, nodes[i - 1], node) != .lt) return error.InvalidGraphMetricBuildManifest; + } + const slots = try self.graphMetricNodeSlotsAlloc(txn, metric_name, job_id, nodes); + defer self.alloc.free(slots); + const result = try self.alloc.alloc(f64, slots.len); + errdefer self.alloc.free(result); + @memset(result, 0); + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const corrections = try temp.alloc(f64, slots.len); + @memset(corrections, 0); + var indexes = std.AutoHashMapUnmanaged(u64, usize).empty; + for (slots, 0..) |slot, i| try indexes.put(temp, slot, i); + var chunks = std.AutoHashMapUnmanaged(u64, void).empty; + var winners = std.AutoHashMapUnmanaged(u64, u64).empty; + for (slots) |slot| { + const chunk = slot / vector_chunk.entries; + if ((try chunks.getOrPut(temp, chunk)).found_existing) continue; + const adjacency_prefix = try self.ordinalAdjacencyPrefixAlloc(metric_name, job_id, phase, chunk); + defer self.alloc.free(adjacency_prefix); + const shuffle_prefix = try self.ordinalContributionPrefixAlloc(metric_name, job_id, phase, iteration, chunk); + defer self.alloc.free(shuffle_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + var next = try cur.seekAtOrAfter(adjacency_prefix); + const adjacency = next != null and std.mem.startsWith(u8, next.?.key, adjacency_prefix); + const prefix = if (adjacency) adjacency_prefix else shuffle_prefix; + if (!adjacency) next = try cur.seekAtOrAfter(prefix); + while (next) |entry| : (next = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const suffix = entry.key[prefix.len..]; + if (suffix.len < 42 or suffix[20] != ':' or suffix[41] != ':') return error.InvalidGraphMetricBuildManifest; + const producer = std.fmt.parseInt(u64, suffix[0..20], 10) catch return error.InvalidGraphMetricBuildManifest; + const attempt = std.fmt.parseInt(u64, suffix[21..41], 10) catch return error.InvalidGraphMetricBuildManifest; + const winner = try winners.getOrPut(temp, producer); + if (!winner.found_existing) { + if (winners.count() > graph_metric_build_max_partition_pages) return error.InvalidGraphMetricBuildManifest; + const producer_page = try self.metricBuildPage(txn, metric_name, job_id, phase, if (adjacency) 0 else iteration, producer) orelse return error.InvalidGraphMetricBuildManifest; + if (producer_page.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + winner.value_ptr.* = producer_page.attempt; + } + if (attempt != winner.value_ptr.*) continue; + const values = if (adjacency) + try self.ordinalAdjacencyValuesAlloc(txn, metric_name, job_id, phase, iteration, entry.value) + else + try ordinal_blocks.decodeValues(self.alloc, entry.value); + defer self.alloc.free(values); + for (values) |value| { + if (value.ordinal / vector_chunk.entries != chunk) return error.InvalidGraphMetricBuildManifest; + const i = indexes.get(value.ordinal) orelse continue; + const sum = result[i] + value.value; + corrections[i] += if (@abs(result[i]) >= @abs(value.value)) (result[i] - sum) + value.value else (value.value - sum) + result[i]; + result[i] = sum; + } + } + } + for (result, corrections) |*value, correction| { + value.* += correction; + if (!std.math.isFinite(value.*)) return error.InvalidGraphMetricScore; + } + return result; + } + + /// Node ordinals are job-scoped and partition-local, so adjacent producer + /// pages never share a mutable chunk. Retry writes replace slots under the + /// existing page-attempt fence; the presence bitmap preserves checkpoints. + fn writeGraphMetricVector(self: *GraphIndex, batch: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, scores: []const GraphMetricScore, divisors: ?[]const u64, retire_iteration: ?u32) !void { + return self.writeGraphMetricVectorRows(batch, metric_name, job_id, lane, iteration, scores, divisors, retire_iteration); + } + + fn writeGraphMetricVectorRows(self: *GraphIndex, batch: anytype, metric_name: []const u8, job_id: u64, lane: []const u8, iteration: u32, scores: anytype, divisors: ?[]const u64, retire_iteration: ?u32) !void { + if (divisors) |values| if (values.len != scores.len) return error.InvalidGraphMetricScore; + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const slots = if (@hasField(@TypeOf(scores[0]), "slot")) blk: { + const values = try self.alloc.alloc(u64, scores.len); + for (scores, 0..) |score, i| values[i] = score.slot; + break :blk values; + } else blk: { + const nodes = try temp.alloc([]const u8, scores.len); + for (scores, 0..) |score, i| nodes[i] = score.node; + break :blk try self.graphMetricNodeSlotsAlloc(batch, metric_name, job_id, nodes); + }; + defer self.alloc.free(slots); + for (slots) |slot| if (slot == 0) return error.InvalidGraphMetricBuildManifest; + var i: usize = 0; + while (i < slots.len) { + const chunk_id = slots[i] / vector_chunk.entries; + const key = try graphMetricVectorChunkKey(temp, metric_name, job_id, lane, iteration, chunk_id); + const degree_key = if (iteration == 0 and divisors != null and std.mem.eql(u8, lane, "factor")) + try graphMetricVectorChunkKey(temp, metric_name, job_id, "degree", 0, chunk_id) + else + null; + var degree_chunk: vector_chunk.Chunk = @splat(0); + if (degree_key) |dk| { + if (batch.get(dk)) |raw| { + if (raw.len != degree_chunk.len) return error.InvalidGraphMetricScore; + @memcpy(°ree_chunk, raw); + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + } + var chunk: vector_chunk.Chunk = @splat(0); + if (batch.get(key)) |raw| { + if (raw.len != chunk.len) return error.InvalidGraphMetricScore; + @memcpy(&chunk, raw); + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + while (i < slots.len and slots[i] / vector_chunk.entries == chunk_id) : (i += 1) { + if (i > 0 and slots[i] <= slots[i - 1]) return error.InvalidGraphMetricBuildManifest; + const value = if (divisors) |degrees| if (degrees[i] == 0) 0 else scores[i].score / @as(f64, @floatFromInt(degrees[i])) else scores[i].score; + try vector_chunk.put(&chunk, @intCast(slots[i] % vector_chunk.entries), value); + if (degree_key != null) try vector_chunk.putU64(°ree_chunk, @intCast(slots[i] % vector_chunk.entries), divisors.?[i]); + } + try batch.put(key, &chunk); + if (degree_key) |dk| try batch.put(dk, °ree_chunk); + if (retire_iteration) |old| { + const stale = try graphMetricVectorChunkKey(temp, metric_name, job_id, lane, old, chunk_id); + batch.delete(stale) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + } + + fn pageRankFactorsForNodesAlloc(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, iteration: u32, nodes: []const []const u8) ![]f64 { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + return self.readGraphMetricVectorAlloc(txn, metric_name, job_id, "factor", iteration, nodes, true); + } + + /// Benchmark oracle: isolate the former string-to-ordinal write boundary + /// from the ordinal rows now carried by production reducers. + pub fn benchmarkVectorRowsAlloc(self: *GraphIndex, txn: anytype, nodes: []const []const u8, reference: bool) !void { + if (reference) { + const scores = try self.alloc.alloc(GraphMetricScore, nodes.len); + defer self.alloc.free(scores); + for (nodes, scores) |node, *score| score.* = .{ .node = node, .score = 0.5 }; + try self.writeGraphMetricVector(txn, "rank", 1, "rank", 1, scores, null, null); + } else { + const scores = try self.alloc.alloc(OrdinalMetricScore, nodes.len); + defer self.alloc.free(scores); + for (nodes, scores, 0..) |node, *score, i| score.* = .{ .node = node, .score = 0.5, .slot = i + 1 }; + try self.writeGraphMetricVectorRows(txn, "rank", 1, "rank", 1, scores, null, null); + } + } + + fn pageRankRanksForNodesAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + nodes: []const []const u8, + required: bool, + ) ![]f64 { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + _ = required; + return self.readGraphMetricVectorAlloc(txn, metric_name, job_id, "rank", iteration, nodes, true); + } + + fn hitsRanksForNodesAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + vector_name: []const u8, + iteration: u32, + nodes: []const []const u8, + required: bool, + ) ![]f64 { + try self.validateGraphMetricVectorManifest(txn, metric_name, job_id); + _ = required; + return self.readGraphMetricVectorAlloc(txn, metric_name, job_id, vector_name, iteration, nodes, true); + } + + fn pageRankOutDegreesForNodesAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + nodes: []const []const u8, + ) ![]u64 { + var key_arena = std.heap.ArenaAllocator.init(self.alloc); + defer key_arena.deinit(); + const keys = try self.alloc.alloc([]u8, nodes.len); + defer self.alloc.free(keys); + const prefix = try self.topologyComponentPrefix(txn, metric_name, job_id, "pagerank_out_degree_total"); + defer self.alloc.free(prefix); + for (nodes, 0..) |node, i| { + keys[i] = try topologyComponentKey(key_arena.allocator(), prefix, node); + } + return try self.readU64KeysAlloc(txn, keys); + } + + /// Fold each ordinal block once, with a durable input cursor and bounded + /// compensated state. Raw vector output, partial norm/dangling mass and + /// the leaf checkpoint commit atomically. Data reducers only scale vectors. + fn executeOrdinalReduceSummary(self: *GraphIndex, metric_name: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob, claimed: GraphMetricBuildPage, max_nodes: usize, max_records: usize) !usize { + return self.executeOrdinalReduceSummaryCheckpoint(metric_name, cfg, job, claimed, max_nodes, max_records, true); + } + + // At most one bounded packing pass followed by one bounded numeric fold. + // Small chunks do not pay another maintenance tick just to consume a + // receipt produced by this same worker; large chunks remain checkpointed. + fn executeOrdinalReduceSummaryCheckpoint(self: *GraphIndex, metric_name: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob, claimed: GraphMetricBuildPage, max_nodes: usize, max_records: usize, may_pack: bool) !usize { + if (self.topology_preparation_only) return self.executeTopologyPackingSummary(metric_name, job, claimed, max_nodes, max_records); + if (max_nodes == 0 or max_records == 0) return error.InvalidGraphMetricBuildProgress; + // Capture before opening the snapshot/validating the lease. Retirement + // can race a reclaimed worker, but its old ticket cannot repopulate it. + const admission_ticket = self.sealed_vectors.ticket(); + // Dense tiles contain at most 256 edges. Bound both tile reads and + // vector-gather work independently of producer checkpoint boundaries. + const record_limit = @min(max_records, ordinal_blocks.max_edges / vector_chunk.entries); + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const namespace = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(namespace); + const state_key = try std.fmt.allocPrint(temp, "{s}fold/{s}/{d}/{d}", .{ namespace, @tagName(claimed.phase), claimed.iteration, claimed.page_id }); + var compaction_chunk: ?u64 = null; + var old_state: []const u8 = ""; + var fold = ordinal_blocks.Fold{}; + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + var reached_end = false; + var total_units: u64 = 0; + var records: usize = 0; + var partial_sum: f64 = 0; + var prior_sum: f64 = 0; + const hub = claimed.phase == .hits_hub_reduce_ranks; + const producer_phase: GraphMetricBuildPhase = if (hub) .hits_hub_contributions else .iterate_contributions; + var scores = std.ArrayListUnmanaged(OrdinalMetricScore).empty; + defer scores.deinit(self.alloc); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric_name, job.job_id, claimed.phase, claimed.iteration, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + prior_sum = current.rank_sum; + fold = .{ .attempt = current.attempt, .prior = current.completed_units }; + if (txn.get(state_key)) |raw| { + old_state = try temp.dupe(u8, raw); + const saved = try ordinal_blocks.Fold.decode(old_state); + if (saved.attempt == fold.attempt and saved.prior == fold.prior) fold = saved; + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + reached_end = try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, current, current.completed_units, if (fold.count != 0) fold.count else @min(max_nodes, ordinal_blocks.fold_entries), &total_units, &slots_list); + const slots = slots_list.items; + var slot_hash = std.hash.Wyhash.init(0); + for (slots, 0..) |slot, i| { + if (i > 0 and slot <= slots[i - 1]) return error.InvalidGraphMetricBuildManifest; + graphMetricConfigFingerprintHashU64(&slot_hash, slot); + } + const fingerprint = slot_hash.final(); + if (fold.count != 0 and (fold.count != slots.len or fold.fingerprint != fingerprint)) return error.InvalidGraphMetricBuildManifest; + fold.count = @intCast(slots.len); + fold.fingerprint = fingerprint; + // Check all remaining chunks before gathering any numeric data. + // A missing receipt must not cause repeated partial vector work + // while another chunk is still being packed. + var previous_chunk: ?u64 = null; + for (slots[fold.position..]) |slot| { + const chunk = slot / vector_chunk.entries; + if (previous_chunk == chunk) continue; + previous_chunk = chunk; + const base = try self.topologyKey(&txn, metric_name, job.job_id, try self.packedAdjacencyBaseAlloc(metric_name, job.job_id, producer_phase, chunk)); + defer self.alloc.free(base); + const receipt_key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + if (txn.get(receipt_key)) |raw| { + _ = try adjacency_blocks.Receipt.decode(raw); + } else |err| switch (err) { + error.NotFound => { + if (claimed.iteration != 0) return error.InvalidGraphMetricBuildManifest; + compaction_chunk = chunk; + break; + }, + else => return err, + } + } + // The phase barrier seals this source lane before reducers run. + // Only this path may reuse owned bytes across read transactions. + var vector_cache = VectorReadCache{ .sealed = &self.sealed_vectors, .admission_ticket = admission_ticket }; + defer vector_cache.deinit(self.alloc); + var scratch = OrdinalFoldScratch{ .arena = std.heap.ArenaAllocator.init(self.alloc) }; + defer scratch.arena.deinit(); + const lane: []const u8 = if (hub) "authority" else switch (cfg.kind) { + .pagerank => "factor", + .eigenvector => "rank", + .hits_authority, .hits_hub => "hub", + else => return error.UnsupportedGraphMetric, + }; + while (compaction_chunk == null and fold.position < slots.len and records < record_limit) { + const chunk = slots[fold.position] / vector_chunk.entries; + scratch.prepareTargets(slots, chunk); + const base = try self.topologyKey(&txn, metric_name, job.job_id, try self.packedAdjacencyBaseAlloc(metric_name, job.job_id, producer_phase, chunk)); + defer self.alloc.free(base); + const receipt_key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + const receipt_raw = txn.get(receipt_key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricBuildManifest, + else => return err, + }; + const receipt = try adjacency_blocks.Receipt.decode(receipt_raw); + const prefix = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/", .{ base, receipt.attempt }); + var expected: u64 = 0; + if (fold.cursor.len != 0) { + if (!std.mem.startsWith(u8, fold.cursor, prefix) or fold.cursor.len != prefix.len + 20) return error.InvalidGraphMetricBuildManifest; + const last = std.fmt.parseInt(u64, fold.cursor[prefix.len..], 10) catch return error.InvalidGraphMetricBuildManifest; + expected = std.math.add(u64, last, 1) catch return error.InvalidGraphMetricBuildManifest; + } + var cur = try txn.openCursor(); + defer cur.close(); + var next = try cur.seekAtOrAfter(if (fold.cursor.len != 0) fold.cursor else prefix); + if (next) |entry| if (std.mem.eql(u8, entry.key, fold.cursor)) { + next = try cur.next(); + }; + while (next) |entry| : (next = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix) or records == record_limit) break; + if (entry.key.len != prefix.len + 20 or expected >= receipt.blocks()) return error.InvalidGraphMetricBuildManifest; + const index = std.fmt.parseInt(u64, entry.key[prefix.len..], 10) catch return error.InvalidGraphMetricBuildManifest; + if (index != expected) return error.InvalidGraphMetricBuildManifest; + try self.foldOrdinalAdjacency(&txn, metric_name, job.job_id, lane, claimed.iteration + @as(u32, @intFromBool(hub)), if (cfg.kind == .pagerank) cfg.damping else 1, entry.value, &vector_cache, &scratch, &fold, chunk, @min(adjacency_blocks.tile_entries, receipt.edges - expected * adjacency_blocks.tile_entries)); + expected += 1; + records += 1; + fold.cursor = try temp.dupe(u8, entry.key); + } + if (next != null and std.mem.startsWith(u8, next.?.key, prefix)) break; + if (expected != receipt.blocks()) return error.InvalidGraphMetricBuildManifest; + while (fold.position < slots.len and slots[fold.position] / vector_chunk.entries == chunk) fold.position += 1; + fold.cursor = ""; + } + if (compaction_chunk == null and fold.position == fold.count) { + for (slots, 0..) |slot, i| { + const value = fold.sums[i] + fold.corrections[i]; + if (!std.math.isFinite(value) or value < 0) return error.InvalidGraphMetricScore; + try scores.append(self.alloc, .{ .node = "", .score = value, .slot = slot }); + if (cfg.kind != .pagerank) partial_sum += value * value; + } + if (cfg.kind == .pagerank) { + const degrees = try self.graphMetricDegreeSlotsAlloc(&txn, metric_name, job.job_id, slots_list.items); + defer self.alloc.free(degrees); + const ranks = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "rank", claimed.iteration, slots_list.items, true); + defer self.alloc.free(ranks); + for (degrees, ranks) |degree, rank| if (degree == 0) { + partial_sum += rank; + }; + } + } + } + if (compaction_chunk) |chunk| { + if (!may_pack) return 0; + const packed_records = try self.compactOrdinalAdjacencyChunk(metric_name, job, claimed, producer_phase, chunk, @min(max_records, graph_metric_build_adoption_page_units)); + return packed_records + try self.executeOrdinalReduceSummaryCheckpoint(metric_name, cfg, job, claimed, max_nodes, max_records, false); + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var current = try self.metricBuildPage(&batch, metric_name, job.job_id, claimed.phase, claimed.iteration, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + if (current.completed_units != fold.prior or current.rank_sum != prior_sum) return error.GraphMetricBuildPageOutputMismatch; + const latest_state = batch.get(state_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + if (!std.mem.eql(u8, old_state, latest_state)) return error.GraphMetricBuildPageOutputMismatch; + if (fold.position != fold.count) { + const encoded = try fold.encode(temp); + try batch.put(state_key, encoded); + } else { + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, if (hub) "raw_hub" else "raw_rank", claimed.iteration, scores.items, null, null); + current.total_units = total_units; + current.completed_units = std.math.add(u64, current.completed_units, slots_list.items.len) catch return error.InvalidGraphMetricBuildProgress; + if (current.completed_units > current.total_units) return error.InvalidGraphMetricBuildProgress; + current.rank_sum += partial_sum; + if (!std.math.isFinite(current.rank_sum)) return error.InvalidGraphMetricScore; + current.cursor = ""; + if (reached_end) { + current.state = .complete; + current.total_units = current.completed_units; + current.lease_expires_at_ms = 0; + current.cursor = ""; + var hash = std.hash.Wyhash.init(0); + graphMetricConfigFingerprintHashU64(&hash, current.completed_units); + graphMetricConfigFingerprintHashU64(&hash, @bitCast(current.rank_sum)); + current.output_fingerprint = hash.final(); + } + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, current); + if (old_state.len != 0) try batch.delete(state_key); + } + try batch.commit(); + return records; + } + + fn executeTopologyPackingSummary(self: *GraphIndex, metric: []const u8, job: GraphMetricBuildJob, claimed: GraphMetricBuildPage, max_nodes: usize, max_records: usize) !usize { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + const producer: GraphMetricBuildPhase = if (claimed.phase == .hits_hub_reduce_ranks) .hits_hub_contributions else .iterate_contributions; + var pack: ?u64 = null; + var prior: u64 = 0; + var total_units: u64 = 0; + const complete = read: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric, job.job_id, claimed.phase, 0, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + prior = current.completed_units; + const end = try self.collectGraphMetricPageSlots(&txn, metric, job.job_id, current, current.completed_units, @min(max_nodes, vector_chunk.entries), &total_units, &slots); + var previous: ?u64 = null; + for (slots.items) |slot| { + const chunk = slot / vector_chunk.entries; + if (previous == chunk) continue; + previous = chunk; + const base = try self.topologyKey(&txn, metric, job.job_id, try self.packedAdjacencyBaseAlloc(metric, job.job_id, producer, chunk)); + defer self.alloc.free(base); + const key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + if (txn.get(key)) |raw| { + _ = try adjacency_blocks.Receipt.decode(raw); + } else |err| switch (err) { + error.NotFound => { + pack = chunk; + break; + }, + else => return err, + } + } + break :read end; + }; + if (pack) |chunk| return self.compactOrdinalAdjacencyChunk(metric, job, claimed, producer, chunk, @min(max_records, graph_metric_build_adoption_page_units)); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var current = try self.metricBuildPage(&batch, metric, job.job_id, claimed.phase, 0, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + if (current.completed_units != prior) return error.GraphMetricBuildPageOutputMismatch; + current.total_units = total_units; + current.completed_units += slots.items.len; + if (current.completed_units > current.total_units) return error.InvalidGraphMetricBuildProgress; + current.cursor = ""; + if (complete) { + current.state = .complete; + current.total_units = current.completed_units; + current.cursor = ""; + current.lease_expires_at_ms = 0; + current.output_fingerprint = current.completed_units; + } + try self.putGraphMetricBuildPageInBatch(&batch, metric, current); + try batch.commit(); + return slots.items.len; + } + + fn executeGraphMetricReduceSummaryBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return self.executeGraphMetricReduceSummaryBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeGraphMetricReduceSummaryBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_nodes: usize, + ) !usize { + if (page.page_id >= graph_metric_build_summary_leaf_base and + (page.phase == .reduce_ranks or page.phase == .hits_hub_reduce_ranks) and cfg.kind != .degree) + return self.executeOrdinalReduceSummary(metric_name, cfg, job, page, max_nodes, graph_metric_build_adoption_page_units); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + var range_lower: []u8 = ""; + defer if (range_lower.len > 0) self.alloc.free(range_lower); + var range_upper: []u8 = ""; + defer if (range_upper.len > 0) self.alloc.free(range_upper); + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + var prior_sum: f64 = 0.0; + var seed_mass = metric_kernels.warm_start.Mass{}; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric_name, job.job_id, page.phase, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, current); + if (current.range_kind != .summary) return error.InvalidGraphMetricBuildPage; + if (current.range_lower.len > 0) range_lower = try self.alloc.dupe(u8, current.range_lower); + if (current.range_upper.len > 0) range_upper = try self.alloc.dupe(u8, current.range_upper); + if (current.cursor.len > 0) resume_cursor = try self.alloc.dupe(u8, current.cursor); + prior_completed_units = current.completed_units; + total_units = current.total_units; + prior_sum = current.rank_sum; + try seed_mass.add(current.total_delta); + } + + if (page.page_id == 0 and (cfg.kind != .degree or total_units > graph_metric_build_checkpoint_reduce_units)) { + var scalar = metric_kernels.warm_start.Mass{}; + var seed = metric_kernels.warm_start.Mass{}; + var units: u64 = 0; + var fingerprint = std.hash.Wyhash.init(0xC641_F32A_9B51_E0D3); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const active = if (page.iteration != 0) try self.graphMetricActivePlan(&txn, metric_name, job.job_id) else null; + const count = if (active) |work| work.count else self.graphMetricDegreeReducePageCount(@intCast(total_units)); + for (0..count) |index| { + if (active) |work| if (work.counts[index] == 0) continue; + const leaf = try self.metricBuildPage(&txn, metric_name, job.job_id, page.phase, page.iteration, graph_metric_build_summary_leaf_base + index) orelse return error.InvalidGraphMetricBuildManifest; + if (leaf.range_kind != .summary or leaf.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + units = std.math.add(u64, units, leaf.completed_units) catch return error.InvalidGraphMetricBuildManifest; + try scalar.add(leaf.rank_sum); + try seed.add(leaf.total_delta); + graphMetricConfigFingerprintHashU64(&fingerprint, leaf.output_fingerprint); + } + } + if (units > total_units) return error.InvalidGraphMetricBuildManifest; + _ = try self.completeGraphMetricBuildPageWithConvergenceForAttempt( + metric_name, + job.job_id, + page.phase, + page.iteration, + page.page_id, + page.worker_id, + page.attempt, + units, + fingerprint.final(), + 0, + try seed.total(), + try scalar.total(), + false, + ); + return @intCast(units); + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + var adopted_topology = false; + const reached_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (page.phase == .initialize_ranks) { + if (try self.topologyBinding(&txn, metric_name, job.job_id)) |binding| adopted_topology = binding.adopted; + if (adopted_topology) break :blk try self.collectGraphMetricInitializedNodesInRange(&txn, metric_name, job.job_id, range_lower, range_upper, resume_cursor, max_nodes, &nodes); + } + break :blk switch (cfg.kind) { + .degree => try self.collectDegreePartialNodesInRange( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + max_nodes, + &nodes, + ), + .pagerank, .eigenvector, .hits_authority, .hits_hub => try self.collectPageRankScannedNodesInRange( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + max_nodes, + &nodes, + ), + }; + }; + + if (!adopted_topology and page.phase == .initialize_ranks and page.page_id >= graph_metric_build_summary_leaf_base) { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = try self.metricBuildPage(&batch, metric_name, job.job_id, page.phase, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current); + try self.requireMutableTopology(&batch, metric_name, job.job_id); + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const prefix = try self.topologyComponentPrefix(&batch, metric_name, job.job_id, "node_slot"); + defer self.alloc.free(prefix); + for (nodes.items, 0..) |node, i| { + const local_slot = std.math.add(u64, prior_completed_units, i) catch return error.GraphMetricBuildBudgetExceeded; + if (local_slot > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + const ordinal = ((page.page_id - graph_metric_build_summary_leaf_base + 1) << 32) | local_slot; + const key = try topologyComponentKey(arena.allocator(), prefix, node); + const prior = try readU64OrZero(&batch, key); + if (prior != 0 and prior != ordinal) return error.InvalidGraphMetricBuildManifest; + try putU64(&batch, key, ordinal); + } + try self.writeGraphMetricMembership(&batch, metric_name, job.job_id, page.page_id - graph_metric_build_summary_leaf_base, prior_completed_units, nodes.items); + try batch.commit(); + } + + var partial_sum: f64 = 0.0; + const seed_generation = if (cfg.kind == .pagerank and page.phase == .initialize_ranks) + try self.pinPageRankSeedGeneration(metric_name, job, page) + else + 0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + switch (page.phase) { + .initialize_ranks => { + partial_sum = @floatFromInt(nodes.items.len); + if (seed_generation != 0) { + try self.validatePageRankSeedGeneration(&txn, metric_name, seed_generation); + const scores = try self.graphMetricScoresInTxnAlloc(&txn, metric_name, seed_generation, nodes.items); + defer self.alloc.free(scores); + for (scores) |score| try seed_mass.add(score orelse 0); + } + }, + .reduce_ranks => switch (cfg.kind) { + // Each node appears once even when several scan partitions + // emitted incident-degree partials. + .degree => partial_sum = @floatFromInt(nodes.items.len), + .pagerank => { + const out_degrees = try self.pageRankOutDegreesForNodesAlloc(&txn, metric_name, job.job_id, nodes.items); + defer self.alloc.free(out_degrees); + const ranks = try self.pageRankRanksForNodesAlloc(&txn, metric_name, job.job_id, page.iteration, nodes.items, false); + defer self.alloc.free(ranks); + for (out_degrees, ranks) |out_degree, rank| if (out_degree == 0) { + partial_sum += rank; + }; + }, + .eigenvector, .hits_authority, .hits_hub => { + const contributions = try self.pageRankContributionsForNodesAlloc(&txn, metric_name, job.job_id, page.iteration, nodes.items); + defer self.alloc.free(contributions); + for (contributions) |contribution| partial_sum += contribution * contribution; + }, + }, + .hits_hub_reduce_ranks => { + const raw_hubs = try self.hitsHubRawForNodesAlloc(&txn, metric_name, job.job_id, page.iteration, nodes.items); + defer self.alloc.free(raw_hubs); + for (raw_hubs) |raw_hub| partial_sum += raw_hub * raw_hub; + }, + else => return error.UnsupportedGraphMetricBuildPhase, + } + if (!std.math.isFinite(partial_sum)) return error.InvalidGraphMetricScore; + } + + const accumulated = prior_sum + partial_sum; + if (!std.math.isFinite(accumulated)) return error.InvalidGraphMetricScore; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(nodes.items.len)); + if (total_units != 0 and completed_units_raw > total_units) return error.InvalidGraphMetricBuildManifest; + const completed_units = completed_units_raw; + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + if (!reached_end) { + const cursor = if (nodes.items.len > 0) nodes.items[nodes.items.len - 1] else resume_cursor; + _ = try self.updateGraphMetricBuildSummaryPageProgressForAttempt( + metric_name, + job.job_id, + page.phase, + page.iteration, + page.page_id, + worker_id, + page.attempt, + cursor, + completed_units, + total_units, + accumulated, + try seed_mass.total(), + ); + return nodes.items.len; + } + + var hasher = std.hash.Wyhash.init(0x9DA8_316C_5294_EB77); + graphMetricConfigFingerprintHashU64(&hasher, @intFromEnum(page.phase)); + graphMetricConfigFingerprintHashU64(&hasher, page.iteration); + graphMetricConfigFingerprintHashU64(&hasher, completed_units); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(accumulated)); + graphMetricConfigFingerprintHashU64(&hasher, @bitCast(try seed_mass.total())); + _ = try self.completeGraphMetricBuildPageWithConvergenceForAttempt( + metric_name, + job.job_id, + page.phase, + page.iteration, + page.page_id, + worker_id, + page.attempt, + completed_units, + hasher.final(), + 0.0, + try seed_mass.total(), + accumulated, + false, + ); + return nodes.items.len; + } + + fn graphMetricReduceSummaryValue( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphMetricBuildPhase, + iteration: u32, + ) !?GraphMetricBuildPage { + const page = try self.metricBuildPage(txn, metric_name, job_id, phase, iteration, 0) orelse return null; + if (page.range_kind != .summary) return null; + if (page.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + if (!std.math.isFinite(page.rank_sum)) return error.InvalidGraphMetricScore; + return page; + } + + fn validatePageRankSeedGeneration(self: *GraphIndex, txn: anytype, metric_name: []const u8, generation: u64) !void { + if (generation != 0 and try self.metricPublishedGeneration(txn, metric_name) != generation) + return error.GraphMetricBuildSuperseded; + } + + /// Freeze the seed identity once per job, including an explicit cold-start + /// marker. Summary checkpoints and initialization pages must never mix + /// publications or configurations. The key is reclaimed with job state. + fn pinPageRankSeedGeneration(self: *GraphIndex, metric_name: []const u8, job: GraphMetricBuildJob, claimed: GraphMetricBuildPage) !u64 { + var job_buf: [20]u8 = undefined; + const key = try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job.job_id}), "seed_generation" }); + defer self.alloc.free(key); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (txn.get(key)) |raw| { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + const generation = std.mem.readInt(u64, raw[0..8], .little); + try self.validatePageRankSeedGeneration(&txn, metric_name, generation); + return generation; + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = try self.metricBuildPage(&batch, metric_name, job.job_id, claimed.phase, claimed.iteration, claimed.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed, current); + var generation = try self.metricPublishedGeneration(&batch, metric_name); + if (generation != 0) { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const fingerprint_key = try self.graphMetricMetaConfigFingerprintKeyAlloc(metric_name, generation); + defer self.alloc.free(fingerprint_key); + if (batch.get(fingerprint_key)) |raw| { + if (raw.len != 8 or std.mem.readInt(u64, raw[0..8], .little) != graphMetricConfigFingerprint(cfg)) generation = 0; + } else |err| switch (err) { + error.NotFound => generation = 0, + else => return err, + } + } + // A competing page may have pinned the job while we acquired the + // writer; its decision wins even if it selected a cold start. + if (batch.get(key)) |raw| { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + generation = std.mem.readInt(u64, raw[0..8], .little); + try self.validatePageRankSeedGeneration(&batch, metric_name, generation); + } else |err| switch (err) { + error.NotFound => try putU64(&batch, key, generation), + else => return err, + } + try batch.commit(); + return generation; + } + + fn pageRankSeedMass(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, generation: u64) !f64 { + if (generation == 0) return 0; + try self.validatePageRankSeedGeneration(txn, metric_name, generation); + if (try self.graphMetricReduceSummaryValue(txn, metric_name, job_id, .initialize_ranks, 0)) |summary| return summary.total_delta; + // Only the single-page case reaches this path. Larger graphs use the + // resumable dependency page above, never one O(V) rescan per worker. + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + const complete = try self.collectPageRankScannedNodesInRange(txn, metric_name, job_id, "", "", "", graph_metric_build_target_reduce_page_units, &nodes); + if (!complete) return error.InvalidGraphMetricBuildManifest; + const scores = try self.graphMetricScoresInTxnAlloc(txn, metric_name, generation, nodes.items); + defer self.alloc.free(scores); + var mass = metric_kernels.warm_start.Mass{}; + for (scores) |score| try mass.add(score orelse 0); + return try mass.total(); + } + + fn graphMetricBuildNodeCount(self: *GraphIndex, metric_name: []const u8, job_id: u64) !usize { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.graphMetricReduceSummaryValue(&txn, metric_name, job_id, .initialize_ranks, 0)) |summary| { + return try graphMetricSummaryCount(summary); + } + const manifest = try self.metricBuildManifest(&txn, metric_name, job_id) orelse return error.GraphMetricBuildManifestNotFound; + const node_count = std.math.cast(usize, manifest.node_count) orelse return error.InvalidGraphMetricBuildManifest; + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, .initialize_ranks, node_count)) { + return error.InvalidGraphMetricBuildManifest; + } + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + try self.collectPageRankScannedNodes(&txn, metric_name, job_id, &nodes); + return nodes.items.len; + } + + fn graphMetricSummaryCount(summary: GraphMetricBuildPage) !usize { + if (summary.range_kind != .summary or summary.state != .complete) return error.InvalidGraphMetricBuildManifest; + return std.math.cast(usize, summary.completed_units) orelse error.InvalidGraphMetricBuildManifest; + } + + fn graphMetricBuildAdoptionCursorAlloc(self: *GraphIndex, output_fingerprint: u64) ![]u8 { + return try std.fmt.allocPrint(self.alloc, "{s}{d}", .{ graph_metric_build_adoption_cursor_prefix, output_fingerprint }); + } + + fn graphMetricBuildAdoptionFingerprint(cursor: []const u8) ?u64 { + if (!std.mem.startsWith(u8, cursor, graph_metric_build_adoption_cursor_prefix)) return null; + return std.fmt.parseInt(u64, cursor[graph_metric_build_adoption_cursor_prefix.len..], 10) catch null; + } + + fn graphMetricBuildPageAdoptionFingerprint( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + claimed_page: GraphMetricBuildPage, + ) !?u64 { + // A renewed claim can carry slices decoded from a write transaction + // that has already committed. Read cursor state from a live snapshot + // instead of dereferencing those borrowed slices after the transaction + // boundary. + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current_page = try self.metricBuildPage( + &txn, + metric_name, + job_id, + claimed_page.phase, + claimed_page.iteration, + claimed_page.page_id, + ) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + return graphMetricBuildAdoptionFingerprint(current_page.cursor); + } + + fn executePageRankScanBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executePageRankScanBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executePageRankScanBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + if (try self.graphMetricBuildPageAdoptionFingerprint(metric_name, job.job_id, page)) |fingerprint| { + const adoption = try self.adoptGraphMetricAttemptOutputPage(metric_name, cfg.kind, job.job_id, page); + if (adoption.reached_end) { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, page.phase, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + } + return adoption.adopted; + } + var compiled_filter = try CompiledGraphMetricEdgeFilter.init(self.alloc, cfg.edge_filter); + defer compiled_filter.deinit(self.alloc); + var out_degrees = std.StringHashMapUnmanaged(u64).empty; + defer self.freeStringHashMapKeys(u64, &out_degrees); + var nodes = std.StringHashMapUnmanaged(void).empty; + defer self.freeStringHashMapKeys(void, &nodes); + + var scanned_units: u64 = 0; + var visited_units: u64 = 0; + var prior_completed_units: u64 = 0; + var page_attempt = page.attempt; + var reached_page_end = true; + var last_scanned_key: std.ArrayListUnmanaged(u8) = .empty; + defer last_scanned_key.deinit(self.alloc); + { + var control_txn = try self.beginReadReverseTxn(); + defer control_txn.abort(); + const execution_page = try self.metricBuildPage(&control_txn, metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + if (execution_page.range_kind != .reverse_edges) return error.InvalidGraphMetricBuildPage; + prior_completed_units = execution_page.completed_units; + page_attempt = execution_page.attempt; + var cur = try typed_edges.Cursor.init(self.alloc, &control_txn, cfg.edge_filter, execution_page.range_lower, execution_page.range_upper, execution_page.cursor); + defer cur.deinit(); + while (try cur.next()) |entry| { + if (max_scan_units) |limit| { + if (visited_units >= limit) { + reached_page_end = false; + break; + } + } + visited_units += 1; + last_scanned_key.clearRetainingCapacity(); + try last_scanned_key.appendSlice(self.alloc, entry.cursor); + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(self.alloc); + scanned_units += 1; + if (!compiled_filter.allows(parsed.edge_type.bytes)) continue; + try self.putPageRankScanNode(&nodes, parsed.source.bytes); + try self.putPageRankScanNode(&nodes, parsed.target.bytes); + const out_degree = try self.getOrPutOwnedStringMap(u64, &out_degrees, parsed.source.bytes); + if (out_degree.found_existing) { + out_degree.value_ptr.* += 1; + } else { + out_degree.value_ptr.* = 1; + } + } + } + + const completed_units_raw = prior_completed_units + scanned_units; + const completed_units = if (page.total_units != 0) @min(completed_units_raw, page.total_units) else completed_units_raw; + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + var out_degree_total: u64 = 0; + var out_degree_it = out_degrees.valueIterator(); + while (out_degree_it.next()) |value| out_degree_total = std.math.add(u64, out_degree_total, value.*) catch + return error.InvalidGraphMetricBuildManifest; + const fingerprint = graphMetricPageRankScanFingerprint(page, completed_units, nodes.count(), out_degree_total); + var adoption_cursor: []u8 = ""; + defer if (adoption_cursor.len > 0) self.alloc.free(adoption_cursor); + if (reached_page_end) adoption_cursor = try self.graphMetricBuildAdoptionCursorAlloc(fingerprint); + const progress_cursor = if (reached_page_end) adoption_cursor else last_scanned_key.items; + var write_page = page; + write_page.attempt = page_attempt; + try self.writePageRankScanPartialsForAttempt( + metric_name, + job, + write_page, + prior_completed_units, + &out_degrees, + &nodes, + worker_id, + progress_cursor, + if (reached_page_end and page.total_units != 0) page.total_units else completed_units, + page.total_units, + ); + if (!reached_page_end) { + return nodes.count(); + } + const complete_units = if (page.total_units != 0) page.total_units else completed_units; + const adoption = try self.adoptGraphMetricAttemptOutputPage(metric_name, cfg.kind, job.job_id, write_page); + if (!adoption.reached_end) { + return nodes.count(); + } + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint); + return nodes.count(); + } + + fn writePageRankScanPartialsForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + prior_completed_units: u64, + out_degrees: *std.StringHashMapUnmanaged(u64), + nodes: *std.StringHashMapUnmanaged(void), + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .scan_edges_and_out_degree, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + + const out_degree_keys = try self.alloc.alloc([]u8, out_degrees.count()); + var initialized_out_degree_keys: usize = 0; + defer { + for (out_degree_keys[0..initialized_out_degree_keys]) |key| self.alloc.free(key); + self.alloc.free(out_degree_keys); + } + const out_degree_deltas = try self.alloc.alloc(u64, out_degrees.count()); + defer self.alloc.free(out_degree_deltas); + var out_it = out_degrees.iterator(); + while (out_it.next()) |entry| { + out_degree_keys[initialized_out_degree_keys] = try self.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc(metric_name, job.job_id, .scan_edges_and_out_degree, claimed_page.iteration, claimed_page.page_id, claimed_page.attempt, entry.key_ptr.*); + out_degree_deltas[initialized_out_degree_keys] = entry.value_ptr.*; + initialized_out_degree_keys += 1; + } + try self.addU64DeltasInBatch(&batch, out_degree_keys, out_degree_deltas, prior_completed_units != 0); + var node_it = nodes.keyIterator(); + while (node_it.next()) |node_ptr| { + const key = try self.graphMetricBuildAttemptPageRankNodePartialKeyAlloc(metric_name, job.job_id, .scan_edges_and_out_degree, claimed_page.iteration, claimed_page.page_id, claimed_page.attempt, node_ptr.*); + defer self.alloc.free(key); + try putU64(&batch, key, 1); + } + _ = try self.updateGraphMetricBuildPageProgressInBatch( + &batch, + metric_name, + job.job_id, + .scan_edges_and_out_degree, + claimed_page.iteration, + claimed_page.page_id, + worker_id, + claimed_page.attempt, + cursor, + completed_units, + total_units, + ); + try batch.commit(); + } + + fn executePageRankInitializeBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executePageRankInitializeBuildPageWithLimit(metric_name, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executePageRankInitializeBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_initialize_units: ?u64, + ) !usize { + var range_lower: []u8 = ""; + defer if (range_lower.len > 0) self.alloc.free(range_lower); + var range_upper: []u8 = ""; + defer if (range_upper.len > 0) self.alloc.free(range_upper); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + if (execution_page.range_lower.len > 0) range_lower = try self.alloc.dupe(u8, execution_page.range_lower); + if (execution_page.range_upper.len > 0) range_upper = try self.alloc.dupe(u8, execution_page.range_upper); + if (execution_page.cursor.len > 0) resume_cursor = try self.alloc.dupe(u8, execution_page.cursor); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricOrdinalNodesInRange( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + if (max_initialize_units) |limit| @intCast(limit) else null, + &nodes, + &slots, + ); + }; + + const total_nodes = try self.graphMetricBuildNodeCount(metric_name, job.job_id); + const initial_rank = if (total_nodes == 0) 0.0 else 1.0 / @as(f64, @floatFromInt(total_nodes)); + const seed_generation = try self.pinPageRankSeedGeneration(metric_name, job, page); + var initialized_nodes: usize = 0; + var out_degree_total: u64 = 0; + var rank_sum: f64 = 0.0; + var initialized = std.ArrayListUnmanaged(PageRankInitializeNode).empty; + defer initialized.deinit(self.alloc); + { + var read_txn = try self.beginReadReverseTxn(); + defer read_txn.abort(); + const out_degrees = try self.pageRankOutDegreesForNodesAlloc(&read_txn, metric_name, job.job_id, nodes.items); + defer self.alloc.free(out_degrees); + const prior_generation = seed_generation; + const seed_mass = try self.pageRankSeedMass(&read_txn, metric_name, job.job_id, seed_generation); + const prior_scores: ?[]?f64 = if (prior_generation == 0) + null + else + try self.graphMetricScoresInTxnAlloc(&read_txn, metric_name, prior_generation, nodes.items); + defer if (prior_scores) |scores| self.alloc.free(scores); + for (nodes.items, out_degrees, 0..) |node, out_degree, node_index| { + const seed = try metric_kernels.warm_start.normalized(if (prior_scores) |scores| scores[node_index] orelse 0 else 0, seed_mass, total_nodes); + out_degree_total += out_degree; + initialized_nodes += 1; + rank_sum += seed; + try initialized.append(self.alloc, .{ .node = node, .slot = slots.items[node_index], .out_degree = out_degree, .initial_rank = seed }); + } + } + try self.writePageRankInitializeOutputForAttempt(metric_name, job, page, initialized.items, initial_rank); + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(initialized_nodes)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + if (!reached_page_end) { + const cursor = if (nodes.items.len > 0) nodes.items[nodes.items.len - 1] else resume_cursor; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, completed_units, total_units); + return initialized_nodes; + } + const cursor = try std.fmt.allocPrint(self.alloc, "pagerank-initialize:nodes={d};rank_sum={d}", .{ initialized_nodes, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankInitializeFingerprint(page, initialized_nodes, out_degree_total, rank_sum); + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, complete_units, total_units); + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint); + return initialized_nodes; + } + + fn writePageRankInitializeOutputForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + initialized: []const PageRankInitializeNode, + initial_rank: f64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .initialize_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + if (!std.math.isFinite(initial_rank)) return error.InvalidGraphMetricScore; + try self.validateGraphMetricVectorManifest(&batch, metric_name, job.job_id); + const scores = try self.alloc.alloc(OrdinalMetricScore, initialized.len); + defer self.alloc.free(scores); + const degrees = try self.alloc.alloc(u64, initialized.len); + defer self.alloc.free(degrees); + for (initialized, 0..) |entry, i| { + scores[i] = .{ .node = entry.node, .slot = entry.slot, .score = entry.initial_rank orelse initial_rank }; + degrees[i] = entry.out_degree; + } + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "rank", 0, scores, null, null); + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "factor", 0, scores, degrees, null); + try batch.commit(); + } + + fn executePageRankContributionBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executePageRankContributionBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executePageRankContributionBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + return self.executeOrdinalContributionPage(metric_name, cfg, job, page, max_scan_units); + } + + fn writePageRankContributionAttemptPartialsForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + prior_completed_units: u64, + contributions: []const PackedF64Entry, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .iterate_contributions, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + var chunk_index: u64 = 0; + var start: usize = 0; + while (start < contributions.len) : (chunk_index += 1) { + const end = @min(start + graph_metric_build_adoption_page_units, contributions.len); + const key = try self.graphMetricBuildAttemptPageRankContributionChunkKeyAlloc( + metric_name, + job.job_id, + .iterate_contributions, + claimed_page.iteration, + claimed_page.page_id, + claimed_page.attempt, + prior_completed_units, + chunk_index, + ); + defer self.alloc.free(key); + const encoded = try self.encodePackedF64EntriesAlloc(contributions[start..end]); + defer self.alloc.free(encoded); + if (batch.get(key)) |prior| { + if (!std.mem.eql(u8, prior, encoded)) return error.GraphMetricBuildPageOutputMismatch; + } else |err| switch (err) { + error.NotFound => try batch.put(key, encoded), + else => return err, + } + start = end; + } + _ = try self.updateGraphMetricBuildPageProgressInBatch( + &batch, + metric_name, + job.job_id, + .iterate_contributions, + claimed_page.iteration, + claimed_page.page_id, + worker_id, + claimed_page.attempt, + cursor, + completed_units, + total_units, + ); + try batch.commit(); + } + + fn executePageRankReduceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executePageRankReduceBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executePageRankReduceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_reduce_units: ?u64, + ) !usize { + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_reduce_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + const total_nodes = try self.graphMetricBuildNodeCount(metric_name, job.job_id); + var reduced = std.ArrayListUnmanaged(OrdinalMetricScore).empty; + defer reduced.deinit(self.alloc); + const sink_mass: f64 = blk: { + var read_txn = try self.beginReadReverseTxn(); + defer read_txn.abort(); + if (try self.graphMetricReduceSummaryValue(&read_txn, metric_name, job.job_id, .reduce_ranks, page.iteration)) |summary| { + break :blk summary.rank_sum; + } + return error.InvalidGraphMetricBuildManifest; + }; + var contribution_sum: f64 = 0.0; + var rank_sum: f64 = 0.0; + if (total_nodes > 0) { + const node_count_f = @as(f64, @floatFromInt(total_nodes)); + const base = (1.0 - cfg.damping) / node_count_f; + const sink_contribution = cfg.damping * sink_mass / node_count_f; + var read_txn = try self.beginReadReverseTxn(); + defer read_txn.abort(); + const contribution_values = try self.readGraphMetricVectorSlotsAlloc(&read_txn, metric_name, job.job_id, "raw_rank", page.iteration, slots_list.items, true); + defer self.alloc.free(contribution_values); + for (contribution_values, slots_list.items) |contribution, slot| { + contribution_sum += contribution; + const next_rank = base + sink_contribution + contribution; + if (!std.math.isFinite(next_rank)) return error.InvalidGraphMetricScore; + rank_sum += next_rank; + try reduced.append(self.alloc, .{ .node = "", .score = next_rank, .slot = slot }); + } + } + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(reduced.items.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + const cursor = try std.fmt.allocPrint(self.alloc, "pagerank-reduce:nodes={d};rank_sum={d}", .{ reduced.items.len, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankReduceFingerprint(page, reduced.items.len, contribution_sum, rank_sum); + if (!reached_page_end) { + _ = try self.writeSingleVectorReduceRanksForAttempt(metric_name, job, page, worker_id, "", completed_units, total_units, reduced.items, 0, false); + return reduced.items.len; + } + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.writeSingleVectorReduceRanksForAttempt(metric_name, job, page, worker_id, cursor, complete_units, total_units, reduced.items, fingerprint, true); + return reduced.items.len; + } + + fn writeSingleVectorReduceRanksForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ranks: anytype, + output_fingerprint: u64, + complete_page: bool, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job.job_id, .reduce_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state == .complete) { + if (!complete_page or page.attempt != claimed_page.attempt) return error.GraphMetricBuildPageNotLeased; + if (page.output_fingerprint != output_fingerprint) return error.GraphMetricBuildPageOutputMismatch; + try batch.commit(); + return page; + } + var expected_page = claimed_page; + expected_page.worker_id = worker_id; + try self.validateGraphMetricBuildPageExecutionLease(expected_page, page); + const effective_total_units = if (total_units != 0) total_units else page.total_units; + if (effective_total_units != 0 and completed_units > effective_total_units) return error.InvalidGraphMetricBuildProgress; + const write_page_rank_factors = (self.metricConfig(metric_name) orelse return error.MetricNotReady).kind == .pagerank; + const out_degrees: ?[]u64 = if (write_page_rank_factors) blk: { + if (@hasField(@TypeOf(ranks[0]), "slot")) { + const slots = try self.alloc.alloc(u64, ranks.len); + defer self.alloc.free(slots); + for (ranks, 0..) |rank, i| slots[i] = rank.slot; + break :blk try self.graphMetricDegreeSlotsAlloc(&batch, metric_name, job.job_id, slots); + } + const nodes = try self.alloc.alloc([]const u8, ranks.len); + defer self.alloc.free(nodes); + for (ranks, 0..) |score, i| nodes[i] = score.node; + break :blk try self.pageRankOutDegreesForNodesAlloc(&batch, metric_name, job.job_id, nodes); + } else null; + defer if (out_degrees) |values| self.alloc.free(values); + try self.validateGraphMetricVectorManifest(&batch, metric_name, job.job_id); + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "rank", claimed_page.iteration + 1, ranks, null, if (claimed_page.iteration > 0) claimed_page.iteration - 1 else null); + if (write_page_rank_factors) + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "factor", claimed_page.iteration + 1, ranks, out_degrees, claimed_page.iteration); + // Immutable inputs remain replayable until the consumer barrier. + + page.completed_units = completed_units; + page.total_units = effective_total_units; + page.cursor = cursor; + page.last_error = ""; + if (complete_page) { + page.state = .complete; + page.worker_id = worker_id; + page.lease_expires_at_ms = 0; + page.completed_units = if (effective_total_units != 0) effective_total_units else completed_units; + page.cursor = ""; + page.output_fingerprint = output_fingerprint; + } + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn executeEigenvectorInitializeBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeEigenvectorInitializeBuildPageWithLimit(metric_name, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeEigenvectorInitializeBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_initialize_units: ?u64, + ) !usize { + var range_lower: []u8 = ""; + defer if (range_lower.len > 0) self.alloc.free(range_lower); + var range_upper: []u8 = ""; + defer if (range_upper.len > 0) self.alloc.free(range_upper); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + if (execution_page.range_lower.len > 0) range_lower = try self.alloc.dupe(u8, execution_page.range_lower); + if (execution_page.range_upper.len > 0) range_upper = try self.alloc.dupe(u8, execution_page.range_upper); + if (execution_page.cursor.len > 0) resume_cursor = try self.alloc.dupe(u8, execution_page.cursor); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricOrdinalNodesInRange( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + if (max_initialize_units) |limit| @intCast(limit) else null, + &nodes, + &slots, + ); + }; + + const total_nodes = try self.graphMetricBuildNodeCount(metric_name, job.job_id); + const initial_rank = if (total_nodes == 0) 0.0 else 1.0 / @sqrt(@as(f64, @floatFromInt(total_nodes))); + var initialized_nodes: usize = 0; + var rank_sum: f64 = 0.0; + var initialized = std.ArrayListUnmanaged(PageRankInitializeNode).empty; + defer initialized.deinit(self.alloc); + { + // Spectral seeds must cover components that previously had zero + // score; use the same canonical cold start as the shared kernel. + for (nodes.items, slots.items) |node, slot| { + const seed = initial_rank; + initialized_nodes += 1; + rank_sum += seed; + try initialized.append(self.alloc, .{ .node = node, .slot = slot, .initial_rank = seed }); + } + } + try self.writeEigenvectorInitializeOutputForAttempt(metric_name, job, page, initialized.items, initial_rank); + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(initialized_nodes)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + if (!reached_page_end) { + const cursor = if (nodes.items.len > 0) nodes.items[nodes.items.len - 1] else resume_cursor; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, completed_units, total_units); + return initialized_nodes; + } + const cursor = try std.fmt.allocPrint(self.alloc, "eigenvector-initialize:nodes={d};rank_sum={d}", .{ initialized_nodes, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankInitializeFingerprint(page, initialized_nodes, 0, rank_sum); + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, complete_units, total_units); + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint); + return initialized_nodes; + } + + fn writeEigenvectorInitializeOutputForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + initialized_nodes: []const PageRankInitializeNode, + initial_rank: f64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .initialize_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + if (!std.math.isFinite(initial_rank)) return error.InvalidGraphMetricScore; + try self.validateGraphMetricVectorManifest(&batch, metric_name, job.job_id); + const scores = try self.alloc.alloc(OrdinalMetricScore, initialized_nodes.len); + defer self.alloc.free(scores); + for (initialized_nodes, 0..) |entry, i| scores[i] = .{ .node = entry.node, .slot = entry.slot, .score = entry.initial_rank orelse initial_rank }; + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "rank", 0, scores, null, null); + try batch.commit(); + } + + fn executeEigenvectorContributionBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeEigenvectorContributionBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executeEigenvectorContributionBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + return self.executeOrdinalContributionPage(metric_name, cfg, job, page, max_scan_units); + } + + fn executeEigenvectorReduceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeEigenvectorReduceBuildPageWithLimit(metric_name, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeEigenvectorReduceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_reduce_units: ?u64, + ) !usize { + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_reduce_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + const norm_sq: f64 = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.graphMetricReduceSummaryValue(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration)) |summary| { + break :blk summary.rank_sum; + } + return error.InvalidGraphMetricBuildManifest; + }; + const norm = @sqrt(norm_sq); + if (!std.math.isFinite(norm)) return error.InvalidGraphMetricScore; + + var reduced = std.ArrayListUnmanaged(OrdinalMetricScore).empty; + defer reduced.deinit(self.alloc); + var contribution_sum: f64 = 0.0; + var rank_sum: f64 = 0.0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const contributions = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "raw_rank", page.iteration, slots_list.items, true); + defer self.alloc.free(contributions); + for (contributions, slots_list.items) |contribution, slot| { + contribution_sum += contribution; + const next_rank = if (norm > 0.0) contribution / norm else 0.0; + if (!std.math.isFinite(next_rank)) return error.InvalidGraphMetricScore; + rank_sum += next_rank; + try reduced.append(self.alloc, .{ .node = "", .score = next_rank, .slot = slot }); + } + } + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(reduced.items.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + const cursor = try std.fmt.allocPrint(self.alloc, "eigenvector-reduce:nodes={d};rank_sum={d}", .{ reduced.items.len, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankReduceFingerprint(page, reduced.items.len, contribution_sum, rank_sum); + if (!reached_page_end) { + _ = try self.writeSingleVectorReduceRanksForAttempt(metric_name, job, page, worker_id, "", completed_units, total_units, reduced.items, 0, false); + return reduced.items.len; + } + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.writeSingleVectorReduceRanksForAttempt(metric_name, job, page, worker_id, cursor, complete_units, total_units, reduced.items, fingerprint, true); + return reduced.items.len; + } + + fn executeHitsInitializeBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeHitsInitializeBuildPageWithLimit(metric_name, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeHitsInitializeBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_initialize_units: ?u64, + ) !usize { + var range_lower: []u8 = ""; + defer if (range_lower.len > 0) self.alloc.free(range_lower); + var range_upper: []u8 = ""; + defer if (range_upper.len > 0) self.alloc.free(range_upper); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + if (execution_page.range_lower.len > 0) range_lower = try self.alloc.dupe(u8, execution_page.range_lower); + if (execution_page.range_upper.len > 0) range_upper = try self.alloc.dupe(u8, execution_page.range_upper); + if (execution_page.cursor.len > 0) resume_cursor = try self.alloc.dupe(u8, execution_page.cursor); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricOrdinalNodesInRange( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + if (max_initialize_units) |limit| @intCast(limit) else null, + &nodes, + &slots, + ); + }; + + const total_nodes = try self.graphMetricBuildNodeCount(metric_name, job.job_id); + const initial_rank = if (total_nodes == 0) 0.0 else 1.0 / @sqrt(@as(f64, @floatFromInt(total_nodes))); + var initialized_nodes: usize = 0; + var rank_sum: f64 = 0.0; + var initialized = std.ArrayListUnmanaged(PageRankInitializeNode).empty; + defer initialized.deinit(self.alloc); + { + for (nodes.items, slots.items) |node, slot| { + const authority_seed = initial_rank; + const hub_seed = initial_rank; + initialized_nodes += 1; + rank_sum += authority_seed + hub_seed; + try initialized.append(self.alloc, .{ .node = node, .slot = slot, .initial_rank = authority_seed, .secondary_rank = hub_seed }); + } + } + try self.writeHitsInitializeOutputForAttempt(metric_name, job, page, initialized.items, initial_rank); + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(initialized_nodes)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + if (!reached_page_end) { + const cursor = if (nodes.items.len > 0) nodes.items[nodes.items.len - 1] else resume_cursor; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, completed_units, total_units); + return initialized_nodes; + } + const cursor = try std.fmt.allocPrint(self.alloc, "hits-initialize:nodes={d};rank_sum={d}", .{ initialized_nodes, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankInitializeFingerprint(page, initialized_nodes, 0, rank_sum); + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, cursor, complete_units, total_units); + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .initialize_ranks, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint); + return initialized_nodes; + } + + fn writeHitsInitializeOutputForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + initialized_nodes: []const PageRankInitializeNode, + initial_rank: f64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .initialize_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + if (!std.math.isFinite(initial_rank)) return error.InvalidGraphMetricScore; + try self.validateGraphMetricVectorManifest(&batch, metric_name, job.job_id); + const scores = try self.alloc.alloc(OrdinalMetricScore, initialized_nodes.len); + defer self.alloc.free(scores); + for (initialized_nodes, 0..) |entry, i| scores[i] = .{ .node = entry.node, .slot = entry.slot, .score = entry.initial_rank orelse initial_rank }; + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "authority", 0, scores, null, null); + for (initialized_nodes, 0..) |entry, i| scores[i].score = entry.secondary_rank orelse initial_rank; + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, "hub", 0, scores, null, null); + try batch.commit(); + } + + fn executeHitsContributionBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeHitsContributionBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executeHitsContributionBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + return self.executeOrdinalContributionPage(metric_name, cfg, job, page, max_scan_units); + } + + fn hitsAuthorityRankForNode(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, iteration: u32, authority_norm: f64, node: []const u8) !f64 { + const contribution = try self.aggregatePageRankContributionForNode(txn, metric_name, job_id, iteration, node); + if (!std.math.isFinite(contribution)) return error.InvalidGraphMetricScore; + return if (authority_norm > 0.0) contribution / authority_norm else 0.0; + } + + fn readHitsHubRawForNode(self: *GraphIndex, txn: anytype, metric_name: []const u8, job_id: u64, iteration: u32, node: []const u8) !f64 { + return try self.aggregateHitsHubRawForNode(txn, metric_name, job_id, iteration, node); + } + + const HitsHubRawSummary = struct { + count: usize = 0, + norm: f64 = 0.0, + raw_fingerprint: u64 = 0, + fingerprint: u64 = 0, + }; + + fn putHitsHubRawSummaryInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + summary: HitsHubRawSummary, + ) !void { + const key = try self.graphMetricBuildHitsHubRawSummaryKeyAlloc(metric_name, job_id, iteration); + defer self.alloc.free(key); + var encoded: [32]u8 = undefined; + std.mem.writeInt(u64, encoded[0..8], @intCast(summary.count), .little); + std.mem.writeInt(u64, encoded[8..16], @bitCast(summary.norm), .little); + std.mem.writeInt(u64, encoded[16..24], summary.raw_fingerprint, .little); + std.mem.writeInt(u64, encoded[24..32], summary.fingerprint, .little); + try batch.put(key, &encoded); + } + + fn hitsHubRawSummary( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) !?HitsHubRawSummary { + const key = try self.graphMetricBuildHitsHubRawSummaryKeyAlloc(metric_name, job_id, iteration); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + if (raw.len != 16 and raw.len != 24 and raw.len != 32) return error.InvalidGraphMetricBuildManifest; + const count = std.mem.readInt(u64, raw[0..8], .little); + const norm = @as(f64, @bitCast(std.mem.readInt(u64, raw[8..16], .little))); + const raw_fingerprint = if (raw.len >= 32) std.mem.readInt(u64, raw[16..24], .little) else 0; + const fingerprint = if (raw.len >= 32) std.mem.readInt(u64, raw[24..32], .little) else if (raw.len >= 24) std.mem.readInt(u64, raw[16..24], .little) else 0; + if (!std.math.isFinite(norm)) return error.InvalidGraphMetricScore; + return .{ .count = @intCast(count), .norm = norm, .raw_fingerprint = raw_fingerprint, .fingerprint = fingerprint }; + } + + fn hitsHubRawNamespaceSummary( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + iteration: u32, + ) !HitsHubRawSummary { + const prefix = try self.graphMetricBuildHitsHubRawPrefixAlloc(metric_name, job_id, iteration); + defer self.alloc.free(prefix); + var count: usize = 0; + var norm_sq: f64 = 0.0; + var raw_fingerprint: u64 = 0; + var current_node: std.ArrayListUnmanaged(u8) = .empty; + defer current_node.deinit(self.alloc); + var node_sum: f64 = 0.0; + var node_correction: f64 = 0.0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + defer self.alloc.free(node); + if (current_node.items.len > 0 and !std.mem.eql(u8, current_node.items, node)) { + const raw_hub = node_sum + node_correction; + if (!std.math.isFinite(raw_hub)) return error.InvalidGraphMetricScore; + norm_sq += raw_hub * raw_hub; + raw_fingerprint ^= graphMetricHitsHubRawEntryFingerprint(current_node.items, raw_hub); + count += 1; + node_sum = 0.0; + node_correction = 0.0; + } + if (current_node.items.len == 0 or !std.mem.eql(u8, current_node.items, node)) { + current_node.clearRetainingCapacity(); + try current_node.appendSlice(self.alloc, node); + } + const shard = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(shard)) return error.InvalidGraphMetricScore; + const next = node_sum + shard; + node_correction += if (@abs(node_sum) >= @abs(shard)) (node_sum - next) + shard else (shard - next) + node_sum; + node_sum = next; + } + if (current_node.items.len > 0) { + const raw_hub = node_sum + node_correction; + if (!std.math.isFinite(raw_hub)) return error.InvalidGraphMetricScore; + norm_sq += raw_hub * raw_hub; + raw_fingerprint ^= graphMetricHitsHubRawEntryFingerprint(current_node.items, raw_hub); + count += 1; + } + const norm = @sqrt(norm_sq); + if (!std.math.isFinite(norm)) return error.InvalidGraphMetricScore; + return .{ .count = count, .norm = norm, .raw_fingerprint = raw_fingerprint }; + } + + fn executeHitsReduceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeHitsReduceBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeHitsReduceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_reduce_units: ?u64, + ) !usize { + _ = cfg; + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_reduce_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + const authority_norm_sq: f64 = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.graphMetricReduceSummaryValue(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration)) |summary| { + break :blk summary.rank_sum; + } + return error.InvalidGraphMetricBuildManifest; + }; + const authority_norm = @sqrt(authority_norm_sq); + if (!std.math.isFinite(authority_norm)) return error.InvalidGraphMetricScore; + + var reduced = std.ArrayListUnmanaged(OrdinalMetricScore).empty; + defer reduced.deinit(self.alloc); + var contribution_sum: f64 = 0.0; + var rank_sum: f64 = 0.0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const contributions = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "raw_rank", page.iteration, slots_list.items, true); + defer self.alloc.free(contributions); + for (contributions, slots_list.items) |contribution, slot| { + const authority = if (authority_norm > 0.0) contribution / authority_norm else 0.0; + if (!std.math.isFinite(authority)) return error.InvalidGraphMetricScore; + contribution_sum += authority; + rank_sum += authority; + try reduced.append(self.alloc, .{ .node = "", .score = authority, .slot = slot }); + } + } + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(reduced.items.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + const cursor = try std.fmt.allocPrint(self.alloc, "hits-authority-reduce:nodes={d};rank_sum={d}", .{ reduced.items.len, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankReduceFingerprint(page, reduced.items.len, contribution_sum, rank_sum); + if (!reached_page_end) { + _ = try self.writeHitsReduceRanksForAttempt(metric_name, job, page, .reduce_ranks, "authority", worker_id, "", completed_units, total_units, reduced.items, 0, false); + return reduced.items.len; + } + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.writeHitsReduceRanksForAttempt(metric_name, job, page, .reduce_ranks, "authority", worker_id, cursor, complete_units, total_units, reduced.items, fingerprint, true); + return reduced.items.len; + } + + fn writeHitsReduceRanksForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + phase: GraphMetricBuildPhase, + vector_name: []const u8, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ranks: anytype, + output_fingerprint: u64, + complete_page: bool, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job.job_id, phase, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state == .complete) { + if (!complete_page or page.attempt != claimed_page.attempt) return error.GraphMetricBuildPageNotLeased; + if (page.output_fingerprint != output_fingerprint) return error.GraphMetricBuildPageOutputMismatch; + try batch.commit(); + return page; + } + var expected_page = claimed_page; + expected_page.worker_id = worker_id; + try self.validateGraphMetricBuildPageExecutionLease(expected_page, page); + const effective_total_units = if (total_units != 0) total_units else page.total_units; + if (effective_total_units != 0 and completed_units > effective_total_units) return error.InvalidGraphMetricBuildProgress; + try self.validateGraphMetricVectorManifest(&batch, metric_name, job.job_id); + try self.writeGraphMetricVectorRows(&batch, metric_name, job.job_id, vector_name, claimed_page.iteration + 1, ranks, null, if (claimed_page.iteration > 0) claimed_page.iteration - 1 else null); + // Both lanes retain replayable inputs until the consumer barrier. + + page.completed_units = completed_units; + page.total_units = effective_total_units; + page.cursor = cursor; + page.last_error = ""; + if (complete_page) { + page.state = .complete; + page.worker_id = worker_id; + page.lease_expires_at_ms = 0; + page.completed_units = if (effective_total_units != 0) effective_total_units else completed_units; + page.cursor = ""; + page.output_fingerprint = output_fingerprint; + } + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + fn executeHitsHubContributionBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeHitsHubContributionBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executeHitsHubContributionBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + return self.executeOrdinalContributionPage(metric_name, cfg, job, page, max_scan_units); + } + + fn writeHitsHubRawAttemptPartialsForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + prior_completed_units: u64, + hub_raw: *std.StringHashMapUnmanaged(f64), + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + total_units: u64, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current_page = try self.metricBuildPage(&batch, metric_name, job.job_id, .hits_hub_contributions, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, current_page); + const raw_keys = try self.alloc.alloc([]u8, hub_raw.count()); + var initialized_keys: usize = 0; + defer { + for (raw_keys[0..initialized_keys]) |key| self.alloc.free(key); + self.alloc.free(raw_keys); + } + const raw_deltas = try self.alloc.alloc(f64, hub_raw.count()); + defer self.alloc.free(raw_deltas); + var it = hub_raw.iterator(); + while (it.next()) |entry| { + if (!std.math.isFinite(entry.value_ptr.*)) return error.InvalidGraphMetricScore; + raw_keys[initialized_keys] = try self.graphMetricBuildAttemptHitsHubRawKeyAlloc(metric_name, job.job_id, .hits_hub_contributions, claimed_page.iteration, claimed_page.page_id, claimed_page.attempt, entry.key_ptr.*); + raw_deltas[initialized_keys] = entry.value_ptr.*; + initialized_keys += 1; + } + try self.addF64DeltasInBatch(&batch, raw_keys, raw_deltas, prior_completed_units != 0); + _ = try self.updateGraphMetricBuildPageProgressInBatch( + &batch, + metric_name, + job.job_id, + .hits_hub_contributions, + claimed_page.iteration, + claimed_page.page_id, + worker_id, + claimed_page.attempt, + cursor, + completed_units, + total_units, + ); + try batch.commit(); + } + + fn ensureHitsHubRawSummaryInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + cfg: GraphMetricConfig, + job_id: u64, + iteration: u32, + ) !HitsHubRawSummary { + if (try self.hitsHubRawSummary(batch, metric_name, job_id, iteration)) |summary| { + const expected = graphMetricHitsHubRawSummaryFingerprint( + metric_name, + cfg, + job_id, + iteration, + 0.0, + summary.count, + summary.norm, + summary.raw_fingerprint, + ); + if (summary.fingerprint != expected) return error.InvalidGraphMetricBuildManifest; + return summary; + } + const raw_summary = try self.hitsHubRawNamespaceSummary(batch, metric_name, job_id, iteration); + const expected_fingerprint = graphMetricHitsHubRawSummaryFingerprint(metric_name, cfg, job_id, iteration, 0.0, raw_summary.count, raw_summary.norm, raw_summary.raw_fingerprint); + const summary: HitsHubRawSummary = .{ + .count = raw_summary.count, + .norm = raw_summary.norm, + .raw_fingerprint = raw_summary.raw_fingerprint, + .fingerprint = expected_fingerprint, + }; + try self.putHitsHubRawSummaryInBatch(batch, metric_name, job_id, iteration, summary); + return summary; + } + + fn ensureHitsHubRawSummaryForPageAttempt( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + worker_id: []const u8, + ) !HitsHubRawSummary { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var expected_page = claimed_page; + expected_page.worker_id = worker_id; + const page = try self.metricBuildPage(&batch, metric_name, job.job_id, .hits_hub_reduce_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(expected_page, page); + const summary = try self.ensureHitsHubRawSummaryInBatch(&batch, metric_name, cfg, job.job_id, claimed_page.iteration); + try batch.commit(); + return summary; + } + + fn executeHitsHubReduceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeHitsHubReduceBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeHitsHubReduceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_reduce_units: ?u64, + ) !usize { + _ = cfg; + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .hits_hub_reduce_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_reduce_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const hub_summary: HitsHubRawSummary = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.graphMetricReduceSummaryValue(&txn, metric_name, job.job_id, .hits_hub_reduce_ranks, page.iteration)) |summary| { + const norm = @sqrt(summary.rank_sum); + if (!std.math.isFinite(norm)) return error.InvalidGraphMetricScore; + break :blk .{ + .count = std.math.cast(usize, self.node_count) orelse std.math.maxInt(usize), + .norm = norm, + .raw_fingerprint = summary.output_fingerprint, + .fingerprint = summary.output_fingerprint, + }; + } + // Every newly planned HITS hub-reduce phase has a resumable + // summary dependency. Refuse a legacy/incomplete layout instead + // of falling back to a graph-sized scan in a write transaction. + return error.InvalidGraphMetricBuildManifest; + }; + + var reduced = std.ArrayListUnmanaged(OrdinalMetricScore).empty; + defer reduced.deinit(self.alloc); + var raw_sum: f64 = 0.0; + var rank_sum: f64 = 0.0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const raw_hubs = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "raw_hub", page.iteration, slots_list.items, true); + defer self.alloc.free(raw_hubs); + for (raw_hubs, slots_list.items) |raw_hub, slot| { + const hub = if (hub_summary.norm > 0.0) raw_hub / hub_summary.norm else 0.0; + if (!std.math.isFinite(hub)) return error.InvalidGraphMetricScore; + raw_sum += raw_hub; + rank_sum += hub; + try reduced.append(self.alloc, .{ .node = "", .score = hub, .slot = slot }); + } + } + + const completed_units_raw = prior_completed_units + @as(u64, @intCast(reduced.items.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + const cursor = try std.fmt.allocPrint(self.alloc, "hits-hub-reduce:nodes={d};rank_sum={d}", .{ reduced.items.len, rank_sum }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricHitsReduceFingerprint(page, reduced.items.len, raw_sum, rank_sum, hub_summary); + if (!reached_page_end) { + _ = try self.writeHitsReduceRanksForAttempt(metric_name, job, page, .hits_hub_reduce_ranks, "hub", worker_id, "", completed_units, total_units, reduced.items, 0, false); + return reduced.items.len; + } + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.writeHitsReduceRanksForAttempt(metric_name, job, page, .hits_hub_reduce_ranks, "hub", worker_id, cursor, complete_units, total_units, reduced.items, fingerprint, true); + return reduced.items.len; + } + + fn executeHitsConvergenceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + return try self.executeHitsConvergenceBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeHitsConvergenceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_check_units: ?u64, + ) !GraphMetricBuildPageExecutionResult { + var prior_completed_units: u64 = 0; + var prior_max_delta: f64 = 0.0; + var prior_total_delta: f64 = 0.0; + var prior_rank_sum: f64 = 0.0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .check_convergence, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + prior_max_delta = execution_page.max_delta; + prior_total_delta = execution_page.total_delta; + prior_rank_sum = execution_page.rank_sum; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_check_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + var checked_nodes: usize = 0; + var max_delta: f64 = 0.0; + var total_delta: f64 = 0.0; + var rank_sum: f64 = 0.0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const prior_authorities = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "authority", page.iteration, slots_list.items, true); + defer self.alloc.free(prior_authorities); + const next_authorities = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "authority", page.iteration + 1, slots_list.items, true); + defer self.alloc.free(next_authorities); + const prior_hubs = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "hub", page.iteration, slots_list.items, true); + defer self.alloc.free(prior_hubs); + const next_hubs = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "hub", page.iteration + 1, slots_list.items, true); + defer self.alloc.free(next_hubs); + for (prior_authorities, next_authorities, prior_hubs, next_hubs) |prior_authority, next_authority, prior_hub, next_hub| { + const authority_delta = @abs(next_authority - prior_authority); + const hub_delta = @abs(next_hub - prior_hub); + max_delta = @max(max_delta, @max(authority_delta, hub_delta)); + total_delta += authority_delta + hub_delta; + rank_sum += next_authority + next_hub; + checked_nodes += 1; + } + } + + max_delta = @max(prior_max_delta, max_delta); + total_delta += prior_total_delta; + rank_sum += prior_rank_sum; + if (!std.math.isFinite(max_delta) or !std.math.isFinite(total_delta) or !std.math.isFinite(rank_sum)) return error.InvalidGraphMetricScore; + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(checked_nodes)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + if (!reached_page_end) { + _ = try self.updateGraphMetricBuildConvergencePageProgressForAttempt(metric_name, job.job_id, page.iteration, page.page_id, worker_id, page.attempt, "", completed_units, total_units, max_delta, total_delta, rank_sum); + return .{ + .phase = .check_convergence, + .page_id = page.page_id, + .completed_page = false, + .completed_units = completed_units, + .total_units = total_units, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .score_count = checked_nodes, + }; + } + const cursor = try std.fmt.allocPrint(self.alloc, "hits-check:nodes={d};delta={d}", .{ checked_nodes, total_delta }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankConvergenceFingerprint(page, checked_nodes, max_delta, total_delta, rank_sum); + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .check_convergence, page.iteration, page.page_id, worker_id, page.attempt, cursor, complete_units, total_units); + const completed = try self.completeGraphMetricBuildConvergencePageForAttempt(metric_name, job.job_id, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint, max_delta, total_delta, rank_sum, total_delta <= cfg.tolerance); + return .{ + .phase = .check_convergence, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .score_count = checked_nodes, + }; + } + + fn executePageRankConvergenceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + return try self.executePageRankConvergenceBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executePageRankConvergenceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_check_units: ?u64, + ) !GraphMetricBuildPageExecutionResult { + var prior_completed_units: u64 = 0; + var prior_max_delta: f64 = 0.0; + var prior_total_delta: f64 = 0.0; + var prior_rank_sum: f64 = 0.0; + var total_units = page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .check_convergence, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + prior_max_delta = execution_page.max_delta; + prior_total_delta = execution_page.total_delta; + prior_rank_sum = execution_page.rank_sum; + total_units = execution_page.total_units; + } + + var slots_list = std.ArrayListUnmanaged(u64).empty; + defer slots_list.deinit(self.alloc); + const reached_page_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + break :blk try self.collectGraphMetricPageSlots(&txn, metric_name, job.job_id, page, prior_completed_units, if (max_check_units) |limit| @intCast(limit) else null, &total_units, &slots_list); + }; + + var checked_nodes: usize = 0; + var max_delta: f64 = 0.0; + var total_delta: f64 = 0.0; + var rank_sum: f64 = 0.0; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const prior_ranks = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "rank", page.iteration, slots_list.items, true); + defer self.alloc.free(prior_ranks); + const next_ranks = try self.readGraphMetricVectorSlotsAlloc(&txn, metric_name, job.job_id, "rank", page.iteration + 1, slots_list.items, true); + defer self.alloc.free(next_ranks); + for (prior_ranks, next_ranks) |prior_rank, next_rank| { + const delta = @abs(next_rank - prior_rank); + max_delta = @max(max_delta, delta); + total_delta += delta; + rank_sum += next_rank; + checked_nodes += 1; + } + } + + max_delta = @max(prior_max_delta, max_delta); + total_delta += prior_total_delta; + rank_sum += prior_rank_sum; + if (!std.math.isFinite(max_delta) or !std.math.isFinite(total_delta) or !std.math.isFinite(rank_sum)) return error.InvalidGraphMetricScore; + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(checked_nodes)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + const cursor = try std.fmt.allocPrint(self.alloc, "pagerank-check:nodes={d};delta={d}", .{ checked_nodes, total_delta }); + defer self.alloc.free(cursor); + const fingerprint = graphMetricPageRankConvergenceFingerprint(page, checked_nodes, max_delta, total_delta, rank_sum); + if (!reached_page_end) { + _ = try self.updateGraphMetricBuildConvergencePageProgressForAttempt(metric_name, job.job_id, page.iteration, page.page_id, worker_id, page.attempt, "", completed_units, total_units, max_delta, total_delta, rank_sum); + return .{ + .phase = .check_convergence, + .page_id = page.page_id, + .completed_page = false, + .completed_units = completed_units, + .total_units = total_units, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .score_count = checked_nodes, + }; + } + const complete_units = if (total_units != 0) total_units else completed_units; + _ = try self.updateGraphMetricBuildPageProgressForAttempt(metric_name, job.job_id, .check_convergence, page.iteration, page.page_id, worker_id, page.attempt, cursor, complete_units, total_units); + const completed = try self.completeGraphMetricBuildConvergencePageForAttempt(metric_name, job.job_id, page.iteration, page.page_id, worker_id, page.attempt, complete_units, fingerprint, max_delta, total_delta, rank_sum, total_delta <= cfg.tolerance); + return .{ + .phase = .check_convergence, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .max_delta = max_delta, + .total_delta = total_delta, + .rank_sum = rank_sum, + .score_count = checked_nodes, + }; + } + + fn executeDegreeScanBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeDegreeScanBuildPageWithLimit(metric_name, cfg, job, page, graph_metric_build_checkpoint_scan_units); + } + + fn executeDegreeScanBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_scan_units: ?u64, + ) !usize { + if (try self.graphMetricBuildPageAdoptionFingerprint(metric_name, job.job_id, page)) |fingerprint| { + const adoption = try self.adoptGraphMetricAttemptOutputPage(metric_name, cfg.kind, job.job_id, page); + if (adoption.reached_end) { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, page.phase, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + } + return adoption.adopted; + } + var compiled_filter = try CompiledGraphMetricEdgeFilter.init(self.alloc, cfg.edge_filter); + defer compiled_filter.deinit(self.alloc); + var map = std.StringHashMapUnmanaged(usize).empty; + defer map.deinit(self.alloc); + var nodes = std.ArrayListUnmanaged(DegreeNode).empty; + defer { + self.freeDegreeNodes(nodes.items); + nodes.deinit(self.alloc); + } + + var scanned_units: u64 = 0; + var visited_units: u64 = 0; + var prior_completed_units: u64 = 0; + var page_attempt = page.attempt; + var reached_page_end = true; + var last_scanned_key: std.ArrayListUnmanaged(u8) = .empty; + defer last_scanned_key.deinit(self.alloc); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + page_attempt = execution_page.attempt; + var cur = try typed_edges.Cursor.init(self.alloc, &txn, cfg.edge_filter, execution_page.range_lower, execution_page.range_upper, execution_page.cursor); + defer cur.deinit(); + while (try cur.next()) |entry| { + if (max_scan_units) |limit| { + if (visited_units >= limit) { + reached_page_end = false; + break; + } + } + visited_units += 1; + last_scanned_key.clearRetainingCapacity(); + try last_scanned_key.appendSlice(self.alloc, entry.cursor); + var parsed = (try parseMetricReverseEdgeKeyView(self.alloc, entry.key, self.index_name)) orelse continue; + defer parsed.deinit(self.alloc); + scanned_units += 1; + if (!compiled_filter.allows(parsed.edge_type.bytes)) continue; + const source_idx = try self.getOrPutDegreeNode(&map, &nodes, parsed.source.bytes); + const target_idx = try self.getOrPutDegreeNode(&map, &nodes, parsed.target.bytes); + nodes.items[source_idx].degree += 1; + nodes.items[target_idx].degree += 1; + } + } + + const scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + defer self.alloc.free(scores); + for (nodes.items, 0..) |node, i| { + scores[i] = .{ + .node = node.key, + .score = @floatFromInt(node.degree), + }; + } + const score_fingerprint = graphMetricScoresFingerprint(scores); + const completed_units_raw = prior_completed_units + scanned_units; + const completed_units = if (page.total_units != 0) @min(completed_units_raw, page.total_units) else completed_units_raw; + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + var adoption_cursor: []u8 = ""; + defer if (adoption_cursor.len > 0) self.alloc.free(adoption_cursor); + if (reached_page_end) adoption_cursor = try self.graphMetricBuildAdoptionCursorAlloc(score_fingerprint); + const progress_cursor = if (reached_page_end) adoption_cursor else last_scanned_key.items; + const progress_units = if (reached_page_end and page.total_units != 0) page.total_units else completed_units; + { + var score_batch = try self.beginWriteReverseBatch(); + errdefer score_batch.abort(); + const current_page = try self.metricBuildPage(&score_batch, metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + var expected_page = page; + expected_page.attempt = page_attempt; + try self.validateGraphMetricBuildPageExecutionLease(expected_page, current_page); + const partial_keys = try self.alloc.alloc([]u8, nodes.items.len); + var initialized_keys: usize = 0; + defer { + for (partial_keys[0..initialized_keys]) |key| self.alloc.free(key); + self.alloc.free(partial_keys); + } + const degree_deltas = try self.alloc.alloc(u64, nodes.items.len); + defer self.alloc.free(degree_deltas); + for (nodes.items, 0..) |node, index| { + partial_keys[index] = try self.graphMetricBuildAttemptDegreePartialKeyAlloc(metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page_attempt, node.key); + degree_deltas[index] = node.degree; + initialized_keys += 1; + } + try self.addU64DeltasInBatch(&score_batch, partial_keys, degree_deltas, prior_completed_units != 0); + _ = try self.updateGraphMetricBuildPageProgressInBatch( + &score_batch, + metric_name, + job.job_id, + .scan_edges_and_out_degree, + page.iteration, + page.page_id, + worker_id, + page_attempt, + progress_cursor, + progress_units, + page.total_units, + ); + try score_batch.commit(); + } + if (!reached_page_end) { + return scores.len; + } + const complete_units = if (page.total_units != 0) page.total_units else completed_units; + var adopt_page = page; + adopt_page.attempt = page_attempt; + const adoption = try self.adoptGraphMetricAttemptOutputPage(metric_name, cfg.kind, job.job_id, adopt_page); + if (!adoption.reached_end) { + return scores.len; + } + _ = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, worker_id, page.attempt, complete_units, score_fingerprint); + return scores.len; + } + + fn executeDegreeReduceBuildPage( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !usize { + return try self.executeDegreeReduceBuildPageWithLimit(metric_name, job, page, graph_metric_build_checkpoint_reduce_units); + } + + fn executeDegreeReduceBuildPageWithLimit( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + max_reduce_units: ?u64, + ) !usize { + var totals = std.StringHashMapUnmanaged(u64).empty; + defer { + var key_it = totals.keyIterator(); + while (key_it.next()) |key_ptr| self.alloc.free(key_ptr.*); + totals.deinit(self.alloc); + } + + var partial_count: u64 = 0; + var prior_completed_units: u64 = 0; + var total_units = page.total_units; + var reached_page_end = true; + var last_reduced_node: []u8 = ""; + defer if (last_reduced_node.len > 0) self.alloc.free(last_reduced_node); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .reduce_ranks, page.iteration, page.page_id) orelse page; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + prior_completed_units = execution_page.completed_units; + total_units = execution_page.total_units; + const prefix = try self.graphMetricBuildDegreePartialPrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(prefix); + const seek_node = if (execution_page.cursor.len > 0) execution_page.cursor else execution_page.range_lower; + const seek_key = if (seek_node.len > 0) + try self.graphMetricBuildDegreePartialKeyAlloc(metric_name, job.job_id, seek_node, 0) + else + try self.alloc.dupe(u8, prefix); + defer self.alloc.free(seek_key); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(seek_key); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (entry.value.len != 8) return error.InvalidGraphMetricBuildManifest; + const value = std.mem.readInt(u64, entry.value[0..8], .little); + if (value == 0) return error.InvalidGraphMetricBuildManifest; + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + defer self.alloc.free(node); + if (execution_page.range_lower.len > 0 and std.mem.order(u8, node, execution_page.range_lower) == .lt) { + continue; + } + if (execution_page.range_upper.len > 0 and std.mem.order(u8, node, execution_page.range_upper) != .lt) { + break; + } + if (execution_page.cursor.len > 0 and std.mem.order(u8, node, execution_page.cursor) != .gt) { + continue; + } + if (!totals.contains(node)) { + if (max_reduce_units) |limit| { + if (totals.count() >= limit) { + reached_page_end = false; + break; + } + } + } + const result = try self.getOrPutOwnedStringMap(u64, &totals, node); + if (result.found_existing) { + result.value_ptr.* += value; + } else { + result.value_ptr.* = value; + try self.replaceOwnedBytes(&last_reduced_node, result.key); + } + partial_count += 1; + } + } + + const scores = try self.alloc.alloc(GraphMetricScore, totals.count()); + defer self.alloc.free(scores); + var idx: usize = 0; + var it = totals.iterator(); + while (it.next()) |entry| : (idx += 1) { + scores[idx] = .{ + .node = entry.key_ptr.*, + .score = @floatFromInt(entry.value_ptr.*), + }; + } + const score_fingerprint = graphMetricScoresFingerprint(scores); + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const reduce_cursor = try std.fmt.allocPrint(self.alloc, "degree-reduce:partials={d};scores={d}", .{ partial_count, scores.len }); + defer self.alloc.free(reduce_cursor); + const completed_units_raw = prior_completed_units + @as(u64, @intCast(scores.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + if (!reached_page_end) { + _ = try self.writeDegreeReduceBuildPageWithScoresForAttempt(metric_name, job, page, worker_id, last_reduced_node, completed_units, scores, 0, false); + return scores.len; + } + _ = try self.writeDegreeReduceBuildPageWithScoresForAttempt(metric_name, job, page, worker_id, reduce_cursor, if (total_units != 0) total_units else completed_units, scores, score_fingerprint, true); + return scores.len; + } + + fn completeDegreeReduceBuildPageWithScoresForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + scores: []const GraphMetricScore, + output_fingerprint: u64, + ) !GraphMetricBuildPage { + return try self.writeDegreeReduceBuildPageWithScoresForAttempt(metric_name, job, claimed_page, worker_id, cursor, completed_units, scores, output_fingerprint, true); + } + + fn writeDegreeReduceBuildPageWithScoresForAttempt( + self: *GraphIndex, + metric_name: []const u8, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + worker_id: []const u8, + cursor: []const u8, + completed_units: u64, + scores: []const GraphMetricScore, + output_fingerprint: u64, + complete_page: bool, + ) !GraphMetricBuildPage { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job.job_id, .reduce_ranks, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + if (page.state == .complete) { + if (!complete_page or page.attempt != claimed_page.attempt) return error.GraphMetricBuildPageNotLeased; + if (page.output_fingerprint != output_fingerprint) return error.GraphMetricBuildPageOutputMismatch; + try batch.commit(); + return page; + } + var expected_page = claimed_page; + expected_page.worker_id = worker_id; + try self.validateGraphMetricBuildPageExecutionLease(expected_page, page); + if (page.total_units != 0 and completed_units > page.total_units) return error.InvalidGraphMetricBuildProgress; + + try self.putPlannedGraphMetricScorePageInBatch(&batch, metric_name, job, metric_name, scores); + + page.state = if (complete_page) .complete else .leased; + page.worker_id = worker_id; + page.lease_expires_at_ms = if (complete_page) 0 else page.lease_expires_at_ms; + page.completed_units = if (complete_page and page.total_units != 0) page.total_units else completed_units; + page.cursor = if (complete_page) "" else cursor; + page.last_error = ""; + page.output_fingerprint = if (complete_page) output_fingerprint else 0; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return page; + } + + /// Materializes one bounded node slice into an unpublished score + /// generation. The coordinator only flips the generation pointer after + /// every page completes, so readers never observe a partially populated + /// generation and coordinator work stays constant-size. + fn executeGraphMetricPublishMaterializationPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + claimed_page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + var range_lower: []u8 = ""; + defer if (range_lower.len > 0) self.alloc.free(range_lower); + var range_upper: []u8 = ""; + defer if (range_upper.len > 0) self.alloc.free(range_upper); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + var prior_completed_units: u64 = 0; + var total_units = claimed_page.total_units; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const page = try self.metricBuildPage(&txn, metric_name, job.job_id, .publish_generation, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, page); + if (page.range_kind != .nodes) return error.InvalidGraphMetricBuildPage; + if (page.range_lower.len > 0) range_lower = try self.alloc.dupe(u8, page.range_lower); + if (page.range_upper.len > 0) range_upper = try self.alloc.dupe(u8, page.range_upper); + if (page.cursor.len > 0) resume_cursor = try self.alloc.dupe(u8, page.cursor); + prior_completed_units = page.completed_units; + total_units = page.total_units; + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + const reached_end = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(self.alloc); + try self.validateGraphMetricVectorManifest(&txn, metric_name, job.job_id); + const complete = try self.collectSealedGraphMetricMembership( + &txn, + metric_name, + job.job_id, + range_lower, + range_upper, + resume_cursor, + graph_metric_build_checkpoint_reduce_units, + 1024 * 1024, + &nodes, + &slots, + ); + try self.validateGraphMetricOrdinalDictionary(&txn, metric_name, job.job_id, nodes.items, slots.items); + break :blk complete; + }; + + var primary_scores = std.ArrayListUnmanaged(GraphMetricScore).empty; + defer primary_scores.deinit(self.alloc); + var pair_scores = std.ArrayListUnmanaged(GraphMetricScore).empty; + defer pair_scores.deinit(self.alloc); + const rank_iteration = job.iteration + 1; + const pair_cfg = self.pairedHitsMetricConfig(cfg); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const primary_ranks = switch (cfg.kind) { + .pagerank, .eigenvector => try self.pageRankRanksForNodesAlloc(&txn, metric_name, job.job_id, rank_iteration, nodes.items, true), + .hits_authority, .hits_hub => try self.hitsRanksForNodesAlloc(&txn, metric_name, job.job_id, hitsVectorName(cfg.kind), rank_iteration, nodes.items, true), + .degree => return error.UnsupportedGraphMetricBuildPhase, + }; + defer self.alloc.free(primary_ranks); + const pair_ranks = if (pair_cfg) |pair| + try self.hitsRanksForNodesAlloc(&txn, metric_name, job.job_id, hitsVectorName(pair.kind), rank_iteration, nodes.items, true) + else + null; + defer if (pair_ranks) |ranks| self.alloc.free(ranks); + for (nodes.items, primary_ranks, 0..) |node, primary_rank, index| { + try primary_scores.append(self.alloc, .{ .node = node, .score = primary_rank }); + if (pair_ranks) |ranks| try pair_scores.append(self.alloc, .{ .node = node, .score = ranks[index] }); + } + } + + const worker_id = if (claimed_page.worker_id.len != 0) claimed_page.worker_id else graph_metric_local_build_worker_id; + const completed_units_raw = prior_completed_units + @as(u64, @intCast(nodes.items.len)); + const completed_units = if (total_units != 0) @min(completed_units_raw, total_units) else completed_units_raw; + var hasher = std.hash.Wyhash.init(0xC412_519E_2D3F_6A87); + graphMetricConfigFingerprintHashU64(&hasher, claimed_page.page_id); + graphMetricConfigFingerprintHashU64(&hasher, rank_iteration); + graphMetricConfigFingerprintHashU64(&hasher, graphMetricScoresFingerprint(primary_scores.items)); + graphMetricConfigFingerprintHashU64(&hasher, graphMetricScoresFingerprint(pair_scores.items)); + const output_fingerprint = hasher.final(); + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try self.metricBuildPage(&batch, metric_name, job.job_id, .publish_generation, claimed_page.iteration, claimed_page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(claimed_page, page); + const primary_prior = try self.plannedGraphMetricPriorScoresAlloc(&batch, metric_name, job.score_generation, primary_scores.items); + defer self.alloc.free(primary_prior); + const pair_prior = if (pair_cfg) |pair| try self.plannedGraphMetricPriorScoresAlloc(&batch, pair.name, job.score_generation, pair_scores.items) else null; + defer if (pair_prior) |prior| self.alloc.free(prior); + try self.putPlannedGraphMetricScorePageWithPrior(&batch, metric_name, job, metric_name, primary_scores.items, primary_prior); + if (pair_cfg) |pair| try self.putPlannedGraphMetricScorePageWithPrior(&batch, metric_name, job, pair.name, pair_scores.items, pair_prior.?); + + page.completed_units = completed_units; + page.total_units = total_units; + page.last_error = ""; + if (reached_end) { + page.state = .complete; + page.worker_id = worker_id; + page.lease_expires_at_ms = 0; + page.cursor = ""; + page.completed_units = if (total_units != 0) total_units else completed_units; + page.output_fingerprint = if (output_fingerprint == 0) 1 else output_fingerprint; + } else { + page.cursor = if (nodes.items.len > 0) nodes.items[nodes.items.len - 1] else resume_cursor; + } + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, page); + try batch.commit(); + return .{ + .phase = .publish_generation, + .page_id = claimed_page.page_id, + .completed_page = reached_end, + .completed_units = page.completed_units, + .total_units = total_units, + .output_fingerprint = page.output_fingerprint, + .score_count = primary_scores.items.len, + }; + } + + fn executeGraphMetricCleanupBuildPage( + self: *GraphIndex, + metric_name: []const u8, + kind: GraphMetricKind, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + var cleanup_page = page; + var page_cursor: []u8 = ""; + defer if (page_cursor.len > 0) self.alloc.free(page_cursor); + var page_output_prefix: []u8 = ""; + defer if (page_output_prefix.len > 0) self.alloc.free(page_output_prefix); + var page_worker_id: []u8 = ""; + defer if (page_worker_id.len > 0) self.alloc.free(page_worker_id); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const execution_page = try self.metricBuildPage(&txn, metric_name, job.job_id, .cleanup_old_generations, page.iteration, page.page_id) orelse + return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, execution_page); + if (execution_page.cursor.len > 0) page_cursor = try self.alloc.dupe(u8, execution_page.cursor); + if (execution_page.output_prefix.len > 0) page_output_prefix = try self.alloc.dupe(u8, execution_page.output_prefix); + if (execution_page.worker_id.len > 0) page_worker_id = try self.alloc.dupe(u8, execution_page.worker_id); + cleanup_page = execution_page; + } + cleanup_page.cursor = page_cursor; + cleanup_page.output_prefix = page_output_prefix; + cleanup_page.worker_id = page_worker_id; + const worker_id = if (cleanup_page.worker_id.len != 0) cleanup_page.worker_id else graph_metric_local_build_worker_id; + const cleanup_cursor = try std.fmt.allocPrint(self.alloc, "cleanup-job:{d}:page:{d}", .{ job.job_id, page.page_id }); + defer self.alloc.free(cleanup_cursor); + const job_namespace_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job.job_id); + defer self.alloc.free(job_namespace_prefix); + const is_final_cleanup = graphMetricBuildCleanupPageIsFinal(kind, cleanup_page); + const cleanup_prefix = try self.graphMetricBuildCleanupPagePrefixAlloc(metric_name, kind, job.job_id, cleanup_page); + defer self.alloc.free(cleanup_prefix); + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var delete_page = try self.deleteKeysWithPrefixPageInBatch(&batch, cleanup_prefix, cleanup_page.cursor, graph_metric_build_cleanup_delete_page_units); + defer delete_page.deinit(self.alloc); + const completed_units_raw = cleanup_page.completed_units + @as(u64, @intCast(delete_page.removed)); + if (!delete_page.reached_end) { + var progress_page = cleanup_page; + progress_page.state = .leased; + progress_page.worker_id = worker_id; + progress_page.cursor = delete_page.cursor; + progress_page.completed_units = completed_units_raw; + progress_page.last_error = ""; + progress_page.output_prefix = cleanup_prefix; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, progress_page); + var progress_job = job; + progress_job.updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + progress_job.worker_id = worker_id; + progress_job.cursor = delete_page.cursor; + progress_job.completed_units = completed_units_raw; + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, progress_job); + + try batch.commit(); + return .{ + .phase = .cleanup_old_generations, + .page_id = page.page_id, + .completed_page = false, + .completed_units = completed_units_raw, + .total_units = cleanup_page.total_units, + .score_count = delete_page.removed, + }; + } + + if (is_final_cleanup) { + _ = try self.deleteGraphMetricBuildJobNamespaceInBatch(&batch, metric_name, job.job_id); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .started_at_ms = job.started_at_ms, + .updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .lease_expires_at_ms = 0, + .phase = .complete, + .iteration = job.iteration, + .worker_id = job.worker_id, + .completed_units = cleanup_page.total_units, + .total_units = cleanup_page.total_units, + }); + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + batch.delete(lease_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } else { + const completed_page = GraphMetricBuildPage{ + .job_id = job.job_id, + .phase = .cleanup_old_generations, + .iteration = page.iteration, + .page_id = page.page_id, + .state = .complete, + .range_kind = .job_control, + .output_prefix = cleanup_prefix, + .worker_id = worker_id, + .cursor = cleanup_cursor, + .completed_units = cleanup_page.total_units, + .total_units = cleanup_page.total_units, + .output_fingerprint = graphMetricBuildJobId(metric_name, job.target_generation, job.started_at_ms) ^ page.page_id ^ completed_units_raw, + }; + try self.putGraphMetricBuildPageInBatch(&batch, metric_name, completed_page); + var progress_job = job; + progress_job.updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + progress_job.worker_id = worker_id; + progress_job.cursor = cleanup_cursor; + progress_job.completed_units = page.page_id + 1; + progress_job.total_units = @intCast(graphMetricBuildCleanupPageCount(kind)); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, progress_job); + } + try batch.commit(); + return .{ + .phase = .cleanup_old_generations, + .page_id = page.page_id, + .completed_page = true, + .completed_units = cleanup_page.total_units, + .total_units = cleanup_page.total_units, + .score_count = delete_page.removed, + .published = is_final_cleanup, + .completed_build = is_final_cleanup, + }; + } + + fn graphMetricBuildPhaseHasPageExecutor(kind: GraphMetricKind, phase: GraphMetricBuildPhase) bool { + return switch (kind) { + .degree => switch (phase) { + .prepare_generation, + .scan_edges_and_out_degree, + .reduce_ranks, + .cleanup_old_generations, + => true, + else => false, + }, + .pagerank => switch (phase) { + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + .publish_generation, + .cleanup_old_generations, + => true, + else => false, + }, + .eigenvector => switch (phase) { + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + .cleanup_old_generations, + => true, + else => false, + }, + .hits_authority, + .hits_hub, + => switch (phase) { + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + .publish_generation, + .cleanup_old_generations, + => true, + else => false, + }, + }; + } + + fn executeGraphMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + // A topology task uses the mature page lease/checkpoint machinery, but + // owns no numerical vectors or publication. Only membership census and + // adjacency packing run before the immutable owner is sealed. + if (self.topology_preparation_only and page.range_kind == .nodes and + (page.phase == .initialize_ranks or page.phase == .reduce_ranks or page.phase == .hits_hub_reduce_ranks)) + { + const completed = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, page.phase, page.iteration, page.page_id, page.worker_id, page.attempt, page.total_units, 1); + return .{ .phase = page.phase, .page_id = page.page_id, .completed_page = true, .completed_units = completed.completed_units, .total_units = completed.total_units, .output_fingerprint = completed.output_fingerprint }; + } + return switch (cfg.kind) { + .degree => try self.executeDegreeMetricBuildPage(metric_name, cfg, job, page), + .pagerank => try self.executePageRankMetricBuildPage(metric_name, cfg, job, page), + .eigenvector => try self.executeEigenvectorMetricBuildPage(metric_name, cfg, job, page), + .hits_authority, + .hits_hub, + => try self.executeHitsMetricBuildPage(metric_name, cfg, job, page), + }; + } + + fn executeEigenvectorMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + return switch (page.phase) { + .prepare_generation => blk: { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const fingerprint = graphMetricBuildJobId(metric_name, job.target_generation, job.started_at_ms); + const completed = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .prepare_generation, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + break :blk .{ + .phase = .prepare_generation, + .page_id = page.page_id, + .completed_page = true, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + }; + }, + .scan_edges_and_out_degree => blk: { + const node_count = try self.executePageRankScanBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .scan_edges_and_out_degree, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .initialize_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeEigenvectorInitializeBuildPage(metric_name, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .initialize_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .iterate_contributions => blk: { + const target_count = try self.executeEigenvectorContributionBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .iterate_contributions, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = target_count, + }; + }, + .reduce_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeEigenvectorReduceBuildPage(metric_name, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .reduce_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .check_convergence => try self.executePageRankConvergenceBuildPage(metric_name, cfg, job, page), + .publish_generation => try self.executeGraphMetricPublishMaterializationPage(metric_name, cfg, job, page), + .cleanup_old_generations => blk: { + break :blk try self.executeGraphMetricCleanupBuildPage(metric_name, cfg.kind, job, page); + }, + else => error.UnsupportedGraphMetricBuildPhase, + }; + } + + fn executeHitsMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + return switch (page.phase) { + .prepare_generation => blk: { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const fingerprint = graphMetricBuildJobId(metric_name, job.target_generation, job.started_at_ms); + const completed = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .prepare_generation, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + break :blk .{ + .phase = .prepare_generation, + .page_id = page.page_id, + .completed_page = true, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + }; + }, + .scan_edges_and_out_degree => blk: { + const node_count = try self.executePageRankScanBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .scan_edges_and_out_degree, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .initialize_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeHitsInitializeBuildPage(metric_name, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .initialize_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .iterate_contributions => blk: { + const target_count = try self.executeHitsContributionBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .iterate_contributions, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = target_count, + }; + }, + .reduce_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeHitsReduceBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .reduce_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .hits_hub_contributions => blk: { + const source_count = try self.executeHitsHubContributionBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .hits_hub_contributions, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = source_count, + }; + }, + .hits_hub_reduce_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeHitsHubReduceBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .hits_hub_reduce_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .check_convergence => try self.executeHitsConvergenceBuildPage(metric_name, cfg, job, page), + .publish_generation => try self.executeGraphMetricPublishMaterializationPage(metric_name, cfg, job, page), + .cleanup_old_generations => blk: { + break :blk try self.executeGraphMetricCleanupBuildPage(metric_name, cfg.kind, job, page); + }, + else => error.UnsupportedGraphMetricBuildPhase, + }; + } + + fn executePageRankMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + return switch (page.phase) { + .prepare_generation => blk: { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + const fingerprint = graphMetricBuildJobId(metric_name, job.target_generation, job.started_at_ms); + const completed = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .prepare_generation, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + break :blk .{ + .phase = .prepare_generation, + .page_id = page.page_id, + .completed_page = true, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + }; + }, + .scan_edges_and_out_degree => blk: { + const node_count = try self.executePageRankScanBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .scan_edges_and_out_degree, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .initialize_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executePageRankInitializeBuildPage(metric_name, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .initialize_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .iterate_contributions => blk: { + const target_count = try self.executePageRankContributionBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .iterate_contributions, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = target_count, + }; + }, + .reduce_ranks => blk: { + const node_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executePageRankReduceBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .reduce_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = node_count, + }; + }, + .check_convergence => try self.executePageRankConvergenceBuildPage(metric_name, cfg, job, page), + .publish_generation => try self.executeGraphMetricPublishMaterializationPage(metric_name, cfg, job, page), + .cleanup_old_generations => blk: { + break :blk try self.executeGraphMetricCleanupBuildPage(metric_name, cfg.kind, job, page); + }, + else => error.UnsupportedGraphMetricBuildPhase, + }; + } + + fn executeDegreeMetricBuildPage( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageExecutionResult { + const worker_id = if (page.worker_id.len != 0) page.worker_id else graph_metric_local_build_worker_id; + return switch (page.phase) { + .prepare_generation => blk: { + const fingerprint = graphMetricBuildJobId(metric_name, job.target_generation, job.started_at_ms); + const completed = try self.completeGraphMetricBuildPageForAttempt(metric_name, job.job_id, .prepare_generation, page.iteration, page.page_id, worker_id, page.attempt, page.total_units, fingerprint); + break :blk .{ + .phase = .prepare_generation, + .page_id = page.page_id, + .completed_page = true, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + }; + }, + .scan_edges_and_out_degree => blk: { + const score_count = try self.executeDegreeScanBuildPage(metric_name, cfg, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .scan_edges_and_out_degree, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = score_count, + }; + }, + .reduce_ranks => blk: { + const score_count = if (page.range_kind == .summary) + try self.executeGraphMetricReduceSummaryBuildPage(metric_name, cfg, job, page) + else + try self.executeDegreeReduceBuildPage(metric_name, job, page); + const completed = try self.readGraphMetricBuildPageCompletionSnapshotOrInput(metric_name, job.job_id, page); + break :blk .{ + .phase = .reduce_ranks, + .page_id = page.page_id, + .completed_page = completed.state == .complete, + .completed_units = completed.completed_units, + .total_units = completed.total_units, + .output_fingerprint = completed.output_fingerprint, + .score_count = score_count, + }; + }, + .cleanup_old_generations => blk: { + break :blk try self.executeGraphMetricCleanupBuildPage(metric_name, cfg.kind, job, page); + }, + else => error.UnsupportedGraphMetricBuildPhase, + }; + } + + fn readGraphMetricBuildPageCompletionSnapshotOrInput( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricBuildPageCompletionSnapshot { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const current = try self.metricBuildPage(&txn, metric_name, job_id, page.phase, page.iteration, page.page_id) orelse page; + return .{ + .state = current.state, + .completed_units = current.completed_units, + .total_units = current.total_units, + .output_fingerprint = current.output_fingerprint, + }; + } + + fn deleteScoreGeneration(self: *GraphIndex, batch: anytype, metric_name: []const u8, generation: u64) !void { + if (generation == 0) return; + const prefix = try self.graphMetricScorePrefixAlloc(metric_name, generation); + defer self.alloc.free(prefix); + _ = try self.deleteKeysWithPrefixInBatch(batch, prefix); + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(rank_prefix); + _ = try self.deleteKeysWithPrefixInBatch(batch, rank_prefix); + const meta_key = try self.graphMetricMetaKeyAlloc(metric_name, generation); + defer self.alloc.free(meta_key); + batch.delete(meta_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + const edge_filter_key = try self.graphMetricMetaEdgeFilterKeyAlloc(metric_name, generation); + defer self.alloc.free(edge_filter_key); + batch.delete(edge_filter_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + const config_fingerprint_key = try self.graphMetricMetaConfigFingerprintKeyAlloc(metric_name, generation); + defer self.alloc.free(config_fingerprint_key); + batch.delete(config_fingerprint_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + + fn enqueueRetiredScoreGenerationInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + generation: u64, + ) !void { + if (generation == 0) return; + const generation_key = try self.graphMetricRetiredScoreGenerationKeyAlloc(metric_name); + defer self.alloc.free(generation_key); + const existing = try readU64OrZero(batch, generation_key); + if (existing == generation) return; + if (existing != 0) { + const next_key = try self.graphMetricNextRetiredScoreGenerationKeyAlloc(metric_name); + defer self.alloc.free(next_key); + const next = try readU64OrZero(batch, next_key); + if (next == generation) return; + if (next != 0) return error.GraphMetricRetiredGenerationBacklog; + try putU64(batch, next_key, generation); + return; + } + try putU64(batch, generation_key, generation); + const phase_key = try self.graphMetricRetiredScoreCleanupPhaseKeyAlloc(metric_name); + defer self.alloc.free(phase_key); + try putU64(batch, phase_key, 1); + const cursor_key = try self.graphMetricRetiredScoreCleanupCursorKeyAlloc(metric_name); + defer self.alloc.free(cursor_key); + try batch.put(cursor_key, ""); + } + + fn scoreGenerationHasKeysInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, generation: u64) !bool { + const score_prefix = try self.graphMetricScorePrefixAlloc(metric_name, generation); + defer self.alloc.free(score_prefix); + if (try self.hasKeysWithPrefixInBatch(batch, score_prefix)) return true; + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(rank_prefix); + return try self.hasKeysWithPrefixInBatch(batch, rank_prefix); + } + + /// Reclaim at most one bounded page from the generations retired by atomic + /// pointer swaps. The two-slot queue absorbs a publish followed by an + /// operator action while cleanup catches up, then applies backpressure. + pub fn cleanupRetiredGraphMetricScoreGenerationPage(self: *GraphIndex, metric_name: []const u8) !bool { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const generation_key = try self.graphMetricRetiredScoreGenerationKeyAlloc(metric_name); + defer self.alloc.free(generation_key); + const next_generation_key = try self.graphMetricNextRetiredScoreGenerationKeyAlloc(metric_name); + defer self.alloc.free(next_generation_key); + const generation = try readU64OrZero(&batch, generation_key); + if (generation == 0) { + try batch.commit(); + return false; + } + const published = try self.metricPublishedGeneration(&batch, metric_name); + if (generation == published) return error.GraphMetricRetiredGenerationPublished; + const phase_key = try self.graphMetricRetiredScoreCleanupPhaseKeyAlloc(metric_name); + defer self.alloc.free(phase_key); + const phase = try readU64OrZero(&batch, phase_key); + const cursor_key = try self.graphMetricRetiredScoreCleanupCursorKeyAlloc(metric_name); + defer self.alloc.free(cursor_key); + const cursor = batch.get(cursor_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + + if (phase == 1 or phase == 2) { + const prefix = if (phase == 1) + try self.graphMetricScorePrefixAlloc(metric_name, generation) + else + try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(prefix); + var deleted = try self.deleteKeysWithPrefixPageInBatch(&batch, prefix, cursor, graph_metric_build_cleanup_delete_page_units); + defer deleted.deinit(self.alloc); + if (!deleted.reached_end) { + try batch.put(cursor_key, deleted.cursor); + try batch.commit(); + return true; + } + try putU64(&batch, phase_key, phase + 1); + try batch.put(cursor_key, ""); + try batch.commit(); + return true; + } + + inline for (.{ + try self.graphMetricMetaKeyAlloc(metric_name, generation), + try self.graphMetricMetaEdgeFilterKeyAlloc(metric_name, generation), + try self.graphMetricMetaConfigFingerprintKeyAlloc(metric_name, generation), + }) |key| { + defer self.alloc.free(key); + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + const next_generation = try readU64OrZero(&batch, next_generation_key); + if (next_generation != 0) { + try putU64(&batch, generation_key, next_generation); + try putU64(&batch, phase_key, 1); + try batch.put(cursor_key, ""); + batch.delete(next_generation_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + try batch.commit(); + return true; + } + inline for (.{ generation_key, phase_key, cursor_key, next_generation_key }) |key| { + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + try batch.commit(); + return true; + } + + fn countGraphMetricScoreGeneration(self: *GraphIndex, metric_name: []const u8, generation: u64) !usize { + if (generation == 0) return 0; + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const prefix = try self.graphMetricScorePrefixAlloc(metric_name, generation); + defer self.alloc.free(prefix); + var count: usize = 0; + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + count += 1; + } + return count; + } + + fn deleteKeysWithPrefixInBatch(self: *GraphIndex, batch: anytype, prefix: []const u8) !usize { + var keys = std.ArrayListUnmanaged([]u8).empty; + defer { + for (keys.items) |key| self.alloc.free(key); + keys.deinit(self.alloc); + } + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + try self.appendOwnedBytes(&keys, entry.key); + } + for (keys.items) |key| { + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + return keys.items.len; + } + + fn hasKeysWithPrefixInBatch(_: *GraphIndex, batch: anytype, prefix: []const u8) !bool { + var cur = try batch.openCursor(); + defer cur.close(); + const entry = try cur.seekAtOrAfter(prefix) orelse return false; + return std.mem.startsWith(u8, entry.key, prefix); + } + + fn deleteKeysWithPrefixPageInBatch( + self: *GraphIndex, + batch: anytype, + prefix: []const u8, + cursor: []const u8, + max_keys: usize, + ) !PrefixDeletePageResult { + var keys = std.ArrayListUnmanaged([]u8).empty; + defer { + for (keys.items) |key| self.alloc.free(key); + keys.deinit(self.alloc); + } + var last_key: []u8 = ""; + defer if (last_key.len > 0) self.alloc.free(last_key); + var reached_end = true; + var cur = try batch.openCursor(); + defer cur.close(); + const seek_key = if (cursor.len > 0) cursor else prefix; + var entry_opt = try cur.seekAtOrAfter(seek_key); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (cursor.len > 0 and std.mem.order(u8, entry.key, cursor) != .gt) continue; + if (max_keys != 0 and keys.items.len >= max_keys) { + reached_end = false; + break; + } + try self.replaceOwnedBytes(&last_key, entry.key); + try self.appendOwnedBytes(&keys, entry.key); + } + for (keys.items) |key| { + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + return .{ + .removed = keys.items.len, + .reached_end = reached_end, + .cursor = if (!reached_end and last_key.len > 0) try self.alloc.dupe(u8, last_key) else "", + }; + } + + pub fn queueGraphMetricBuild(self: *GraphIndex, metric: []const u8, generation: u64) !GraphMetricStatus { + const name = try self.graphMetricLifecycleOwnerName(metric); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try self.graphMetricControlKeyAlloc(&.{ name, "requested" }); + defer self.alloc.free(key); + if (try self.metricDisabled(&batch, name)) return error.GraphMetricDisabled; + if (try self.metricBuildJob(&batch, name)) |job| { + if (job.target_generation == generation and job.phase != .complete and job.retry_count == 0 and job.last_error.len == 0) { + if (try self.metricBuildLease(&batch, name)) |lease| { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + if (lease.job_id == job.job_id and lease.target_generation == generation and lease.lease_expires_at_ms > now_ms) { + batch.abort(); + return self.graphMetricStatus(metric); + } + } + } + } + if (try readU64OrZero(&batch, key) == generation and generation != 0) { + batch.abort(); + return self.graphMetricStatus(metric); + } + try putU64(&batch, key, generation); + try batch.commit(); + return self.graphMetricStatus(metric); + } + + pub fn graphMetricBuildRequested(self: *GraphIndex, metric: []const u8) !bool { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + return self.metricBuildRequestedInTxn(&txn, try self.graphMetricLifecycleOwnerName(metric)); + } + + fn metricBuildRequestedInTxn(self: *GraphIndex, txn: anytype, metric: []const u8) !bool { + const key = try self.graphMetricControlKeyAlloc(&.{ metric, "requested" }); + defer self.alloc.free(key); + if (txn.get(key)) |_| return true else |err| return if (err == error.NotFound) false else err; + } + + fn propagateTopologyTaskFailure(self: *GraphIndex, cfg: GraphMetricConfig, generation: u64, reason: []const u8) !void { + for (self.metric_configs) |candidate| { + // A compatible HITS pair owns one request and delivery marker. + // Reporting the hub separately could consume a retry accepted + // after the authority lifecycle already observed this failure. + if (!std.mem.eql(u8, candidate.name, self.graphMetricLifecycleOwnerConfig(candidate).name)) continue; + if (!graphMetricKindUsesIterativeBuild(candidate.kind) or !candidate.edge_filter.equivalent(cfg.edge_filter)) continue; + if (cfg.kind == .eigenvector and (candidate.kind == .hits_authority or candidate.kind == .hits_hub)) continue; + const status = try self.graphMetricSchedulerStatus(candidate.name, null); + if (status.state == .disabled or status.maintenance_paused or status.target_edge_generation != generation) continue; + const requested = try self.graphMetricBuildRequested(candidate.name); + if (!requested and (candidate.refresh != .background or status.failed_target_generation == generation or status.state == .fresh or status.state == .building)) continue; + try self.recordGraphMetricFailureReasonAtGeneration(candidate.name, reason, generation, cfg.name); + } + } + + fn topologyTaskKeyAlloc(alloc: Allocator, name: []const u8) ![]u8 { + _ = try topologyTaskNameIncarnation(name); + return std.fmt.allocPrint(alloc, "{s}{s}", .{ topology_task_prefix, name[topology_task_name_prefix.len..][0..64] }); + } + + fn topologyTaskNameIncarnation(name: []const u8) !u64 { + if (!std.mem.startsWith(u8, name, topology_task_name_prefix) or name.len != topology_task_name_prefix.len + 85 or name[topology_task_name_prefix.len + 64] != '/') return error.InvalidGraphMetricBuildManifest; + const incarnation = std.fmt.parseInt(u64, name[name.len - 20 ..], 10) catch return error.InvalidGraphMetricBuildManifest; + if (incarnation == 0) return error.InvalidGraphMetricBuildManifest; + return incarnation; + } + + fn topologyTaskIncarnation(raw: []const u8) !u64 { + if (raw.len < 49 or raw[8] > 3) return error.InvalidGraphMetricBuildManifest; + const incarnation = std.mem.readInt(u64, raw[9..17], .little); + if (incarnation == 0) return error.InvalidGraphMetricBuildManifest; + return incarnation; + } + + fn topologyTaskNameAlloc(alloc: Allocator, key: []const u8, raw: []const u8) ![]u8 { + if (!std.mem.startsWith(u8, key, topology_task_prefix) or key.len != topology_task_prefix.len + 64) return error.InvalidGraphMetricBuildManifest; + return std.fmt.allocPrint(alloc, "{s}{s}/{d:0>20}", .{ topology_task_name_prefix, key[topology_task_prefix.len..], try topologyTaskIncarnation(raw) }); + } + + fn topologyTaskConfigAlloc(alloc: Allocator, name: []const u8, raw: []const u8) !GraphMetricConfig { + if (try topologyTaskIncarnation(raw) != try topologyTaskNameIncarnation(name)) return error.GraphMetricBuildSuperseded; + var checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(raw[0 .. raw.len - 32], &checksum, .{}); + if (!std.mem.eql(u8, &checksum, raw[raw.len - 32 ..])) return error.InvalidGraphMetricBuildManifest; + return .{ .name = name, .kind = if (raw[8] & 1 != 0) .hits_authority else .eigenvector, .refresh = .manual, .max_iterations = 1, .edge_filter = (try decodeGraphMetricEdgeFilterAlloc(alloc, raw[17 .. raw.len - 32])) orelse return error.InvalidGraphMetricBuildManifest }; + } + + /// Admit a generation/filter preparation independently of numerical jobs. + /// All requests share one task per orientation; waiting metrics retain only + /// their durable request, not a build lease or a numerical admission slot. + pub fn prepareGraphMetricTopology(self: *GraphIndex, cfg: GraphMetricConfig, generation: u64) !bool { + return try self.prepareGraphMetricTopologyDetailed(cfg, generation) == .ready; + } + + pub const TopologyPreparationAdmission = enum { ready, queued, waiting }; + pub const max_pending_topology_tasks = 16; + + pub fn prepareGraphMetricTopologyDetailed(self: *GraphIndex, cfg: GraphMetricConfig, generation: u64) !TopologyPreparationAdmission { + if (!graphMetricKindUsesIterativeBuild(cfg.kind)) return .ready; + if (!try self.prepareGraphMetricPartitionForConfigStep(cfg, 4096)) return .waiting; + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + if (generation != try self.graphMetricFilterGeneration(&batch, cfg.edge_filter)) return error.GraphMetricBuildSnapshotChanged; + const filter = try topology_owner.filterDigest(temp, cfg.edge_filter); + const identity = topology_owner.identity(filter, try self.graphMetricPartitionPlanRaw(&batch, cfg)); + var reverse = cfg.kind == .hits_authority or cfg.kind == .hits_hub; + // Union cold dependencies before choosing the task identity: PageRank + // and HITS queued together share one bidirectional preparation. + for (self.metric_configs) |candidate| { + if (reverse) break; + if ((candidate.kind != .hits_authority and candidate.kind != .hits_hub) or !candidate.edge_filter.equivalent(cfg.edge_filter)) continue; + const owner_name = try self.graphMetricLifecycleOwnerName(candidate.name); + if (try self.metricDisabled(&batch, owner_name) or try self.metricMaintenancePaused(&batch, owner_name)) continue; + reverse = candidate.refresh == .background or try self.metricBuildRequestedInTxn(&batch, owner_name); + } + const ready = try topology_owner.readyKey(temp, identity, reverse); + if (batch.get(ready)) |_| { + batch.abort(); + return .ready; + } else |err| if (err != error.NotFound) return err; + const id = topology_owner.ownerId(identity, "preparation", @intFromBool(reverse)); + const key = try std.fmt.allocPrint(temp, "{s}{s}", .{ topology_task_prefix, id }); + if (batch.get(key)) |_| { + batch.abort(); + return .waiting; + } else |err| if (err != error.NotFound) return err; + // Preparation has its own bounded admission, independent of numerical + // leases. A full queue is backpressure, not a failed user request. + const queue_full = full: { + var cur = try batch.openCursor(); + defer cur.close(); + var entry = try cur.seekAtOrAfter(topology_task_prefix); + var count: usize = 0; + while (entry) |item| : (entry = try cur.next()) { + if (!std.mem.startsWith(u8, item.key, topology_task_prefix)) break; + count += 1; + if (count == max_pending_topology_tasks) break :full true; + } + break :full false; + }; + if (queue_full) { + batch.abort(); + return .waiting; + } + const incarnation = std.math.add(u64, try readU64OrZero(&batch, topology_task_incarnation_key), 1) catch return error.GraphMetricBuildBudgetExceeded; + try putU64(&batch, topology_task_incarnation_key, incarnation); + const raw = try temp.alloc(u8, 49 + graphMetricEdgeFilterEncodedLen(cfg.edge_filter)); + std.mem.writeInt(u64, raw[0..8], generation, .little); + raw[8] = @intFromBool(reverse); + std.mem.writeInt(u64, raw[9..17], incarnation, .little); + encodeGraphMetricEdgeFilter(cfg.edge_filter, raw[17 .. raw.len - 32]); + std.crypto.hash.sha2.Sha256.hash(raw[0 .. raw.len - 32], raw[raw.len - 32 ..][0..32], .{}); + try batch.put(key, raw); + try batch.commit(); + return .queued; + } + + /// One bounded index-scoped task step. The borrowed execution view has its + /// own control namespace and no owned backend handles or numerical cache. + /// It reuses durable page leases/CAS/recovery without depending on a user + /// metric's name, lifetime, parameters, or publication state. + pub fn runGraphMetricTopologyPreparationStep(self: *GraphIndex, worker: []const u8) !bool { + // One local preparation execution slot. Durable incarnations also + // fence retries against delayed workers on separate reopened handles. + if (!self.topology_preparation_mutex.tryLock()) return false; + defer self.topology_preparation_mutex.unlock(); + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const task = read: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = if (self.topology_preparation_cursor) |cursor| blk: { + const seek = try std.fmt.allocPrint(temp, "{s}{s}", .{ topology_task_prefix, cursor }); + const found = try cur.seekAtOrAfter(seek); + break :blk if (found != null and std.mem.eql(u8, found.?.key, seek)) try cur.next() else found; + } else try cur.seekAtOrAfter(topology_task_prefix); + if (entry_opt == null or !std.mem.startsWith(u8, entry_opt.?.key, topology_task_prefix)) { + entry_opt = try cur.seekAtOrAfter(topology_task_prefix); + } + const entry = entry_opt orelse return false; + if (!std.mem.startsWith(u8, entry.key, topology_task_prefix)) return false; + if (entry.key.len != topology_task_prefix.len + 64) return error.InvalidGraphMetricBuildManifest; + self.topology_preparation_cursor = entry.key[topology_task_prefix.len..][0..64].*; + break :read .{ .key = try temp.dupe(u8, entry.key), .raw = try temp.dupe(u8, entry.value), .name = try topologyTaskNameAlloc(temp, entry.key, entry.value) }; + }; + const cfg = try topologyTaskConfigAlloc(temp, task.name, task.raw); + const generation = std.mem.readInt(u64, task.raw[0..8], .little); + var view = self.*; + view.metric_configs = &.{cfg}; + view.sealed_vectors = .{ .capacity = 0 }; + view.topology_preparation_only = true; + var failure_reason: ?[]const u8 = null; + const finished = check: { + if (task.raw[8] & 2 != 0) break :check true; + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (generation != try self.graphMetricFilterGeneration(&txn, cfg.edge_filter)) break :check true; + var interested = false; + for (self.metric_configs) |candidate| { + if (!graphMetricKindUsesIterativeBuild(candidate.kind) or !candidate.edge_filter.equivalent(cfg.edge_filter)) continue; + if (cfg.kind == .eigenvector and (candidate.kind == .hits_authority or candidate.kind == .hits_hub)) continue; + const status = try self.graphMetricSchedulerStatus(candidate.name, null); + if (status.maintenance_paused or status.state == .disabled) continue; + if (candidate.refresh == .background or try self.graphMetricBuildRequested(candidate.name)) { + interested = true; + break; + } + } + if (!interested) break :check true; + if (try view.metricBuildJob(&txn, task.name)) |job| { + if (job.last_error.len != 0 or job.retry_count != 0) { + failure_reason = try temp.dupe(u8, if (job.last_error.len != 0) job.last_error else "GraphMetricTopologyPreparationFailed"); + break :check true; + } + } + const filter = try topology_owner.filterDigest(temp, cfg.edge_filter); + const identity = topology_owner.identity(filter, try self.graphMetricPartitionPlanRaw(&txn, cfg)); + const ready = try topology_owner.readyKey(temp, identity, cfg.kind == .hits_authority); + if (txn.get(ready)) |_| break :check true else |err| if (err != error.NotFound) return err; + break :check false; + }; + if (finished) { + if (failure_reason) |reason| try self.propagateTopologyTaskFailure(cfg, generation, reason); + return self.retireTopologyTaskPage(task.key, task.name, task.raw); + } + var started = view.ensureGraphMetricPlannedBuildFromCachedPlan(task.name, generation) catch |err| switch (err) { + error.GraphMetricBuildSuperseded, error.GraphMetricBuildSnapshotChanged => return false, + else => return err, + }; + started.deinit(self.alloc); + const coordinator = try view.runGraphMetricPlannedCoordinatorStepForMetric(task.name); + if (coordinator.failed_build) return true; // Persisted failure is propagated before retirement on the next step. + // Sealing can occur in the coordinator barrier. Never run a numerical + // convergence/publication page after this task has exposed its owner. + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const job = try view.metricBuildJob(&txn, task.name) orelse return true; + const binding = try view.topologyBinding(&txn, task.name, job.job_id) orelse return error.InvalidGraphMetricBuildManifest; + const owner_key = try topology_owner.catalogKey(temp, binding.id); + if ((try topology_owner.Record.decode(try txn.get(owner_key))).state == .sealed) return true; + } + const step = try view.runGraphMetricPlannedWorkerPageStepForMetric(task.name, worker); + return step.claimed_page or step.completed_page or step.failed_build or coordinator.advanced_phase or coordinator.retired_input_records != 0; + } + + fn retireTopologyTaskPage(self: *GraphIndex, key: []const u8, name: []const u8, raw: []u8) !bool { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const current = batch.get(key) catch |err| switch (err) { + error.NotFound => { + batch.abort(); + return false; + }, + else => return err, + }; + if (try topologyTaskIncarnation(current) != try topologyTaskNameIncarnation(name)) { + batch.abort(); + return false; + } + // Mark retirement before removing even the job pointer. A retry uses + // a fresh control namespace and cannot be erased by a stale cleanup. + raw[8] |= 2; + std.crypto.hash.sha2.Sha256.hash(raw[0 .. raw.len - 32], raw[raw.len - 32 ..][0..32], .{}); + try batch.put(key, raw); + const prefix = try self.graphMetricControlKeyAlloc(&.{name}); + defer self.alloc.free(prefix); + var removed = try self.deleteKeysWithPrefixPageInBatch(&batch, prefix, "", graph_metric_build_cleanup_delete_page_units); + defer removed.deinit(self.alloc); + if (removed.reached_end) try batch.delete(key); + try batch.commit(); + return true; + } + + fn topologyBindingKey(self: *GraphIndex, metric: []const u8, job_id: u64) ![]u8 { + const prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric, job_id); + defer self.alloc.free(prefix); + return std.fmt.allocPrint(self.alloc, "{s}topology-owner", .{prefix}); + } + + fn topologyComponentPrefix(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, component: []const u8) ![]u8 { + var job_buf: [20]u8 = undefined; + return self.topologyKey(txn, metric, job_id, try self.graphMetricControlKeyAlloc(&.{ metric, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}), component })); + } + + fn topologyComponentKey(alloc: Allocator, prefix: []const u8, component: []const u8) ![]u8 { + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(alloc); + try key.appendSlice(alloc, prefix); + try internal_keys.appendEncodedComponent(&key, alloc, component); + return key.toOwnedSlice(alloc); + } + + fn topologyBinding(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64) !?topology_owner.Binding { + const key = try self.topologyBindingKey(metric, job_id); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + const binding = try topology_owner.Binding.decode(raw); + const owner_key = try topology_owner.catalogKey(self.alloc, binding.id); + defer self.alloc.free(owner_key); + const owner = try topology_owner.Record.decode(txn.get(owner_key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildSuperseded, + else => return err, + }); + if (owner.state == .deleting) return error.GraphMetricBuildSuperseded; + if (owner.format_epoch != topology_owner.epoch) return error.InvalidGraphMetricBuildManifest; + if (binding.adopted and owner.state != .sealed) return error.InvalidGraphMetricBuildManifest; + return binding; + } + + /// Takes ownership of a job-local key and resolves just the immutable + /// topology part. Numeric vectors, producer staging, and folds stay local. + fn topologyKey(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, local: []u8) ![]u8 { + defer self.alloc.free(local); + const binding = try self.topologyBinding(txn, metric, job_id) orelse return self.alloc.dupe(u8, local); + const job_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric, job_id); + defer self.alloc.free(job_prefix); + if (!std.mem.startsWith(u8, local, job_prefix)) return error.InvalidGraphMetricBuildManifest; + const prefix = try topology_owner.dataPrefix(self.alloc, binding.id); + defer self.alloc.free(prefix); + return std.fmt.allocPrint(self.alloc, "{s}{s}", .{ prefix, local[job_prefix.len..] }); + } + + fn requireMutableTopology(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64) !void { + const binding = try self.topologyBinding(txn, metric, job_id) orelse return; + if (binding.adopted) return error.InvalidGraphMetricBuildManifest; + const key = try topology_owner.catalogKey(self.alloc, binding.id); + defer self.alloc.free(key); + const owner = try topology_owner.Record.decode(try txn.get(key)); + if (owner.state != .building) return error.GraphMetricBuildSuperseded; + if (owner.generation != try self.graphMetricCurrentGenerationInTxn(txn, metric)) return error.GraphMetricBuildSnapshotChanged; + } + + fn ensureTopologyBindingInBatch(self: *GraphIndex, batch: anytype, metric: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob) !void { + if (!graphMetricKindUsesIterativeBuild(cfg.kind)) return; + if (try self.topologyBinding(batch, metric, job.job_id) != null) return; + const filter = try topology_owner.filterDigest(self.alloc, cfg.edge_filter); + const digest = topology_owner.identity(filter, try self.graphMetricPartitionPlanRaw(batch, cfg)); + const bidirectional = cfg.kind == .hits_authority or cfg.kind == .hits_hub; + const ready_key = try topology_owner.readyKey(self.alloc, digest, bidirectional); + defer self.alloc.free(ready_key); + const namespace = try self.graphMetricBuildJobNamespacePrefixAlloc(metric, job.job_id); + defer self.alloc.free(namespace); + var binding = topology_owner.Binding{ .id = topology_owner.ownerId(digest, namespace, job.score_generation), .adopted = false }; + if (batch.get(ready_key)) |raw| { + if (raw.len != 64) return error.InvalidGraphMetricBuildManifest; + const id: topology_owner.Id = raw[0..64].*; + const key = try topology_owner.catalogKey(self.alloc, id); + defer self.alloc.free(key); + const owner = try topology_owner.Record.decode(try batch.get(key)); + if (owner.state != .sealed or owner.format_epoch != topology_owner.epoch or owner.generation != job.target_generation or + !std.mem.eql(u8, &owner.identity, &digest) or (bidirectional and !owner.bidirectional)) + return error.InvalidGraphMetricBuildManifest; + binding = .{ .id = id, .adopted = true }; + } else |err| if (err != error.NotFound) return err; + if (!binding.adopted) { + const key = try topology_owner.catalogKey(self.alloc, binding.id); + defer self.alloc.free(key); + const owner = topology_owner.Record{ .generation = job.target_generation, .filter = filter, .identity = digest, .bidirectional = bidirectional }; + // Never resurrect a tombstoned owner with the same producer ID. + if (batch.get(key)) |_| return error.GraphMetricBuildSuperseded else |err| if (err != error.NotFound) return err; + try batch.put(key, &owner.encode()); + } + const binding_key = try self.topologyBindingKey(metric, job.job_id); + defer self.alloc.free(binding_key); + const pins = try topology_owner.pinsPrefix(self.alloc, binding.id); + defer self.alloc.free(pins); + const pin_key = try std.fmt.allocPrint(self.alloc, "{s}{s}", .{ pins, topology_owner.ownerId(digest, namespace, job.score_generation) }); + defer self.alloc.free(pin_key); + const pin = try self.alloc.alloc(u8, 16 + metric.len); + defer self.alloc.free(pin); + std.mem.writeInt(u64, pin[0..8], job.job_id, .little); + std.mem.writeInt(u64, pin[8..16], graphMetricConfigFingerprint(cfg), .little); + @memcpy(pin[16..], metric); + // Owner validation, pin, and job binding are one serializable commit. + try batch.put(pin_key, pin); + try batch.put(binding_key, &binding.encode()); + if (binding.adopted) { + const plan_raw = try self.graphMetricPartitionPlanRaw(batch, cfg); + var plan = (try self.decodeGraphMetricPartitionPlanAlloc(plan_raw)) orelse return error.InvalidGraphMetricBuildManifest; + defer plan.deinit(self.alloc); + for ([_]GraphMetricBuildPhase{ .scan_edges_and_out_degree, .iterate_contributions, .hits_hub_contributions }) |phase| { + if (phase == .hits_hub_contributions and !bidirectional) continue; + for (0..plan.edge_page_count) |i| { + const page_id = graphMetricBuildPhasePageIdBase(cfg.kind, phase) + i; + var page = try self.metricBuildPage(batch, metric, job.job_id, phase, 0, page_id) orelse return error.InvalidGraphMetricBuildManifest; + page.state = .complete; + page.completed_units = page.total_units; + page.output_fingerprint = job.target_generation; + try self.putGraphMetricBuildPageInBatch(batch, metric, page); + } + } + } + } + + fn topologyMembershipLeaf(self: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, leaf_id: u64) !?GraphMetricBuildPage { + if (try self.topologyBinding(txn, metric, job_id)) |binding| if (binding.adopted) { + const key = try self.topologyKey(txn, metric, job_id, try self.graphMetricBuildPageKeyAlloc(metric, job_id, .initialize_ranks, 0, leaf_id)); + defer self.alloc.free(key); + return decodeGraphMetricBuildPage(try txn.get(key)) orelse error.InvalidGraphMetricBuildManifest; + }; + return self.metricBuildPage(txn, metric, job_id, .initialize_ranks, 0, leaf_id); + } + + fn sealTopologyInBatch(self: *GraphIndex, batch: anytype, metric: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob, summary: GraphMetricBuildPhaseSummary) !void { + if (summary.iteration != 0 or summary.state != .complete or !graphMetricKindUsesIterativeBuild(cfg.kind)) return; + const bidirectional = cfg.kind == .hits_authority or cfg.kind == .hits_hub; + if (summary.phase != (if (bidirectional) GraphMetricBuildPhase.hits_hub_reduce_ranks else .reduce_ranks)) return; + const binding = try self.topologyBinding(batch, metric, job.job_id) orelse return; + if (binding.adopted) return; + const key = try topology_owner.catalogKey(self.alloc, binding.id); + defer self.alloc.free(key); + var owner = try topology_owner.Record.decode(try batch.get(key)); + if (owner.state == .sealed) return; + try self.requireMutableTopology(batch, metric, job.job_id); + // These phase barriers prove every membership leaf and every target + // chunk receipt is durable. A partial packing attempt is never exposed. + const initialized = try self.metricBuildPhaseSummary(batch, metric, job.job_id, .initialize_ranks, 0) orelse return error.InvalidGraphMetricBuildManifest; + const produced = try self.metricBuildPhaseSummary(batch, metric, job.job_id, .iterate_contributions, 0) orelse return error.InvalidGraphMetricBuildManifest; + if (initialized.state != .complete or produced.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + if (bidirectional) { + const forward = try self.metricBuildPhaseSummary(batch, metric, job.job_id, .reduce_ranks, 0) orelse return error.InvalidGraphMetricBuildManifest; + const reverse = try self.metricBuildPhaseSummary(batch, metric, job.job_id, .hits_hub_contributions, 0) orelse return error.InvalidGraphMetricBuildManifest; + if (forward.state != .complete or reverse.state != .complete) return error.GraphMetricBuildPhaseNotComplete; + } + owner.state = .sealed; + try batch.put(key, &owner.encode()); + for ([_]bool{ false, true }) |reverse| { + if (reverse and !bidirectional) continue; + const ready = try topology_owner.readyKey(self.alloc, owner.identity, reverse); + defer self.alloc.free(ready); + // Concurrent producers keep distinct staging/attempt spaces. The + // first sealed owner wins; losing owners live only while pinned. + if (batch.get(ready)) |_| continue else |err| if (err != error.NotFound) return err; + try batch.put(ready, &binding.id); + } + } + + /// Index-scoped maintenance, including indexes with zero configured metrics. + /// Deletes at most 512 data records and examines 64 pins in one transaction. + /// A durable deleting tombstone prevents adoption or late producer writes. + pub const TopologyCleanupResult = struct { + progressed: bool = false, + removed: usize = 0, + }; + + pub fn cleanupGraphMetricTopologyPage(self: *GraphIndex) !bool { + return (try self.cleanupGraphMetricTopologyPageDetailed()).progressed; + } + + fn cleanupGraphMetricFilterPlans(self: *GraphIndex) !usize { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const prefix = graph_metric_filter_plan_prefix; + var retained = std.StringHashMapUnmanaged(void).empty; + for (self.metric_configs) |cfg| { + if (cfg.edge_filter.mode == .all) continue; + const key = try self.graphMetricPartitionPlanKeyAlloc(cfg.edge_filter); + defer self.alloc.free(key); + try retained.put(temp, try temp.dupe(u8, key), {}); + } + var obsolete = std.ArrayListUnmanaged([]const u8).empty; + var next_cursor: @TypeOf(self.filter_plan_gc_cursor) = null; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + const start = if (self.filter_plan_gc_cursor) |suffix| try std.mem.concat(temp, u8, &.{ prefix, suffix.bytes[0..suffix.len] }) else prefix; + var item = try cur.seekAtOrAfter(start); + if (self.filter_plan_gc_cursor != null) if (item) |entry| if (std.mem.eql(u8, entry.key, start)) { + item = try cur.next(); + }; + var inspected: usize = 0; + while (item) |entry| : (item = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + if (inspected == 64) break; + const suffix = entry.key[prefix.len..]; + if (suffix.len != 64 and !(suffix.len == 68 and suffix[64] == '/')) return error.InvalidGraphMetricBuildManifest; + next_cursor = .{ .bytes = undefined, .len = @intCast(suffix.len) }; + @memcpy(next_cursor.?.bytes[0..suffix.len], suffix); + inspected += 1; + if (!retained.contains(entry.key[0 .. prefix.len + 64])) try obsolete.append(temp, try temp.dupe(u8, entry.key)); + } + if (item == null or !std.mem.startsWith(u8, item.?.key, prefix)) next_cursor = null; + } + if (obsolete.items.len > 0) { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + for (obsolete.items) |key| batch.delete(key) catch |err| if (err != error.NotFound) return err; + try batch.commit(); + } + self.filter_plan_gc_cursor = next_cursor; + return obsolete.items.len; + } + + pub fn cleanupGraphMetricTopologyPageDetailed(self: *GraphIndex) !TopologyCleanupResult { + if (!self.topology_gc_mutex.tryLock()) return .{}; + defer self.topology_gc_mutex.unlock(); + const obsolete_plans = try self.cleanupGraphMetricFilterPlans(); + if (obsolete_plans > 0) return .{ .progressed = true, .removed = obsolete_plans }; + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const key = inspect: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const resume_key = if (self.topology_gc_cursor) |id| try topology_owner.catalogKey(temp, id) else topology_owner.catalog_prefix; + var cur = try txn.openCursor(); + var cursor_open = true; + defer if (cursor_open) cur.close(); + var entry = try cur.seekAtOrAfter(resume_key); + if (self.topology_gc_cursor != null) if (entry) |e| if (std.mem.eql(u8, e.key, resume_key)) { + entry = try cur.next(); + }; + if (entry == null or !std.mem.startsWith(u8, entry.?.key, topology_owner.catalog_prefix)) { + self.topology_gc_cursor = null; + return .{}; + } + const selected = try temp.dupe(u8, entry.?.key); + if (selected.len != topology_owner.catalog_prefix.len + 64) return error.InvalidGraphMetricBuildManifest; + const id: topology_owner.Id = selected[topology_owner.catalog_prefix.len..][0..64].*; + const owner = try topology_owner.Record.decode(entry.?.value); + cur.close(); + cursor_open = false; + if (try self.topologyOwnerRetained(&txn, temp, owner, id) and + !try self.hasKeysWithPrefixInBatch(&txn, try topology_owner.pinsPrefix(temp, id)) and + !try self.hasKeysWithPrefixInBatch(&txn, try std.fmt.allocPrint(temp, "{s}retired/", .{try topology_owner.dataPrefix(temp, id)}))) + { + self.topology_gc_cursor = id; + return .{}; + } + break :inspect selected; + }; + // Revalidate all liveness under the writer transaction. A job may pin + // this owner between the read-only census and acquisition of the lock. + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const id: topology_owner.Id = key[topology_owner.catalog_prefix.len..][0..64].*; + const owner_raw = batch.get(key) catch |err| switch (err) { + error.NotFound => { + batch.abort(); + self.topology_gc_cursor = id; + return .{}; + }, + else => return err, + }; + var owner = try topology_owner.Record.decode(owner_raw); + if (owner.state != .deleting) { + const retained = try self.topologyOwnerRetained(&batch, temp, owner, id); + const pins = try topology_owner.pinsPrefix(temp, id); + var pin_cur = try batch.openCursor(); + var pin_cur_open = true; + defer if (pin_cur_open) pin_cur.close(); + var pin_entry = try pin_cur.seekAtOrAfter(pins); + var stale = std.ArrayListUnmanaged([]const u8).empty; + var examined: usize = 0; + var live = false; + while (pin_entry) |pin| : (pin_entry = try pin_cur.next()) { + if (!std.mem.startsWith(u8, pin.key, pins) or examined == 64) break; + examined += 1; + if (pin.value.len <= 16) return error.InvalidGraphMetricBuildManifest; + const job_id = std.mem.readInt(u64, pin.value[0..8], .little); + const fingerprint = std.mem.readInt(u64, pin.value[8..16], .little); + const metric = try temp.dupe(u8, pin.value[16..]); + var active = false; + const pin_config = self.metricConfig(metric) orelse prep: { + if (!std.mem.startsWith(u8, metric, topology_task_name_prefix)) break :prep null; + const task_key = try topologyTaskKeyAlloc(temp, metric); + const task_raw = batch.get(task_key) catch |err| switch (err) { + error.NotFound => break :prep null, + else => return err, + }; + break :prep topologyTaskConfigAlloc(temp, metric, task_raw) catch |err| switch (err) { + error.GraphMetricBuildSuperseded => null, + else => return err, + }; + }; + if (pin_config) |cfg| if (graphMetricConfigFingerprint(cfg) == fingerprint) { + if (try self.metricBuildJob(&batch, metric)) |job| { + active = owner.format_epoch == topology_owner.epoch and job.job_id == job_id and job.target_generation == owner.generation and + job.phase != .complete and job.last_error.len == 0 and job.phase != .cleanup_old_generations; + if (active) { + const binding = try self.topologyBinding(&batch, metric, job_id); + active = if (binding) |b| std.mem.eql(u8, &b.id, &id) else false; + } + } + }; + if (active) { + live = true; + break; + } + try stale.append(temp, try temp.dupe(u8, pin.key)); + } + const more_pins = pin_entry != null and std.mem.startsWith(u8, pin_entry.?.key, pins); + pin_cur.close(); + pin_cur_open = false; + for (stale.items) |stale_key| try batch.delete(stale_key); + if (live or retained) { + const pruned = try self.cleanupTopologyAttemptInBatch(&batch, temp, id); + if (stale.items.len + pruned != 0) try batch.commit() else batch.abort(); + self.topology_gc_cursor = id; + // Cursor-only census progress is not eligible work: reporting + // it keeps multi-worker idle loops alive indefinitely as they + // wrap the catalog. Periodic sweeps continue the census even + // when there are no numerical jobs or reclamation writes. + return .{ .progressed = stale.items.len + pruned != 0, .removed = stale.items.len + pruned }; + } + if (more_pins) { + // Resume this owner, not the next one, after a bounded pin page. + try batch.commit(); + return .{ .progressed = stale.items.len != 0, .removed = stale.items.len }; + } + owner.state = .deleting; + try batch.put(key, &owner.encode()); + for ([_]bool{ false, true }) |reverse| { + const ready = try topology_owner.readyKey(temp, owner.identity, reverse); + if (batch.get(ready)) |raw| { + if (std.mem.eql(u8, raw, &id)) try batch.delete(ready); + } else |err| if (err != error.NotFound) return err; + } + } + const prefix = try topology_owner.dataPrefix(temp, id); + var deleted = try self.deleteKeysWithPrefixPageInBatch(&batch, prefix, "", graph_metric_build_cleanup_delete_page_units); + defer deleted.deinit(self.alloc); + // Deleting from the beginning is a durable resume cursor: removed keys + // cannot reappear, and no job can bind to or write a deleting owner. + if (deleted.reached_end) { + try batch.delete(key); + } + try batch.commit(); + if (deleted.reached_end) self.topology_gc_cursor = id; + return .{ .progressed = true, .removed = deleted.removed + @intFromBool(deleted.reached_end) }; + } + + fn topologyOwnerRetained(self: *GraphIndex, txn: anytype, temp: Allocator, owner: topology_owner.Record, id: topology_owner.Id) !bool { + if (owner.state != .sealed or owner.format_epoch != topology_owner.epoch) return false; + for (self.metric_configs) |cfg| { + if (!graphMetricKindUsesIterativeBuild(cfg.kind)) continue; + const filter = try topology_owner.filterDigest(temp, cfg.edge_filter); + if (!std.mem.eql(u8, &filter, &owner.filter)) continue; + if (owner.generation != try self.graphMetricFilterGeneration(txn, cfg.edge_filter)) continue; + const plan = self.graphMetricPartitionPlanRaw(txn, cfg) catch |err| switch (err) { + error.GraphMetricBuildSnapshotChanged => continue, + else => return err, + }; + if (!std.mem.eql(u8, &topology_owner.identity(owner.filter, plan), &owner.identity)) continue; + const reverse = cfg.kind == .hits_authority or cfg.kind == .hits_hub; + const ready = try topology_owner.readyKey(temp, owner.identity, reverse); + if (txn.get(ready)) |value| { + if (std.mem.eql(u8, value, &id)) return true; + } else |err| if (err != error.NotFound) return err; + } + return false; + } + + fn cleanupTopologyAttemptInBatch(self: *GraphIndex, batch: anytype, temp: Allocator, id: topology_owner.Id) !usize { + const data = try topology_owner.dataPrefix(temp, id); + const tasks = try std.fmt.allocPrint(temp, "{s}retired/", .{data}); + const task = blk: { + var cur = try batch.openCursor(); + defer cur.close(); + const entry = try cur.seekAtOrAfter(tasks) orelse return 0; + if (!std.mem.startsWith(u8, entry.key, tasks)) return 0; + break :blk .{ .key = try temp.dupe(u8, entry.key), .prefix = try temp.dupe(u8, entry.value) }; + }; + const packed_prefix = try std.fmt.allocPrint(temp, "{s}adjacency-packed/", .{data}); + if (!std.mem.startsWith(u8, task.prefix, packed_prefix) or !std.mem.endsWith(u8, task.prefix, "/") or + std.mem.indexOf(u8, task.prefix[packed_prefix.len..], "/data/") == null) + return error.InvalidGraphMetricBuildManifest; + const separator = packed_prefix.len + std.mem.indexOf(u8, task.prefix[packed_prefix.len..], "/data/").?; + const attempt_text = task.prefix[separator + 6 .. task.prefix.len - 1]; + if (attempt_text.len != 20) return error.InvalidGraphMetricBuildManifest; + const attempt = std.fmt.parseInt(u64, attempt_text, 10) catch return error.InvalidGraphMetricBuildManifest; + const base = task.prefix[0 .. separator + 1]; + const receipt_key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + if (batch.get(receipt_key)) |raw| { + if ((try adjacency_blocks.Receipt.decode(raw)).attempt == attempt) return error.InvalidGraphMetricBuildManifest; + } else |err| switch (err) { + error.NotFound => { + const state_key = try std.fmt.allocPrint(temp, "{s}state", .{base}); + const state = try adjacency_blocks.State.decode(try batch.get(state_key)); + if (state.attempt == attempt) return error.InvalidGraphMetricBuildManifest; + }, + else => return err, + } + var deleted = try self.deleteKeysWithPrefixPageInBatch(batch, task.prefix, "", graph_metric_build_cleanup_delete_page_units); + defer deleted.deinit(self.alloc); + if (deleted.reached_end) try batch.delete(task.key); + return deleted.removed + @intFromBool(deleted.reached_end); + } + + fn graphMetricBuildJobNamespacePrefixAlloc(self: *GraphIndex, metric_name: []const u8, job_id: u64) ![]u8 { + const job_id_text = try std.fmt.allocPrint(self.alloc, "{d}", .{job_id}); + defer self.alloc.free(job_id_text); + return try self.graphMetricControlKeyAlloc(&.{ metric_name, "job", job_id_text }); + } + + fn cleanupGraphMetricBuildJobKeys(self: *GraphIndex, metric_name: []const u8, job_id: u64) !usize { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const removed = try self.cleanupGraphMetricBuildJobKeysInBatch(&batch, metric_name, job_id); + try batch.commit(); + return removed; + } + + /// Retires one bounded page from the oldest failed build namespace. + /// Cleanup tasks are keyed by job id, so repeated failures queue without + /// overwriting one another and process restarts resume from the stored + /// cursor. + pub fn cleanupFailedGraphMetricBuildJobPage(self: *GraphIndex, metric_name: []const u8) !bool { + const prefix = try self.graphMetricFailedJobCleanupPrefixAlloc(metric_name); + defer self.alloc.free(prefix); + var task_key: []u8 = ""; + defer if (task_key.len > 0) self.alloc.free(task_key); + var job_id_text: []u8 = ""; + defer if (job_id_text.len > 0) self.alloc.free(job_id_text); + var resume_cursor: []u8 = ""; + defer if (resume_cursor.len > 0) self.alloc.free(resume_cursor); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + const entry = (try cur.seekAtOrAfter(prefix)) orelse return false; + if (!std.mem.startsWith(u8, entry.key, prefix)) return false; + task_key = try self.alloc.dupe(u8, entry.key); + job_id_text = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, prefix)) orelse return error.InvalidGraphMetricBuildManifest; + if (entry.value.len > 0) resume_cursor = try self.alloc.dupe(u8, entry.value); + } + const job_id = std.fmt.parseInt(u64, job_id_text, 10) catch return error.InvalidGraphMetricBuildManifest; + const job_prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + defer self.alloc.free(job_prefix); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + var deleted = try self.deleteKeysWithPrefixPageInBatch(&batch, job_prefix, resume_cursor, graph_metric_build_cleanup_delete_page_units); + defer deleted.deinit(self.alloc); + if (deleted.reached_end) { + batch.delete(task_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } else { + try batch.put(task_key, deleted.cursor); + } + try batch.commit(); + return true; + } + + fn cleanupGraphMetricBuildJobKeysInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, job_id: u64) !usize { + if (try self.metricBuildJob(batch, metric_name)) |job| { + if (job.job_id == job_id and job.phase != .complete) return error.GraphMetricBuildJobActive; + } + return try self.deleteGraphMetricBuildJobNamespaceInBatch(batch, metric_name, job_id); + } + + fn deleteGraphMetricBuildJobNamespaceInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, job_id: u64) !usize { + const prefix = try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + defer self.alloc.free(prefix); + return try self.deleteKeysWithPrefixInBatch(batch, prefix); + } + + fn adoptGraphMetricAttemptOutputPage( + self: *GraphIndex, + metric_name: []const u8, + kind: GraphMetricKind, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricAttemptAdoptionResult { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const result = switch (page.phase) { + .scan_edges_and_out_degree => if (kind == .degree) + try self.adoptDegreeScanAttemptOutputInBatch(&batch, metric_name, job_id, page) + else + try self.adoptPageRankScanAttemptOutputInBatch(&batch, metric_name, job_id, page), + else => return error.UnsupportedGraphMetricBuildPhase, + }; + try batch.commit(); + return result; + } + + fn adoptDegreeScanAttemptOutputInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricAttemptAdoptionResult { + const current_page = try self.metricBuildPage(batch, metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current_page); + const attempt_prefix = try self.graphMetricBuildAttemptDegreePartialPrefixAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt); + defer self.alloc.free(attempt_prefix); + var entries = std.ArrayListUnmanaged(struct { + node: []u8, + value: u64, + }).empty; + defer { + for (entries.items) |entry| self.alloc.free(entry.node); + entries.deinit(self.alloc); + } + var reached_end = true; + { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(attempt_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, attempt_prefix)) break; + if (entries.items.len >= graph_metric_build_adoption_page_units) { + reached_end = false; + break; + } + if (entry.value.len != 8) return error.InvalidGraphMetricBuildManifest; + const value = std.mem.readInt(u64, entry.value[0..8], .little); + if (value == 0) return error.InvalidGraphMetricBuildManifest; + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, attempt_prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(node); + try entries.append(self.alloc, .{ .node = node, .value = value }); + } + } + for (entries.items) |entry| { + const partial_key = try self.graphMetricBuildDegreePartialKeyAlloc(metric_name, job_id, entry.node, page.page_id); + defer self.alloc.free(partial_key); + try putU64(batch, partial_key, entry.value); + const attempt_key = try self.graphMetricBuildAttemptDegreePartialKeyAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt, entry.node); + defer self.alloc.free(attempt_key); + batch.delete(attempt_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + return .{ .adopted = entries.items.len, .reached_end = reached_end }; + } + + fn adoptPageRankScanAttemptOutputInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricAttemptAdoptionResult { + const current_page = try self.metricBuildPage(batch, metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current_page); + try self.requireMutableTopology(batch, metric_name, job_id); + const out_degree_prefix = try self.graphMetricBuildAttemptPageRankOutDegreePartialPrefixAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt); + defer self.alloc.free(out_degree_prefix); + const node_prefix = try self.graphMetricBuildAttemptPageRankNodePartialPrefixAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt); + defer self.alloc.free(node_prefix); + + var out_degrees = std.ArrayListUnmanaged(struct { + node: []u8, + value: u64, + }).empty; + defer { + for (out_degrees.items) |entry| self.alloc.free(entry.node); + out_degrees.deinit(self.alloc); + } + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + + var out_degrees_reached_end = true; + { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(out_degree_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, out_degree_prefix)) break; + if (out_degrees.items.len >= graph_metric_build_adoption_page_units) { + out_degrees_reached_end = false; + break; + } + if (entry.value.len != 8) return error.InvalidGraphMetricBuildManifest; + const value = std.mem.readInt(u64, entry.value[0..8], .little); + if (value == 0) return error.InvalidGraphMetricBuildManifest; + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, out_degree_prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(node); + try out_degrees.append(self.alloc, .{ .node = node, .value = value }); + } + } + var nodes_reached_end = out_degrees_reached_end; + if (out_degrees_reached_end) { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(node_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, node_prefix)) break; + if (out_degrees.items.len + nodes.items.len >= graph_metric_build_adoption_page_units) { + nodes_reached_end = false; + break; + } + if (entry.value.len != 8 or std.mem.readInt(u64, entry.value[0..8], .little) != 1) { + return error.InvalidGraphMetricBuildManifest; + } + const node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, node_prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(node); + try nodes.append(self.alloc, node); + } + } + + const partial_keys = try self.alloc.alloc([]u8, out_degrees.items.len); + const total_keys = try self.alloc.alloc([]u8, out_degrees.items.len); + const replacements = try self.alloc.alloc(u64, out_degrees.items.len); + var initialized_keys: usize = 0; + defer { + for (partial_keys[0..initialized_keys]) |key| self.alloc.free(key); + for (total_keys[0..initialized_keys]) |key| self.alloc.free(key); + self.alloc.free(partial_keys); + self.alloc.free(total_keys); + self.alloc.free(replacements); + } + const total_prefix = try self.topologyComponentPrefix(batch, metric_name, job_id, "pagerank_out_degree_total"); + defer self.alloc.free(total_prefix); + for (out_degrees.items, 0..) |entry, i| { + partial_keys[i] = try self.graphMetricBuildPageRankOutDegreePartialKeyAlloc(metric_name, job_id, entry.node, page.page_id); + errdefer self.alloc.free(partial_keys[i]); + total_keys[i] = try topologyComponentKey(self.alloc, total_prefix, entry.node); + replacements[i] = entry.value; + initialized_keys += 1; + } + try self.replaceU64PageDeltasInBatch(batch, partial_keys, total_keys, replacements); + for (out_degrees.items) |entry| { + const attempt_key = try self.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt, entry.node); + defer self.alloc.free(attempt_key); + batch.delete(attempt_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + for (nodes.items) |node| { + const partial_key = try self.graphMetricBuildPageRankNodePartialKeyAlloc(metric_name, job_id, node, page.page_id); + defer self.alloc.free(partial_key); + try putU64(batch, partial_key, 1); + const attempt_key = try self.graphMetricBuildAttemptPageRankNodePartialKeyAlloc(metric_name, job_id, .scan_edges_and_out_degree, page.iteration, page.page_id, page.attempt, node); + defer self.alloc.free(attempt_key); + batch.delete(attempt_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + return .{ + .adopted = out_degrees.items.len + nodes.items.len, + .reached_end = out_degrees_reached_end and nodes_reached_end, + }; + } + + fn adoptPageRankContributionAttemptOutputInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricAttemptAdoptionResult { + const current_page = try self.metricBuildPage(batch, metric_name, job_id, .iterate_contributions, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current_page); + const attempt_prefix = try self.graphMetricBuildAttemptPageRankContributionPrefixAlloc(metric_name, job_id, .iterate_contributions, page.iteration, page.page_id, page.attempt); + defer self.alloc.free(attempt_prefix); + var chunk_key: []u8 = ""; + defer if (chunk_key.len > 0) self.alloc.free(chunk_key); + var chunk_value: []u8 = ""; + defer if (chunk_value.len > 0) self.alloc.free(chunk_value); + var reached_end = true; + { + var cur = try batch.openCursor(); + defer cur.close(); + const entry = (try cur.seekAtOrAfter(attempt_prefix)) orelse return .{ .reached_end = true }; + if (!std.mem.startsWith(u8, entry.key, attempt_prefix)) return .{ .reached_end = true }; + chunk_key = try self.alloc.dupe(u8, entry.key); + chunk_value = try self.alloc.dupe(u8, entry.value); + if (try cur.next()) |next| reached_end = !std.mem.startsWith(u8, next.key, attempt_prefix); + } + const entries = try self.decodePackedF64EntriesAlloc(chunk_value); + defer self.alloc.free(entries); + try self.publishPackedF64CheckpointShardsInBatch( + batch, + metric_name, + job_id, + page.iteration, + entries, + current_page, + chunk_key[attempt_prefix.len..], + ); + batch.delete(chunk_key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildPageOutputMismatch, + else => return err, + }; + return .{ .adopted = entries.len, .reached_end = reached_end }; + } + + fn adoptHitsHubRawAttemptOutputInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + job_id: u64, + page: GraphMetricBuildPage, + ) !GraphMetricAttemptAdoptionResult { + const current_page = try self.metricBuildPage(batch, metric_name, job_id, .hits_hub_contributions, page.iteration, page.page_id) orelse return error.GraphMetricBuildPageNotFound; + try self.validateGraphMetricBuildPageExecutionLease(page, current_page); + const attempt_prefix = try self.graphMetricBuildAttemptHitsHubRawPrefixAlloc(metric_name, job_id, .hits_hub_contributions, page.iteration, page.page_id, page.attempt); + defer self.alloc.free(attempt_prefix); + var entries = std.ArrayListUnmanaged(struct { + source_node: []u8, + value: f64, + }).empty; + defer { + for (entries.items) |entry| self.alloc.free(entry.source_node); + entries.deinit(self.alloc); + } + var reached_end = true; + { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(attempt_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, attempt_prefix)) break; + if (entries.items.len >= graph_metric_build_adoption_page_units) { + reached_end = false; + break; + } + if (entry.value.len != 8) return error.InvalidGraphMetricBuildManifest; + const value = @as(f64, @bitCast(std.mem.readInt(u64, entry.value[0..8], .little))); + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + const source_node = (try graphMetricFirstComponentAfterPrefixAlloc(self.alloc, entry.key, attempt_prefix)) orelse + return error.InvalidGraphMetricBuildManifest; + errdefer self.alloc.free(source_node); + try entries.append(self.alloc, .{ .source_node = source_node, .value = value }); + } + } + const page_keys = try self.alloc.alloc([]u8, entries.items.len); + var initialized_keys: usize = 0; + defer { + for (page_keys[0..initialized_keys]) |key| self.alloc.free(key); + self.alloc.free(page_keys); + } + for (entries.items, 0..) |entry, i| { + page_keys[i] = try self.graphMetricBuildHitsHubRawKeyAlloc(metric_name, job_id, page.iteration, entry.source_node, page.page_id); + initialized_keys += 1; + try putF64(batch, page_keys[i], entry.value); + // Retire pre-shard-reducer totals during a rolling upgrade. The + // authoritative value is now the stable node/page shard stream. + const legacy_total_key = try self.graphMetricBuildHitsHubRawTotalKeyAlloc(metric_name, job_id, page.iteration, entry.source_node); + defer self.alloc.free(legacy_total_key); + batch.delete(legacy_total_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + for (entries.items) |entry| { + const attempt_key = try self.graphMetricBuildAttemptHitsHubRawKeyAlloc(metric_name, job_id, .hits_hub_contributions, page.iteration, page.page_id, page.attempt, entry.source_node); + defer self.alloc.free(attempt_key); + batch.delete(attempt_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + return .{ .adopted = entries.items.len, .reached_end = reached_end }; + } + + pub fn deleteGraphMetricMaterialization(self: *GraphIndex, metric_name: []const u8) !void { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + try self.deleteGraphMetricMaterializationInBatch(&batch, cfg.name, now_ms); + if (self.pairedHitsMetricConfig(cfg)) |pair| { + try self.deleteGraphMetricMaterializationInBatch(&batch, pair.name, now_ms); + } + try batch.commit(); + } + + fn deleteGraphMetricMaterializationInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + now_ms: u64, + ) !void { + const active_job_id = if (try self.metricBuildJob(batch, metric_name)) |job| job.job_id else 0; + // A durable tombstone distinguishes an operator-requested delete from + // an uninitialized background metric. Explicit refresh/rebuild/resume + // clears it; autonomous maintenance must leave the metric deleted. + const disabled_key = try self.graphMetricDisabledKeyAlloc(metric_name); + defer self.alloc.free(disabled_key); + try putU64(batch, disabled_key, 1); + + // Visibility changes are O(1). Physical reclamation is checkpointed in + // bounded pages by the maintenance runtime, so delete latency and the + // writer transaction do not grow with graph size. + inline for (.{ + try self.graphMetricPublishedGenerationKeyAlloc(metric_name), + try self.graphMetricDirtyGenerationKeyAlloc(metric_name), + try self.graphMetricMaintenancePausedKeyAlloc(metric_name), + try self.graphMetricBuildLeaseKeyAlloc(metric_name), + try self.graphMetricBuildJobKeyAlloc(metric_name), + try self.graphMetricControlKeyAlloc(&.{ metric_name, "requested" }), + // Full materialization cleanup subsumes generation retirement. + // Dropping these control records makes operator deletion immune to + // a saturated two-slot retirement queue. + try self.graphMetricRetiredScoreGenerationKeyAlloc(metric_name), + try self.graphMetricNextRetiredScoreGenerationKeyAlloc(metric_name), + try self.graphMetricRetiredScoreCleanupPhaseKeyAlloc(metric_name), + try self.graphMetricRetiredScoreCleanupCursorKeyAlloc(metric_name), + }) |key| { + defer self.alloc.free(key); + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + const cleanup_phase_key = try self.graphMetricDeleteCleanupPhaseKeyAlloc(metric_name); + defer self.alloc.free(cleanup_phase_key); + try putU64(batch, cleanup_phase_key, 1); + const cleanup_cursor_key = try self.graphMetricDeleteCleanupCursorKeyAlloc(metric_name); + defer self.alloc.free(cleanup_cursor_key); + try batch.put(cleanup_cursor_key, ""); + const cleanup_job_id_key = try self.graphMetricDeleteCleanupJobIdKeyAlloc(metric_name); + defer self.alloc.free(cleanup_job_id_key); + try putU64(batch, cleanup_job_id_key, active_job_id); + try self.appendGraphMetricEvent(batch, metric_name, .{ + .kind = .delete, + .at_ms = now_ms, + .target_edge_generation = try self.graphMetricCurrentGenerationInTxn(batch, metric_name), + .published_generation = 0, + .score_count = 0, + }); + } + + /// Reclaim one bounded page for an operator-deleted metric. Returns true + /// when work was performed; callers may fairly interleave metrics. + pub fn cleanupDeletedGraphMetricMaterializationPage(self: *GraphIndex, metric_name: []const u8) !bool { + const data_prefix = try self.graphMetricKeyAlloc(&.{metric_name}); + defer self.alloc.free(data_prefix); + const phase_key = try self.graphMetricDeleteCleanupPhaseKeyAlloc(metric_name); + defer self.alloc.free(phase_key); + const cursor_key = try self.graphMetricDeleteCleanupCursorKeyAlloc(metric_name); + defer self.alloc.free(cursor_key); + const job_id_key = try self.graphMetricDeleteCleanupJobIdKeyAlloc(metric_name); + defer self.alloc.free(job_id_key); + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + if (!try self.metricDisabled(&batch, metric_name)) { + try batch.commit(); + return false; + } + const phase = try readU64OrZero(&batch, phase_key); + if (phase == 0) { + try batch.commit(); + return false; + } + const cursor = batch.get(cursor_key) catch |err| switch (err) { + error.NotFound => "", + else => return err, + }; + const job_id = try readU64OrZero(&batch, job_id_key); + const prefix = if (phase == 1) + data_prefix + else if (job_id != 0) + try self.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id) + else + ""; + defer if (phase != 1 and prefix.len > 0) self.alloc.free(prefix); + + if (prefix.len > 0) { + var deleted = try self.deleteKeysWithPrefixPageInBatch(&batch, prefix, cursor, graph_metric_build_cleanup_delete_page_units); + defer deleted.deinit(self.alloc); + if (!deleted.reached_end) { + try batch.put(cursor_key, deleted.cursor); + try batch.commit(); + return true; + } + } + + if (phase == 1 and job_id != 0) { + try putU64(&batch, phase_key, 2); + try batch.put(cursor_key, ""); + } else { + inline for (.{ phase_key, cursor_key, job_id_key }) |key| { + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + try batch.commit(); + return true; + } + + pub fn pauseGraphMetricMaintenance(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + try self.pauseGraphMetricMaintenanceInBatch(&batch, cfg.name, now_ms); + if (self.pairedHitsMetricConfig(cfg)) |pair| { + try self.pauseGraphMetricMaintenanceInBatch(&batch, pair.name, now_ms); + } + try batch.commit(); + return try self.graphMetricStatus(metric_name); + } + + fn pauseGraphMetricMaintenanceInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, now_ms: u64) !void { + const key = try self.graphMetricMaintenancePausedKeyAlloc(metric_name); + defer self.alloc.free(key); + try putU64(batch, key, 1); + try self.appendGraphMetricEvent(batch, metric_name, .{ + .kind = .pause, + .at_ms = now_ms, + .target_edge_generation = try self.graphMetricCurrentGenerationInTxn(batch, metric_name), + .published_generation = try self.metricPublishedEdgeGeneration(batch, metric_name), + .score_count = 0, + }); + } + + pub fn resumeGraphMetricMaintenance(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + try self.resumeGraphMetricMaintenanceInBatch(&batch, cfg.name, now_ms); + if (self.pairedHitsMetricConfig(cfg)) |pair| { + try self.resumeGraphMetricMaintenanceInBatch(&batch, pair.name, now_ms); + } + try batch.commit(); + return try self.graphMetricStatus(metric_name); + } + + fn resumeGraphMetricMaintenanceInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8, now_ms: u64) !void { + const key = try self.graphMetricMaintenancePausedKeyAlloc(metric_name); + defer self.alloc.free(key); + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + const disabled_key = try self.graphMetricDisabledKeyAlloc(metric_name); + defer self.alloc.free(disabled_key); + batch.delete(disabled_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + inline for (.{ + try self.graphMetricDeleteCleanupPhaseKeyAlloc(metric_name), + try self.graphMetricDeleteCleanupCursorKeyAlloc(metric_name), + try self.graphMetricDeleteCleanupJobIdKeyAlloc(metric_name), + }) |cleanup_key| { + defer self.alloc.free(cleanup_key); + batch.delete(cleanup_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + try self.appendGraphMetricEvent(batch, metric_name, .{ + .kind = .@"resume", + .at_ms = now_ms, + .target_edge_generation = try self.graphMetricCurrentGenerationInTxn(batch, metric_name), + .published_generation = try self.metricPublishedEdgeGeneration(batch, metric_name), + .score_count = 0, + }); + } + + /// Re-enable an operator-deleted metric without changing pause state or + /// emitting a synthetic resume event. Explicit refresh/rebuild actions use + /// this immediately before acquiring their build lease. + pub fn enableGraphMetric(self: *GraphIndex, metric_name: []const u8) !void { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const pair_cfg = self.pairedHitsMetricConfig(cfg); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const requested_disabled = try self.metricDisabled(&txn, cfg.name); + const pair_disabled = if (pair_cfg) |pair| try self.metricDisabled(&txn, pair.name) else false; + if (!requested_disabled and !pair_disabled) return; + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + try self.enableGraphMetricInBatch(&batch, cfg.name); + if (pair_cfg) |pair| { + try self.enableGraphMetricInBatch(&batch, pair.name); + } + try batch.commit(); + } + + fn enableGraphMetricInBatch(self: *GraphIndex, batch: anytype, metric_name: []const u8) !void { + const disabled_key = try self.graphMetricDisabledKeyAlloc(metric_name); + defer self.alloc.free(disabled_key); + batch.delete(disabled_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + inline for (.{ + try self.graphMetricDeleteCleanupPhaseKeyAlloc(metric_name), + try self.graphMetricDeleteCleanupCursorKeyAlloc(metric_name), + try self.graphMetricDeleteCleanupJobIdKeyAlloc(metric_name), + }) |key| { + defer self.alloc.free(key); + batch.delete(key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + + fn publishGraphMetricScores( + self: *GraphIndex, + metric_name: []const u8, + target_generation: u64, + scores: []const GraphMetricScore, + meta: GraphMetricMeta, + ) !GraphMetricStatus { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const score_generation = if (try self.metricBuildJob(&batch, metric_name)) |job| blk: { + if (job.target_generation != target_generation) return error.GraphMetricBuildJobMismatch; + break :blk job.score_generation; + } else try self.allocateGraphMetricScoreGenerationInBatch(&batch, metric_name, cfg, target_generation); + const prior_published = try self.metricPublishedGeneration(&batch, metric_name); + try self.putGraphMetricScoresInBatch(&batch, metric_name, score_generation, scores); + var published_meta = meta; + published_meta.target_edge_generation = target_generation; + try self.publishGraphMetricPointerInBatch(&batch, metric_name, score_generation, published_meta); + if (prior_published != 0 and prior_published != score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, metric_name, prior_published); + } + try self.clearGraphMetricFailureInBatch(&batch, metric_name); + try self.appendGraphMetricEvent(&batch, metric_name, .{ + .kind = .publish, + .at_ms = meta.computed_at_ms, + .target_edge_generation = target_generation, + .published_generation = target_generation, + .score_count = scores.len, + }); + try batch.commit(); + return try self.graphMetricStatus(metric_name); + } + + fn publishVerifiedGraphMetricBuild( + self: *GraphIndex, + metric_name: []const u8, + job_id: u64, + score_count: usize, + meta: GraphMetricMeta, + ) !GraphMetricStatus { + const verification = try self.verifyGraphMetricBuildPublishReady(metric_name, job_id); + if (verification.config_fingerprint != meta.config_fingerprint) return error.InvalidGraphMetricBuildManifest; + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.job_id != job_id) return error.GraphMetricBuildJobMismatch; + const prior_published = try self.metricPublishedGeneration(&batch, metric_name); + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + try self.validateGraphMetricBuildExecutionInTxn(&batch, metric_name, job, cfg); + if (job.phase != .publish_generation) return error.GraphMetricBuildPublishNotReady; + var published_meta = meta; + published_meta.target_edge_generation = verification.target_generation; + try self.verifyGraphMetricRankReady(&batch, metric_name, job, metric_name, score_count); + try self.publishGraphMetricPointerInBatch(&batch, metric_name, verification.score_generation, published_meta); + try self.appendGraphMetricEvent(&batch, metric_name, .{ + .kind = .publish, + .at_ms = meta.computed_at_ms, + .target_edge_generation = verification.target_generation, + .published_generation = verification.target_generation, + .score_count = score_count, + }); + if (prior_published != 0 and prior_published != verification.score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, metric_name, prior_published); + } + try self.clearGraphMetricFailureInBatch(&batch, metric_name); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = job.job_id, + .target_generation = job.target_generation, + .score_generation = job.score_generation, + .started_at_ms = job.started_at_ms, + .updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .lease_expires_at_ms = job.lease_expires_at_ms, + .phase = .cleanup_old_generations, + .iteration = job.iteration, + .worker_id = job.worker_id, + .completed_units = verification.completed_pages, + .total_units = verification.expected_pages, + }); + if (try self.metricBuildLease(&batch, metric_name)) |lease| { + if (lease.job_id == job.job_id) { + var updated_lease = lease; + updated_lease.phase = .cleanup_old_generations; + updated_lease.iteration = job.iteration; + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(updated_lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(updated_lease, encoded); + try batch.put(lease_key, encoded); + } + } + try batch.commit(); + return try self.graphMetricStatus(metric_name); + } + + fn publishGraphMetricScorePair( + self: *GraphIndex, + first_metric_name: []const u8, + first_scores: []const GraphMetricScore, + second_metric_name: []const u8, + second_scores: []const GraphMetricScore, + target_generation: u64, + first_meta: GraphMetricMeta, + second_meta: GraphMetricMeta, + ) !void { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const cfg = self.metricConfig(first_metric_name) orelse return error.MetricNotReady; + const score_generation = if (try self.metricBuildJob(&batch, first_metric_name)) |job| blk: { + if (job.target_generation != target_generation) return error.GraphMetricBuildJobMismatch; + break :blk job.score_generation; + } else try self.allocateGraphMetricScoreGenerationInBatch(&batch, first_metric_name, cfg, target_generation); + const first_prior_published = try self.metricPublishedGeneration(&batch, first_metric_name); + const second_prior_published = try self.metricPublishedGeneration(&batch, second_metric_name); + var published_first_meta = first_meta; + published_first_meta.target_edge_generation = target_generation; + var published_second_meta = second_meta; + published_second_meta.target_edge_generation = target_generation; + try self.putGraphMetricScoresInBatch(&batch, first_metric_name, score_generation, first_scores); + try self.putGraphMetricScoresInBatch(&batch, second_metric_name, score_generation, second_scores); + try self.publishGraphMetricPointerInBatch(&batch, first_metric_name, score_generation, published_first_meta); + try self.publishGraphMetricPointerInBatch(&batch, second_metric_name, score_generation, published_second_meta); + try self.appendGraphMetricEvent(&batch, first_metric_name, .{ + .kind = .publish, + .at_ms = first_meta.computed_at_ms, + .target_edge_generation = target_generation, + .published_generation = target_generation, + .score_count = first_scores.len, + }); + try self.appendGraphMetricEvent(&batch, second_metric_name, .{ + .kind = .publish, + .at_ms = second_meta.computed_at_ms, + .target_edge_generation = target_generation, + .published_generation = target_generation, + .score_count = second_scores.len, + }); + if (first_prior_published != 0 and first_prior_published != score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, first_metric_name, first_prior_published); + } + if (second_prior_published != 0 and second_prior_published != score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, second_metric_name, second_prior_published); + } + try self.clearGraphMetricFailureInBatch(&batch, first_metric_name); + try self.clearGraphMetricFailureInBatch(&batch, second_metric_name); + try batch.commit(); + } + + fn putGraphMetricScoresInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + target_generation: u64, + scores: []const GraphMetricScore, + ) !void { + try self.putGraphMetricScorePageInBatch(batch, metric_name, target_generation, scores); + try self.mergeGraphMetricRankPrefixInBatch(batch, metric_name, target_generation, scores); + } + + /// Writes the node-keyed primary lane. Planned builds also co-write an + /// ordered staging run; direct small builds maintain their bounded tier. + fn putGraphMetricScorePageInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + target_generation: u64, + scores: []const GraphMetricScore, + ) !void { + // The check shares the writer transaction with the score writes. A + // build that started before an operator delete can therefore neither + // recreate deleted scores nor race the tombstone at commit time. + if (try self.metricDisabled(batch, metric_name)) return error.GraphMetricDisabled; + var generation_buf: [20]u8 = undefined; + const generation_text = try std.fmt.bufPrint(&generation_buf, "{d}", .{target_generation}); + var score_key = std.ArrayListUnmanaged(u8).empty; + defer score_key.deinit(self.alloc); + for (scores) |score| { + if (!std.math.isFinite(score.score)) return error.InvalidGraphMetricScore; + try self.writeGraphMetricKey(&score_key, &.{ metric_name, "score", generation_text, score.node }); + try putF64(batch, score_key.items, score.score); + } + } + + fn graphMetricRankStagePrefixAlloc(self: *GraphIndex, owner: []const u8, job_id: u64, metric: []const u8, lane: []const u8) ![]u8 { + var job_buf: [20]u8 = undefined; + const job_text = try std.fmt.bufPrint(&job_buf, "{d}", .{job_id}); + return self.graphMetricControlKeyAlloc(&.{ owner, "job", job_text, lane, metric }); + } + + /// Small, sorted producer runs use the backend's existing ordered merge. + /// Score, staging key, and page-attempt checkpoint share one transaction. + /// Reclaimed attempts can replace a value without leaving a stale rank key. + fn putPlannedGraphMetricScorePageInBatch(self: *GraphIndex, batch: anytype, owner: []const u8, job: GraphMetricBuildJob, metric: []const u8, scores: []const GraphMetricScore) !void { + const prior = try self.plannedGraphMetricPriorScoresAlloc(batch, metric, job.score_generation, scores); + defer self.alloc.free(prior); + try self.putPlannedGraphMetricScorePageWithPrior(batch, owner, job, metric, scores, prior); + } + + fn plannedGraphMetricPriorScoresAlloc(self: *GraphIndex, batch: anytype, metric: []const u8, generation: u64, scores: []const GraphMetricScore) ![]?f64 { + // Read the entire previous page before staging mutations. In backends + // with a linear pending-write overlay, interleaving reads and writes + // makes a large publication checkpoint quadratic. + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const keys = try arena.allocator().alloc([]const u8, scores.len); + for (scores, 0..) |score, i| { + const primary = try self.graphMetricScoreKeyAlloc(metric, generation, score.node); + defer self.alloc.free(primary); + keys[i] = try arena.allocator().dupe(u8, primary); + } + const prior = try self.getManyValuesAlloc(batch, keys); + defer self.alloc.free(prior); + const old_scores = try self.alloc.alloc(?f64, scores.len); + errdefer self.alloc.free(old_scores); + for (prior, old_scores) |raw, *old| old.* = if (raw) |value| decodeF64(value) orelse return error.InvalidGraphMetricScore else null; + return old_scores; + } + + fn putPlannedGraphMetricScorePageWithPrior(self: *GraphIndex, batch: anytype, owner: []const u8, job: GraphMetricBuildJob, metric: []const u8, scores: []const GraphMetricScore, old_scores: []const ?f64) !void { + std.debug.assert(scores.len == old_scores.len); + const stage = try self.graphMetricRankStagePrefixAlloc(owner, job.job_id, metric, "rank-run"); + defer self.alloc.free(stage); + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric, job.score_generation); + defer self.alloc.free(rank_prefix); + var key = std.ArrayListUnmanaged(u8).empty; + defer key.deinit(self.alloc); + for (scores, old_scores) |score, old_value| { + if (old_value) |old| { + if (old != score.score) { + const old_rank = try self.graphMetricRankKeyAlloc(metric, job.score_generation, old, score.node); + defer self.alloc.free(old_rank); + key.clearRetainingCapacity(); + try key.appendSlice(self.alloc, stage); + try key.appendSlice(self.alloc, old_rank[rank_prefix.len..]); + batch.delete(key.items) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + const rank = try self.graphMetricRankKeyAlloc(metric, job.score_generation, score.score, score.node); + defer self.alloc.free(rank); + key.clearRetainingCapacity(); + try key.appendSlice(self.alloc, stage); + try key.appendSlice(self.alloc, rank[rank_prefix.len..]); + try putF64(batch, key.items, score.score); + } + try self.putGraphMetricScorePageInBatch(batch, metric, job.score_generation, scores); + } + + const graph_metric_rank_checkpoint_entries = 256; + + /// Measures only atomic score/staging/cursor publication, excluding graph + /// computation and the final top-k merge. Every sample uses a fresh job. + pub fn benchmarkScorePublication(self: *GraphIndex, scores: []const GraphMetricScore, limit: usize, generation: u64) !usize { + const job = GraphMetricBuildJob{ .job_id = generation, .score_generation = generation, .target_generation = generation }; + const cursor = try self.graphMetricRankStagePrefixAlloc("bench", generation, "bench", "publication-benchmark-cursor"); + defer self.alloc.free(cursor); + var offset: usize = 0; + var commits: usize = 0; + while (offset < scores.len) { + const end = @min(scores.len, offset + limit); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + try self.putPlannedGraphMetricScorePageInBatch(&batch, "bench", job, "bench", scores[offset..end]); + try putU64(&batch, cursor, end); + try batch.commit(); + commits += 1; + offset = end; + } + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try readU64OrZero(&txn, cursor) != scores.len) return error.InvalidBenchmarkResult; + for (scores) |score| { + const key = try self.graphMetricScoreKeyAlloc("bench", generation, score.node); + defer self.alloc.free(key); + const actual = decodeF64(try txn.get(key)) orelse return error.InvalidBenchmarkResult; + if (actual != score.score) return error.InvalidBenchmarkResult; + } + return commits; + } + + /// The producer barrier makes the staging lane immutable. Each transaction + /// copies at most 256 ordered winners and persists its cursor atomically. + /// Only the final receipt authorizes publication; both HITS lanes require + /// receipts. Temporary runs live in the already bounded-cleanup job tree. + fn checkpointGraphMetricRankPrefix(self: *GraphIndex, owner: []const u8, cfg: GraphMetricConfig, job: GraphMetricBuildJob, metric: []const u8, score_count: usize) !bool { + const limit = @min(score_count, graph_metric_rank_entry_limit); + const stage = try self.graphMetricRankStagePrefixAlloc(owner, job.job_id, metric, "rank-run"); + defer self.alloc.free(stage); + const progress_key = try self.graphMetricRankStagePrefixAlloc(owner, job.job_id, metric, "rank-progress"); + defer self.alloc.free(progress_key); + const ready_key = try self.graphMetricRankStagePrefixAlloc(owner, job.job_id, metric, "rank-ready"); + defer self.alloc.free(ready_key); + const output = try self.graphMetricRankPrefixAlloc(metric, job.score_generation); + defer self.alloc.free(output); + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const persisted = try self.metricBuildJob(&batch, owner) orelse return error.GraphMetricBuildJobNotFound; + if (persisted.job_id != job.job_id or persisted.phase != .publish_generation or persisted.iteration != job.iteration) return error.GraphMetricBuildJobMismatch; + try self.validateGraphMetricBuildExecutionInTxn(&batch, owner, persisted, cfg); + if (try self.metricDisabled(&batch, metric)) return error.GraphMetricDisabled; + if (batch.get(ready_key)) |raw| { + if (raw.len != 8 or std.mem.readInt(u64, raw[0..8], .little) != limit) return error.InvalidGraphMetricRankEntry; + try batch.commit(); + return true; + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + var count: usize = 0; + var cursor: []const u8 = ""; + if (batch.get(progress_key)) |raw| { + if (raw.len <= 8) return error.InvalidGraphMetricRankEntry; + count = std.math.cast(usize, std.mem.readInt(u64, raw[0..8], .little)) orelse return error.InvalidGraphMetricRankEntry; + cursor = try temp.dupe(u8, raw[8..]); + if (count == 0 or count >= limit or !std.mem.startsWith(u8, cursor, stage)) return error.InvalidGraphMetricRankEntry; + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + const Pending = struct { key: []const u8, value: []const u8 }; + var pending = std.ArrayListUnmanaged(Pending).empty; + { + var cur = try batch.openCursor(); + defer cur.close(); + var next = try cur.seekAtOrAfter(if (cursor.len == 0) stage else cursor); + if (next) |entry| if (std.mem.eql(u8, entry.key, cursor)) { + next = try cur.next(); + }; + while (count < limit and pending.items.len < graph_metric_rank_checkpoint_entries) { + const entry = next orelse return error.GraphMetricBuildPublishNotReady; + if (!std.mem.startsWith(u8, entry.key, stage)) return error.GraphMetricBuildPublishNotReady; + const score = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(score)) return error.InvalidGraphMetricScore; + const node = (try self.graphMetricNodeFromRankKeyAlloc(entry.key, stage)) orelse return error.InvalidGraphMetricRankEntry; + defer self.alloc.free(node); + const canonical = try self.graphMetricRankKeyAlloc(metric, job.score_generation, score, node); + defer self.alloc.free(canonical); + if (!std.mem.eql(u8, canonical[output.len..], entry.key[stage.len..])) return error.InvalidGraphMetricRankEntry; + const primary_key = try self.graphMetricScoreKeyAlloc(metric, job.score_generation, node); + defer self.alloc.free(primary_key); + const primary = batch.get(primary_key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildPublishNotReady, + else => return err, + }; + if (!std.mem.eql(u8, primary, entry.value)) return error.InvalidGraphMetricScore; + const key = try std.mem.concat(temp, u8, &.{ output, entry.key[stage.len..] }); + try pending.append(temp, .{ .key = key, .value = try temp.dupe(u8, entry.value) }); + cursor = try temp.dupe(u8, entry.key); + count += 1; + next = try cur.next(); + } + } + for (pending.items) |entry| try batch.put(entry.key, entry.value); + if (count == limit) { + try putU64(&batch, ready_key, count); + } else { + const progress = try temp.alloc(u8, 8 + cursor.len); + std.mem.writeInt(u64, progress[0..8], count, .little); + @memcpy(progress[8..], cursor); + try batch.put(progress_key, progress); + } + try batch.commit(); + return count == limit; + } + + fn verifyGraphMetricRankReady(self: *GraphIndex, txn: anytype, owner: []const u8, job: GraphMetricBuildJob, metric: []const u8, score_count: usize) !void { + const key = try self.graphMetricRankStagePrefixAlloc(owner, job.job_id, metric, "rank-ready"); + defer self.alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return error.GraphMetricBuildPublishNotReady, + else => return err, + }; + if (raw.len != 8 or std.mem.readInt(u64, raw[0..8], .little) != @min(score_count, graph_metric_rank_entry_limit)) return error.InvalidGraphMetricRankEntry; + } + + const RankIndexCandidate = struct { + key: []u8, + score: f64, + existed: bool, + + fn lessThan(_: void, left: @This(), right: @This()) bool { + return std.mem.lessThan(u8, left.key, right.key); + } + }; + + /// Merge a score page into the generation's exact supported top-K prefix. + /// Existing entries and the page-local top-K are the only possible members + /// of the new global top-K, keeping memory and durable rank writes bounded + /// independently of graph size. + fn mergeGraphMetricRankPrefixInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + generation: u64, + scores: []const GraphMetricScore, + ) !void { + if (scores.len == 0) return; + + var page_top = std.PriorityQueue(GraphMetricScore, void, graphMetricScoreWorstFirst).initContext({}); + defer page_top.deinit(self.alloc); + try page_top.ensureTotalCapacity(self.alloc, @min(scores.len, graph_metric_rank_entry_limit)); + for (scores) |score| { + if (page_top.items.len < graph_metric_rank_entry_limit) { + try page_top.push(self.alloc, score); + } else if (graphMetricScoreComesBefore(score, page_top.peek().?)) { + _ = page_top.pop(); + try page_top.push(self.alloc, score); + } + } + + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(rank_prefix); + // Once the prefix is full, most later pages do not change it. Compare + // the page's best candidate with the durable worst boundary before + // allocating/copying the 10k-entry merge set. + var existing_count: usize = 0; + var worst_existing_key: []u8 = ""; + defer if (worst_existing_key.len > 0) self.alloc.free(worst_existing_key); + { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(rank_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, rank_prefix)) break; + existing_count += 1; + if (existing_count == graph_metric_rank_entry_limit) { + worst_existing_key = try self.alloc.dupe(u8, entry.key); + break; + } + } + } + if (existing_count == graph_metric_rank_entry_limit) { + var best = page_top.items[0]; + for (page_top.items[1..]) |candidate| { + if (graphMetricScoreComesBefore(candidate, best)) best = candidate; + } + const best_key = try self.graphMetricRankKeyAlloc(metric_name, generation, best.score, best.node); + defer self.alloc.free(best_key); + if (!std.mem.lessThan(u8, best_key, worst_existing_key)) return; + } + + var candidates = std.ArrayListUnmanaged(RankIndexCandidate).empty; + defer { + for (candidates.items) |candidate| self.alloc.free(candidate.key); + candidates.deinit(self.alloc); + } + try candidates.ensureTotalCapacity(self.alloc, graph_metric_rank_entry_limit + page_top.items.len); + var existing = std.StringHashMapUnmanaged(void).empty; + defer existing.deinit(self.alloc); + try existing.ensureTotalCapacity(self.alloc, graph_metric_rank_entry_limit); + { + var cur = try batch.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(rank_prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, rank_prefix)) break; + // Generations produced by this format are bounded. Failing + // closed here avoids silently treating an incomplete prefix + // as authoritative if persistent state is malformed. + if (candidates.items.len == graph_metric_rank_entry_limit) return error.InvalidGraphMetricRankEntry; + const score = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(score)) return error.InvalidGraphMetricScore; + const key = try self.alloc.dupe(u8, entry.key); + errdefer self.alloc.free(key); + try candidates.append(self.alloc, .{ .key = key, .score = score, .existed = true }); + try existing.put(self.alloc, key, {}); + } + } + + for (page_top.items) |score| { + const key = try self.graphMetricRankKeyAlloc(metric_name, generation, score.score, score.node); + if (existing.contains(key)) { + self.alloc.free(key); + continue; + } + try candidates.append(self.alloc, .{ .key = key, .score = score.score, .existed = false }); + } + std.mem.sort(RankIndexCandidate, candidates.items, {}, RankIndexCandidate.lessThan); + const keep = @min(candidates.items.len, graph_metric_rank_entry_limit); + for (candidates.items, 0..) |candidate, i| { + if (i < keep) { + if (!candidate.existed) try putF64(batch, candidate.key, candidate.score); + } else if (candidate.existed) { + batch.delete(candidate.key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + } + } + } + + /// Replaces the bounded secondary rank index with a top-K set selected from + /// the complete immutable score generation. Deleting and installing the + /// prefix in the publication transaction means readers can observe neither + /// a partial rank tier nor a generation pointer without its exact tier. + fn putExactGraphMetricRankPrefixInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + generation: u64, + scores: []const GraphMetricScore, + ) !void { + if (scores.len > graph_metric_rank_entry_limit) return error.InvalidGraphMetricTopK; + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(rank_prefix); + _ = try self.deleteKeysWithPrefixInBatch(batch, rank_prefix); + for (scores, 0..) |score, index| { + if (!std.math.isFinite(score.score)) return error.InvalidGraphMetricScore; + if (index > 0 and !graphMetricScoreComesBefore(scores[index - 1], score)) + return error.InvalidGraphMetricRankEntry; + const key = try self.graphMetricRankKeyAlloc(metric_name, generation, score.score, score.node); + defer self.alloc.free(key); + try putF64(batch, key, score.score); + } + } + + fn publishGraphMetricPointerInBatch( + self: *GraphIndex, + batch: anytype, + metric_name: []const u8, + target_generation: u64, + meta: GraphMetricMeta, + ) !void { + // Publication is the visibility linearization point. Never let an + // already-running worker clear a newer operator delete. + if (try self.metricDisabled(batch, metric_name)) return error.GraphMetricDisabled; + // An expired or legacy alias-owned job may finish after a replacement + // job. Keep the public pointer monotonic in the edge snapshot even + // when score-generation ids come from different HITS aliases. + if (try self.metricPublishedEdgeGeneration(batch, metric_name) > meta.target_edge_generation) { + return error.GraphMetricBuildSuperseded; + } + const published_key = try self.graphMetricPublishedGenerationKeyAlloc(metric_name); + defer self.alloc.free(published_key); + try putU64(batch, published_key, target_generation); + const dirty_key = try self.graphMetricDirtyGenerationKeyAlloc(metric_name); + defer self.alloc.free(dirty_key); + try putU64(batch, dirty_key, meta.target_edge_generation); + const meta_key = try self.graphMetricMetaKeyAlloc(metric_name, target_generation); + defer self.alloc.free(meta_key); + var meta_buf: [graph_metric_meta_encoded_len]u8 = undefined; + encodeGraphMetricMeta(meta, &meta_buf); + try batch.put(meta_key, &meta_buf); + const edge_filter_key = try self.graphMetricMetaEdgeFilterKeyAlloc(metric_name, target_generation); + defer self.alloc.free(edge_filter_key); + const edge_filter_encoded = try self.alloc.alloc(u8, graphMetricEdgeFilterEncodedLen(meta.edge_filter)); + defer self.alloc.free(edge_filter_encoded); + encodeGraphMetricEdgeFilter(meta.edge_filter, edge_filter_encoded); + try batch.put(edge_filter_key, edge_filter_encoded); + const config_fingerprint_key = try self.graphMetricMetaConfigFingerprintKeyAlloc(metric_name, target_generation); + defer self.alloc.free(config_fingerprint_key); + try putU64(batch, config_fingerprint_key, meta.config_fingerprint); + } + + fn pairedHitsMetricConfig(self: *const GraphIndex, cfg: GraphMetricConfig) ?GraphMetricConfig { + for (self.metric_configs) |candidate| { + if (std.mem.eql(u8, candidate.name, cfg.name)) continue; + if (!graphMetricHitsPairCompatible(cfg, candidate)) continue; + return candidate; + } + return null; + } + + /// Compatible HITS vectors are one materialization lifecycle. Authority is + /// the stable owner so either public alias observes and drives the same + /// lease, job, and build-page namespace. + fn graphMetricLifecycleOwnerConfig(self: *const GraphIndex, cfg: GraphMetricConfig) GraphMetricConfig { + if (cfg.kind != .hits_hub) return cfg; + if (self.pairedHitsMetricConfig(cfg)) |pair| { + if (pair.kind == .hits_authority) return pair; + } + return cfg; + } + + fn graphMetricLifecycleOwnerName(self: *const GraphIndex, metric_name: []const u8) ![]const u8 { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + return self.graphMetricLifecycleOwnerConfig(cfg).name; + } + + pub fn runGraphMetric(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const requested_cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const cfg = self.graphMetricLifecycleOwnerConfig(requested_cfg); + const owner_name = cfg.name; + try self.acquireGraphMetricBuildLease(owner_name, try self.graphMetricCurrentGeneration(owner_name)); + var release_needed = true; + defer if (release_needed) self.releaseGraphMetricBuildLease(owner_name) catch {}; + + var resolved = self.runGraphMetricResolved(cfg, owner_name) catch |err| { + self.recordGraphMetricFailure(owner_name, err) catch {}; + self.releaseGraphMetricBuildLease(owner_name) catch {}; + release_needed = false; + return err; + }; + resolved.deinit(self.alloc); + try self.completeGraphMetricBuildJob(owner_name); + try self.releaseGraphMetricBuildLease(owner_name); + release_needed = false; + return try self.graphMetricStatus(metric_name); + } + + fn runGraphMetricResolved(self: *GraphIndex, cfg: GraphMetricConfig, metric_name: []const u8) !GraphMetricStatus { + return switch (cfg.kind) { + .pagerank => try self.runPageRankMetric(metric_name), + .degree => try self.runDegreeMetric(metric_name), + .eigenvector => try self.runEigenvectorMetric(metric_name), + .hits_authority, .hits_hub => try self.runHitsMetric(metric_name), + }; + } + + pub fn ensureGraphMetricPlannedBuild( + self: *GraphIndex, + metric_name: []const u8, + target_generation: u64, + ) !GraphMetricStatus { + return self.ensureGraphMetricPlannedBuildWithPlanning(metric_name, target_generation, true); + } + + pub fn ensureGraphMetricPlannedBuildFromCachedPlan(self: *GraphIndex, metric_name: []const u8, target_generation: u64) !GraphMetricStatus { + return self.ensureGraphMetricPlannedBuildWithPlanning(metric_name, target_generation, false); + } + + fn ensureGraphMetricPlannedBuildWithPlanning(self: *GraphIndex, metric_name: []const u8, target_generation: u64, comptime drain: bool) !GraphMetricStatus { + const owner_name = try self.graphMetricLifecycleOwnerName(metric_name); + + var active_same_generation = false; + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.metricBuildLease(&txn, owner_name)) |lease| { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + if (lease.lease_expires_at_ms > now_ms) { + if (lease.target_generation != target_generation) return error.GraphMetricBuildAlreadyRunning; + _ = try self.metricBuildJob(&txn, owner_name) orelse return error.GraphMetricBuildJobNotFound; + active_same_generation = true; + } + } + } + if (active_same_generation) return try self.graphMetricStatus(metric_name); + + var snapshot_attempt: usize = 0; + while (snapshot_attempt < 3) : (snapshot_attempt += 1) { + self.acquireGraphMetricBuildLeaseWithPlanning(owner_name, target_generation, drain) catch |err| switch (err) { + error.GraphMetricBuildSnapshotChanged => { + if (snapshot_attempt + 1 == 3) return err; + continue; + }, + error.GraphMetricBuildAlreadyRunning => { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const lease = try self.metricBuildLease(&txn, owner_name) orelse return err; + if (lease.target_generation != target_generation) return err; + _ = try self.metricBuildJob(&txn, owner_name) orelse return error.GraphMetricBuildJobNotFound; + }, + else => return err, + }; + break; + } + return try self.graphMetricStatus(metric_name); + } + + pub fn runGraphMetricPlannedDrain( + self: *GraphIndex, + metric_name: []const u8, + target_generation: u64, + options: GraphMetricPlannedDrainOptions, + ) !GraphMetricStatus { + if (options.worker_ids.len == 0) return error.InvalidGraphMetricBuildWorker; + for (options.worker_ids) |worker_id| { + if (worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + } + + var status = try self.ensureGraphMetricPlannedBuild(metric_name, target_generation); + status.deinit(self.alloc); + + var idle_workers: usize = 0; + var step_index: usize = 0; + while (step_index < options.max_steps) : (step_index += 1) { + const worker_id = options.worker_ids[step_index % options.worker_ids.len]; + const worker_step = try self.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, worker_id); + if (worker_step.completed_build or worker_step.failed_build) return try self.graphMetricStatus(metric_name); + + const coordinator_step = try self.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + if (coordinator_step.completed_build or coordinator_step.failed_build) return try self.graphMetricStatus(metric_name); + + const progressed = + worker_step.claimed_page or + worker_step.completed_page or + worker_step.advanced_phase or + coordinator_step.advanced_phase or + coordinator_step.checkpointed_publication or + coordinator_step.retired_input_records != 0; + // A bounded checkpoint keeps its lease. A different worker can + // legitimately have no eligible page until the owner runs again. + idle_workers = if (progressed) 0 else idle_workers + 1; + if (idle_workers >= options.worker_ids.len and worker_step.phase != .cleanup_old_generations) { + return error.GraphMetricBuildNoEligiblePage; + } + } + return error.GraphMetricBuildNoEligiblePage; + } + + pub fn failGraphMetricPlannedBuild( + self: *GraphIndex, + metric_name: []const u8, + err: anyerror, + ) !GraphMetricStatus { + return try self.failGraphMetricPlannedBuildWithReason(metric_name, @errorName(err)); + } + + fn failGraphMetricPlannedBuildWithReason( + self: *GraphIndex, + metric_name: []const u8, + failure_reason: []const u8, + ) !GraphMetricStatus { + const owner_name = try self.graphMetricLifecycleOwnerName(metric_name); + { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const job = try self.metricBuildJob(&txn, owner_name) orelse return error.GraphMetricBuildJobNotFound; + if (job.phase == .complete) return error.GraphMetricBuildNotActive; + const lease = try self.metricBuildLease(&txn, owner_name) orelse return error.GraphMetricBuildNotActive; + if (lease.job_id != job.job_id) return error.GraphMetricBuildNotActive; + } + try self.recordGraphMetricFailureReason(owner_name, failure_reason); + return try self.graphMetricStatus(metric_name); + } + + fn failGraphMetricPlannedBuildForExhaustedPage( + self: *GraphIndex, + metric_name: []const u8, + exhaustion: GraphMetricBuildPageExhaustion, + ) !GraphMetricStatus { + const cause = if (exhaustion.last_error.len > 0) + exhaustion.last_error + else + "GraphMetricBuildPageLeaseExpired"; + const failure_reason = try std.fmt.allocPrint( + self.alloc, + "GraphMetricBuildPageAttemptsExhausted: phase={s}, iteration={d}, page_id={d}, attempt={d}, cause={s}", + .{ @tagName(exhaustion.phase), exhaustion.iteration, exhaustion.page_id, exhaustion.attempt, cause }, + ); + defer self.alloc.free(failure_reason); + return try self.failGraphMetricPlannedBuildWithReason(metric_name, failure_reason); + } + + pub fn runPageRankMetric(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .pagerank) return error.UnsupportedGraphMetric; + + var nodes = std.ArrayListUnmanaged(PageRankNode).empty; + defer { + self.freePageRankNodes(nodes.items); + nodes.deinit(self.alloc); + } + var edges = std.ArrayListUnmanaged(PageRankEdge).empty; + defer edges.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .scan_edges_and_out_degree, 0); + try self.collectPageRankGraph(cfg.edge_filter, &nodes, &edges); + + const target_generation = try self.graphMetricCurrentGeneration(metric_name); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .initialize_ranks, 0); + var result = try metric_kernels.pageRankAlloc(self.alloc, nodes.items.len, edges.items, .{ + .damping = cfg.damping, + .tolerance = cfg.tolerance, + .max_iterations = cfg.max_iterations, + .max_nodes = @max(nodes.items.len, 1), + .max_edges = @max(edges.items.len, 1), + .max_work_items = std.math.maxInt(u64), + }); + defer result.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .check_convergence, result.iterations_completed); + + const scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + defer self.alloc.free(scores); + for (nodes.items, 0..) |node, i| { + scores[i] = .{ .node = node.key, .score = result.scores[i] }; + } + + try self.updateGraphMetricBuildLeaseProgress(metric_name, .publish_generation, result.iterations_completed); + return try self.publishGraphMetricScores(metric_name, target_generation, scores, .{ + .converged = result.converged, + .iterations_completed = result.iterations_completed, + .delta = result.delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + } + + pub fn runDegreeMetric(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .degree) return error.UnsupportedGraphMetric; + + var nodes = std.ArrayListUnmanaged(DegreeNode).empty; + defer { + self.freeDegreeNodes(nodes.items); + nodes.deinit(self.alloc); + } + try self.updateGraphMetricBuildLeaseProgress(metric_name, .scan_edges_and_out_degree, 0); + try self.collectDegreeGraph(cfg.edge_filter, &nodes); + + const scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + defer self.alloc.free(scores); + for (nodes.items, 0..) |node, i| { + scores[i] = .{ + .node = node.key, + .score = @floatFromInt(node.degree), + }; + } + + try self.updateGraphMetricBuildLeaseProgress(metric_name, .publish_generation, 1); + return try self.publishGraphMetricScores(metric_name, try self.graphMetricCurrentGeneration(metric_name), scores, .{ + .converged = true, + .iterations_completed = 1, + .delta = 0.0, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + } + + pub fn runDegreeMetricPlanned(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .degree) return error.UnsupportedGraphMetric; + try self.acquireGraphMetricBuildLease(metric_name, try self.graphMetricCurrentGeneration(metric_name)); + var release_needed = true; + defer if (release_needed) self.releaseGraphMetricBuildLease(metric_name) catch {}; + + var status = self.runDegreeMetricPlannedActive(metric_name, cfg) catch |err| { + self.recordGraphMetricFailure(metric_name, err) catch {}; + self.releaseGraphMetricBuildLease(metric_name) catch {}; + release_needed = false; + return err; + }; + status.deinit(self.alloc); + try self.releaseGraphMetricBuildLease(metric_name); + release_needed = false; + return try self.graphMetricStatus(metric_name); + } + + fn runDegreeMetricPlannedActive(self: *GraphIndex, metric_name: []const u8, cfg: GraphMetricConfig) !GraphMetricStatus { + return try self.runGraphMetricPlannedActive(metric_name, cfg); + } + + pub fn runPageRankMetricPlanned(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .pagerank) return error.UnsupportedGraphMetric; + try self.acquireGraphMetricBuildLease(metric_name, try self.graphMetricCurrentGeneration(metric_name)); + var release_needed = true; + defer if (release_needed) self.releaseGraphMetricBuildLease(metric_name) catch {}; + + var status = self.runGraphMetricPlannedActive(metric_name, cfg) catch |err| { + self.recordGraphMetricFailure(metric_name, err) catch {}; + self.releaseGraphMetricBuildLease(metric_name) catch {}; + release_needed = false; + return err; + }; + status.deinit(self.alloc); + try self.releaseGraphMetricBuildLease(metric_name); + release_needed = false; + return try self.graphMetricStatus(metric_name); + } + + pub const TopologyBuildBenchmark = struct { + physical_edge_units: u64 = 0, + checkpoints: usize = 0, + adopted: bool = false, + }; + + /// Benchmark fixture only. The reference removes the reuse directory entry + /// (not source data) to measure an independent build against an adopted one. + pub fn benchmarkTopologyBuild(self: *GraphIndex, metric: []const u8, reuse: bool) !TopologyBuildBenchmark { + const cfg = self.metricConfig(metric) orelse return error.InvalidBenchmarkResult; + if (!reuse) { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const filter = try topology_owner.filterDigest(self.alloc, cfg.edge_filter); + const digest = topology_owner.identity(filter, try self.graphMetricPartitionPlanRaw(&batch, cfg)); + for ([_]bool{ false, true }) |reverse| { + const key = try topology_owner.readyKey(self.alloc, digest, reverse); + defer self.alloc.free(key); + batch.delete(key) catch |err| if (err != error.NotFound) return err; + } + try batch.commit(); + } + var status = try self.ensureGraphMetricPlannedBuild(metric, try self.graphMetricCurrentGeneration(metric)); + status.deinit(self.alloc); + var result = TopologyBuildBenchmark{}; + const job_id = blk: { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const job = (try self.metricBuildJob(&txn, metric)).?; + result.adopted = (try self.topologyBinding(&txn, metric, job.job_id)).?.adopted; + break :blk job.job_id; + }; + for (0..1_000_000) |_| { + const step = try self.runGraphMetricPlannedWorkerStep(metric, cfg, graph_metric_local_build_worker_id); + result.checkpoints += 1; + if (step.completed_page and (step.phase == .scan_edges_and_out_degree or step.phase == .iterate_contributions or step.phase == .hits_hub_contributions)) { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const page = (try self.metricBuildPage(&txn, metric, job_id, step.phase, 0, step.page_id)).?; + result.physical_edge_units += page.completed_units; + } + if (step.completed_build and step.phase == .cleanup_old_generations) return result; + } + return error.InvalidBenchmarkResult; + } + + pub fn runEigenvectorMetricPlanned(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .eigenvector) return error.UnsupportedGraphMetric; + try self.acquireGraphMetricBuildLease(metric_name, try self.graphMetricCurrentGeneration(metric_name)); + var release_needed = true; + defer if (release_needed) self.releaseGraphMetricBuildLease(metric_name) catch {}; + + var status = self.runGraphMetricPlannedActive(metric_name, cfg) catch |err| { + self.recordGraphMetricFailure(metric_name, err) catch {}; + self.releaseGraphMetricBuildLease(metric_name) catch {}; + release_needed = false; + return err; + }; + status.deinit(self.alloc); + try self.releaseGraphMetricBuildLease(metric_name); + release_needed = false; + return try self.graphMetricStatus(metric_name); + } + + pub fn runHitsMetricPlanned(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const requested_cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (requested_cfg.kind != .hits_authority and requested_cfg.kind != .hits_hub) return error.UnsupportedGraphMetric; + const cfg = self.graphMetricLifecycleOwnerConfig(requested_cfg); + const owner_name = cfg.name; + try self.acquireGraphMetricBuildLease(owner_name, try self.graphMetricCurrentGeneration(owner_name)); + var release_needed = true; + defer if (release_needed) self.releaseGraphMetricBuildLease(owner_name) catch {}; + + var status = self.runGraphMetricPlannedActive(owner_name, cfg) catch |err| { + self.recordGraphMetricFailure(owner_name, err) catch {}; + self.releaseGraphMetricBuildLease(owner_name) catch {}; + release_needed = false; + return err; + }; + status.deinit(self.alloc); + try self.releaseGraphMetricBuildLease(owner_name); + release_needed = false; + return try self.graphMetricStatus(metric_name); + } + + fn runGraphMetricPlannedActive(self: *GraphIndex, metric_name: []const u8, cfg: GraphMetricConfig) !GraphMetricStatus { + while (true) { + const step = try self.runGraphMetricPlannedWorkerStep(metric_name, cfg, graph_metric_local_build_worker_id); + if (step.failed_build) return try self.graphMetricStatus(metric_name); + if (step.completed_build and step.phase == .cleanup_old_generations) return try self.graphMetricStatus(metric_name); + if (!step.claimed_page and !step.advanced_phase and !step.checkpointed_publication and step.retired_input_records == 0) return error.GraphMetricBuildNoEligiblePage; + } + } + + fn runDegreeMetricPlannedWorkerStep( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + worker_id: []const u8, + ) !GraphMetricBuildWorkerStepResult { + if (cfg.kind != .degree) return error.UnsupportedGraphMetric; + return try self.runGraphMetricPlannedWorkerStep(metric_name, cfg, worker_id); + } + + fn runGraphMetricPlannedWorkerStep( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + worker_id: []const u8, + ) !GraphMetricBuildWorkerStepResult { + var page_step = try self.runGraphMetricPlannedWorkerPageStep(metric_name, cfg, worker_id); + if (page_step.claimed_page) { + if (page_step.completed_page and page_step.phase != .cleanup_old_generations) { + const advanced = try self.runGraphMetricPlannedCoordinatorStep(metric_name, cfg); + page_step.advanced_phase = advanced.advanced_phase; + page_step.retired_input_records = advanced.retired_input_records; + page_step.checkpointed_publication = advanced.checkpointed_publication; + page_step.failed_build = advanced.failed_build; + } + return page_step; + } + if (page_step.phase == .cleanup_old_generations) return page_step; + const advanced = try self.runGraphMetricPlannedCoordinatorStep(metric_name, cfg); + if (advanced.advanced_phase or advanced.completed_build or advanced.failed_build or advanced.checkpointed_publication or advanced.retired_input_records != 0) return advanced; + return page_step; + } + + pub fn runGraphMetricPlannedWorkerPageStep( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + worker_id: []const u8, + ) !GraphMetricBuildWorkerStepResult { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.runGraphMetricPlannedWorkerPageStepAt(metric_name, cfg, worker_id, now_ms); + } + + pub fn runGraphMetricPlannedWorkerPageStepAt( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + worker_id: []const u8, + now_ms: u64, + ) !GraphMetricBuildWorkerStepResult { + const job = blk: { + var job_txn = try self.beginReadReverseTxn(); + defer job_txn.abort(); + const decoded_job = try self.metricBuildJob(&job_txn, metric_name) orelse return error.GraphMetricBuildJobNotFound; + // Cleanup incrementally retires the immutable job namespace, which + // includes the manifest. Once publication has advanced the durable + // job into cleanup, execution is fenced by that job record and page + // leases; requiring a manifest that cleanup is designed to delete + // makes multi-batch retirement impossible. + if (decoded_job.phase != .complete and decoded_job.phase != .cleanup_old_generations) { + const manifest = try self.metricBuildManifest(&job_txn, metric_name, decoded_job.job_id) orelse + return error.GraphMetricBuildManifestNotFound; + try validateGraphMetricBuildExecution(manifest, decoded_job, cfg); + } + break :blk try self.cloneGraphMetricBuildJobAlloc(decoded_job); + }; + defer self.deinitClonedGraphMetricBuildJob(job); + + if (job.target_generation != try self.graphMetricCurrentGeneration(metric_name)) return error.GraphMetricBuildSuperseded; + // A worker may own a different reopened handle from its coordinator. + // Observe retirement locally too, including old jobs whose coordinator + // could not reach this handle's cache. Full keys still isolate epochs. + if (job.phase != .reduce_ranks and job.phase != .hits_hub_reduce_ranks) + self.sealed_vectors.retire(self.alloc, sealedVectorScope(metric_name)); + if (job.phase == .complete) return .{ .phase = .complete, .published = true, .completed_build = true }; + if (job.phase == .publish_generation and !graphMetricKindUsesPlannedIterativeRunner(cfg.kind)) { + return .{ .phase = .publish_generation }; + } + + if (!graphMetricBuildPhaseHasPageExecutor(cfg.kind, job.phase)) return error.UnsupportedGraphMetricBuildPhase; + const claim_iteration: u32 = if (job.phase == .cleanup_old_generations) 0 else job.iteration; + const page = try self.claimNextGraphMetricBuildPageAt(metric_name, job.job_id, job.phase, claim_iteration, worker_id, now_ms) orelse { + return .{ .phase = job.phase }; + }; + + const execution = self.executeGraphMetricBuildPage(metric_name, cfg, job, page) catch |err| switch (err) { + error.GraphMetricBuildPageNotLeased => return .{ + .phase = job.phase, + .page_id = page.page_id, + }, + else => { + // Persist page-level failures so deterministic execution + // errors are retried under the bounded attempt policy instead + // of crashing every maintenance tick forever. If ownership + // changed concurrently, the replacement worker owns the next + // decision and this stale result is harmless. + _ = self.failGraphMetricBuildPageForAttempt( + metric_name, + job.job_id, + page.phase, + page.iteration, + page.page_id, + page.worker_id, + page.attempt, + @errorName(err), + ) catch |fail_err| switch (fail_err) { + error.GraphMetricBuildPageNotLeased => return .{ + .phase = job.phase, + .page_id = page.page_id, + }, + else => return fail_err, + }; + return .{ + .phase = job.phase, + .page_id = page.page_id, + .claimed_page = true, + }; + }, + }; + return .{ + .phase = execution.phase, + .page_id = execution.page_id, + .claimed_page = true, + .completed_page = execution.completed_page, + .published = execution.published, + .completed_build = execution.completed_build, + }; + } + + pub fn runGraphMetricPlannedWorkerPageStepForMetric( + self: *GraphIndex, + metric_name: []const u8, + worker_id: []const u8, + ) !GraphMetricBuildWorkerStepResult { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.runGraphMetricPlannedWorkerPageStepForMetricAt(metric_name, worker_id, now_ms); + } + + pub fn runGraphMetricPlannedWorkerPageStepForMetricAt( + self: *GraphIndex, + metric_name: []const u8, + worker_id: []const u8, + now_ms: u64, + ) !GraphMetricBuildWorkerStepResult { + const requested_cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const cfg = self.graphMetricLifecycleOwnerConfig(requested_cfg); + return try self.runGraphMetricPlannedWorkerPageStepAt(cfg.name, cfg, worker_id, now_ms); + } + + pub fn runGraphMetricPlannedCoordinatorStep( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + ) !GraphMetricBuildWorkerStepResult { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.runGraphMetricPlannedCoordinatorStepAt(metric_name, cfg, now_ms); + } + + pub fn runGraphMetricPlannedCoordinatorStepAt( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + now_ms: u64, + ) !GraphMetricBuildWorkerStepResult { + var execution_failure: ?anyerror = null; + const job = blk: { + var job_txn = try self.beginReadReverseTxn(); + defer job_txn.abort(); + const decoded_job = try self.metricBuildJob(&job_txn, metric_name) orelse return error.GraphMetricBuildJobNotFound; + self.validateGraphMetricBuildExecutionInTxn(&job_txn, metric_name, decoded_job, cfg) catch |err| switch (err) { + error.InvalidGraphMetricBuildManifest, error.GraphMetricBuildManifestNotFound => execution_failure = err, + else => return err, + }; + break :blk try self.cloneGraphMetricBuildJobAlloc(decoded_job); + }; + defer self.deinitClonedGraphMetricBuildJob(job); + + if (execution_failure) |err| { + var failed = try self.failGraphMetricPlannedBuild(metric_name, err); + defer failed.deinit(self.alloc); + return .{ .phase = job.phase, .failed_build = true }; + } + if (job.target_generation != try self.graphMetricCurrentGeneration(metric_name)) { + var failed = try self.failGraphMetricPlannedBuild(metric_name, error.GraphMetricBuildSuperseded); + defer failed.deinit(self.alloc); + return .{ .phase = job.phase, .failed_build = true }; + } + if (job.phase == .complete) return .{ .phase = .complete, .published = true, .completed_build = true }; + if (job.phase == .publish_generation) { + if (graphMetricKindUsesPlannedIterativeRunner(cfg.kind)) { + if (try self.graphMetricBuildPhaseExhaustedPageAlloc(metric_name, job.job_id, job.phase, job.iteration, now_ms)) |exhaustion_value| { + var exhaustion = exhaustion_value; + defer exhaustion.deinit(self.alloc); + var failed = try self.failGraphMetricPlannedBuildForExhaustedPage(metric_name, exhaustion); + defer failed.deinit(self.alloc); + return .{ .phase = .publish_generation, .failed_build = true }; + } + const summary = try self.summarizeGraphMetricBuildPhase(metric_name, job.job_id, .publish_generation, job.iteration); + if (summary.state != .complete) return .{ .phase = .publish_generation }; + } + const published = self.publishGraphMetricBuildFromCoordinator(metric_name, cfg, job) catch |err| { + var failed = try self.failGraphMetricPlannedBuild(metric_name, err); + defer failed.deinit(self.alloc); + return .{ .phase = .publish_generation, .failed_build = true }; + }; + return .{ .phase = .publish_generation, .advanced_phase = published, .checkpointed_publication = !published }; + } + if (job.phase == .cleanup_old_generations) return .{ .phase = .cleanup_old_generations }; + if (!graphMetricBuildPhaseHasPageExecutor(cfg.kind, job.phase)) return error.UnsupportedGraphMetricBuildPhase; + if (try self.graphMetricBuildPhaseExhaustedPageAlloc(metric_name, job.job_id, job.phase, job.iteration, now_ms)) |exhaustion_value| { + var exhaustion = exhaustion_value; + defer exhaustion.deinit(self.alloc); + var failed = try self.failGraphMetricPlannedBuildForExhaustedPage(metric_name, exhaustion); + defer failed.deinit(self.alloc); + return .{ .phase = job.phase, .failed_build = true }; + } + var retired: usize = 0; + const advanced = try self.advanceGraphMetricBuildPhaseWithRetirement(metric_name, job.job_id, job.phase, job.iteration, &retired); + return .{ .phase = job.phase, .advanced_phase = advanced, .retired_input_records = retired }; + } + + pub fn runGraphMetricPlannedCoordinatorStepForMetric( + self: *GraphIndex, + metric_name: []const u8, + ) !GraphMetricBuildWorkerStepResult { + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + return try self.runGraphMetricPlannedCoordinatorStepForMetricAt(metric_name, now_ms); + } + + pub fn runGraphMetricPlannedCoordinatorStepForMetricAt( + self: *GraphIndex, + metric_name: []const u8, + now_ms: u64, + ) !GraphMetricBuildWorkerStepResult { + const requested_cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const cfg = self.graphMetricLifecycleOwnerConfig(requested_cfg); + return try self.runGraphMetricPlannedCoordinatorStepAt(cfg.name, cfg, now_ms); + } + + fn publishGraphMetricBuildFromCoordinator( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + ) !bool { + _ = try self.verifyGraphMetricBuildPublishReady(metric_name, job.job_id); + const planned_count = try self.graphMetricBuildPlannedScoreCount(metric_name, cfg, job); + if (!try self.checkpointGraphMetricRankPrefix(metric_name, cfg, job, metric_name, planned_count)) return false; + if (self.pairedHitsMetricConfig(cfg)) |pair| { + if (!try self.checkpointGraphMetricRankPrefix(metric_name, cfg, job, pair.name, planned_count)) return false; + } + switch (cfg.kind) { + .degree => { + const score_count = try self.graphMetricBuildPlannedScoreCount(metric_name, cfg, job); + var status = try self.publishVerifiedGraphMetricBuild(metric_name, job.job_id, score_count, .{ + .converged = true, + .iterations_completed = 1, + .delta = 0.0, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + status.deinit(self.alloc); + }, + .pagerank => { + const verification = try self.verifyGraphMetricBuildPublishReady(metric_name, job.job_id); + if (!verification.converged and !verification.fixed_iteration_limit) return error.GraphMetricBuildPublishNotReady; + const score_count = try self.graphMetricBuildPlannedScoreCount(metric_name, cfg, job); + var status = try self.publishVerifiedGraphMetricBuild(metric_name, job.job_id, score_count, .{ + .converged = verification.converged, + .iterations_completed = verification.iteration + 1, + .delta = verification.total_delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + status.deinit(self.alloc); + }, + .eigenvector => { + const verification = try self.verifyGraphMetricBuildPublishReady(metric_name, job.job_id); + if (!verification.converged and !verification.fixed_iteration_limit) return error.GraphMetricBuildPublishNotReady; + const score_count = try self.graphMetricBuildPlannedScoreCount(metric_name, cfg, job); + var status = try self.publishVerifiedGraphMetricBuild(metric_name, job.job_id, score_count, .{ + .converged = verification.converged, + .iterations_completed = verification.iteration + 1, + .delta = verification.total_delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + status.deinit(self.alloc); + }, + .hits_authority, + .hits_hub, + => { + var status = try self.publishHitsGraphMetricBuildFromCoordinator(metric_name, cfg, job); + status.deinit(self.alloc); + }, + } + return true; + } + + fn graphMetricBuildPlannedScoreCount( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + ) !usize { + if (cfg.kind != .degree) return try self.graphMetricBuildNodeCount(metric_name, job.job_id); + + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + if (try self.graphMetricReduceSummaryValue(&txn, metric_name, job.job_id, .reduce_ranks, job.iteration)) |summary| { + return try graphMetricSummaryCount(summary); + } + const manifest = try self.metricBuildManifest(&txn, metric_name, job.job_id) orelse return error.GraphMetricBuildManifestNotFound; + const node_count = std.math.cast(usize, manifest.node_count) orelse return error.InvalidGraphMetricBuildManifest; + if (graphMetricBuildPhaseNeedsSummaryPage(cfg.kind, .reduce_ranks, node_count)) { + return error.InvalidGraphMetricBuildManifest; + } + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| self.alloc.free(node); + nodes.deinit(self.alloc); + } + _ = try self.collectDegreePartialNodesInRange(&txn, metric_name, job.job_id, "", "", "", null, &nodes); + return nodes.items.len; + } + + fn hitsVectorName(kind: GraphMetricKind) []const u8 { + return switch (kind) { + .hits_authority => "authority", + .hits_hub => "hub", + else => unreachable, + }; + } + + fn publishHitsGraphMetricBuildFromCoordinator( + self: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + job: GraphMetricBuildJob, + ) !GraphMetricStatus { + const verification = try self.verifyGraphMetricBuildPublishReady(metric_name, job.job_id); + if (!verification.converged and !verification.fixed_iteration_limit) return error.GraphMetricBuildPublishNotReady; + const rank_iteration = verification.iteration + 1; + const score_count = try self.graphMetricBuildPlannedScoreCount(metric_name, cfg, job); + + const meta = GraphMetricMeta{ + .target_edge_generation = verification.target_generation, + .converged = verification.converged, + .iterations_completed = rank_iteration, + .delta = verification.total_delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }; + + const pair_cfg = self.pairedHitsMetricConfig(cfg); + if (pair_cfg == null) { + return try self.publishVerifiedGraphMetricBuild(metric_name, job.job_id, score_count, meta); + } + + const pair = pair_cfg.?; + const pair_score_count = score_count; + var pair_meta = meta; + pair_meta.config_fingerprint = graphMetricConfigFingerprint(pair); + + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const persisted_job = try self.metricBuildJob(&batch, metric_name) orelse return error.GraphMetricBuildJobNotFound; + if (persisted_job.job_id != job.job_id) return error.GraphMetricBuildJobMismatch; + try self.validateGraphMetricBuildExecutionInTxn(&batch, metric_name, persisted_job, cfg); + if (persisted_job.phase != .publish_generation) return error.GraphMetricBuildPublishNotReady; + const prior_published = try self.metricPublishedGeneration(&batch, metric_name); + const pair_prior_published = try self.metricPublishedGeneration(&batch, pair.name); + try self.verifyGraphMetricRankReady(&batch, metric_name, persisted_job, metric_name, score_count); + try self.verifyGraphMetricRankReady(&batch, metric_name, persisted_job, pair.name, score_count); + try self.publishGraphMetricPointerInBatch(&batch, metric_name, verification.score_generation, meta); + try self.publishGraphMetricPointerInBatch(&batch, pair.name, verification.score_generation, pair_meta); + try self.appendGraphMetricEvent(&batch, metric_name, .{ + .kind = .publish, + .at_ms = meta.computed_at_ms, + .target_edge_generation = verification.target_generation, + .published_generation = verification.target_generation, + .score_count = score_count, + }); + try self.appendGraphMetricEvent(&batch, pair.name, .{ + .kind = .publish, + .at_ms = pair_meta.computed_at_ms, + .target_edge_generation = verification.target_generation, + .published_generation = verification.target_generation, + .score_count = pair_score_count, + }); + if (prior_published != 0 and prior_published != verification.score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, metric_name, prior_published); + } + if (pair_prior_published != 0 and pair_prior_published != verification.score_generation) { + try self.enqueueRetiredScoreGenerationInBatch(&batch, pair.name, pair_prior_published); + } + try self.clearGraphMetricFailureInBatch(&batch, metric_name); + try self.clearGraphMetricFailureInBatch(&batch, pair.name); + try self.putGraphMetricBuildJobInBatch(&batch, metric_name, .{ + .job_id = persisted_job.job_id, + .target_generation = persisted_job.target_generation, + .score_generation = persisted_job.score_generation, + .started_at_ms = persisted_job.started_at_ms, + .updated_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .lease_expires_at_ms = persisted_job.lease_expires_at_ms, + .phase = .cleanup_old_generations, + .iteration = persisted_job.iteration, + .worker_id = persisted_job.worker_id, + .completed_units = verification.completed_pages, + .total_units = verification.expected_pages, + }); + if (try self.metricBuildLease(&batch, metric_name)) |lease| { + if (lease.job_id == persisted_job.job_id) { + var updated_lease = lease; + updated_lease.phase = .cleanup_old_generations; + updated_lease.iteration = persisted_job.iteration; + const lease_key = try self.graphMetricBuildLeaseKeyAlloc(metric_name); + defer self.alloc.free(lease_key); + const encoded = try self.alloc.alloc(u8, graphMetricBuildLeaseEncodedLen(updated_lease)); + defer self.alloc.free(encoded); + encodeGraphMetricBuildLease(updated_lease, encoded); + try batch.put(lease_key, encoded); + } + } + try batch.commit(); + return try self.graphMetricStatus(metric_name); + } + + pub fn runEigenvectorMetric(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .eigenvector) return error.UnsupportedGraphMetric; + + var nodes = std.ArrayListUnmanaged(PageRankNode).empty; + defer { + self.freePageRankNodes(nodes.items); + nodes.deinit(self.alloc); + } + var edges = std.ArrayListUnmanaged(PageRankEdge).empty; + defer edges.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .scan_edges_and_out_degree, 0); + try self.collectPageRankGraph(cfg.edge_filter, &nodes, &edges); + + const target_generation = try self.graphMetricCurrentGeneration(metric_name); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .initialize_ranks, 0); + var result = try metric_kernels.eigenvectorAlloc(self.alloc, nodes.items.len, edges.items, .{ + .tolerance = cfg.tolerance, + .max_iterations = cfg.max_iterations, + .max_nodes = @max(nodes.items.len, 1), + .max_edges = @max(edges.items.len, 1), + .max_work_items = std.math.maxInt(u64), + }); + defer result.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .check_convergence, result.iterations_completed); + + const scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + defer self.alloc.free(scores); + for (nodes.items, 0..) |node, i| { + scores[i] = .{ .node = node.key, .score = result.scores[i] }; + } + + try self.updateGraphMetricBuildLeaseProgress(metric_name, .publish_generation, result.iterations_completed); + return try self.publishGraphMetricScores(metric_name, target_generation, scores, .{ + .converged = result.converged, + .iterations_completed = result.iterations_completed, + .delta = result.delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }); + } + + pub fn runHitsMetric(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + if (cfg.kind != .hits_authority and cfg.kind != .hits_hub) return error.UnsupportedGraphMetric; + + var nodes = std.ArrayListUnmanaged(PageRankNode).empty; + defer { + self.freePageRankNodes(nodes.items); + nodes.deinit(self.alloc); + } + var edges = std.ArrayListUnmanaged(PageRankEdge).empty; + defer edges.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .scan_edges_and_out_degree, 0); + try self.collectPageRankGraph(cfg.edge_filter, &nodes, &edges); + + const target_generation = try self.graphMetricCurrentGeneration(metric_name); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .initialize_ranks, 0); + var result = try metric_kernels.hitsAlloc(self.alloc, nodes.items.len, edges.items, .{ + .tolerance = cfg.tolerance, + .max_iterations = cfg.max_iterations, + .max_nodes = @max(nodes.items.len, 1), + .max_edges = @max(edges.items.len, 1), + .max_work_items = std.math.maxInt(u64), + }); + defer result.deinit(self.alloc); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .check_convergence, result.iterations_completed); + const scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + defer self.alloc.free(scores); + var pair_scores: []GraphMetricScore = &.{}; + defer if (pair_scores.len > 0) self.alloc.free(pair_scores); + const pair_cfg = self.pairedHitsMetricConfig(cfg); + if (pair_cfg != null) pair_scores = try self.alloc.alloc(GraphMetricScore, nodes.items.len); + for (nodes.items, 0..) |node, i| { + const score = switch (cfg.kind) { + .hits_authority => result.authorities[i], + .hits_hub => result.hubs[i], + else => unreachable, + }; + if (!std.math.isFinite(score)) return error.InvalidGraphMetricScore; + scores[i] = .{ .node = node.key, .score = score }; + if (pair_cfg) |pair| { + const pair_score = switch (pair.kind) { + .hits_authority => result.authorities[i], + .hits_hub => result.hubs[i], + else => unreachable, + }; + if (!std.math.isFinite(pair_score)) return error.InvalidGraphMetricScore; + pair_scores[i] = .{ .node = node.key, .score = pair_score }; + } + } + + const meta = GraphMetricMeta{ + .converged = result.converged, + .iterations_completed = result.iterations_completed, + .delta = result.delta, + .computed_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms), + .config_fingerprint = graphMetricConfigFingerprint(cfg), + .edge_filter = cfg.edge_filter, + }; + if (pair_cfg) |pair| { + var pair_meta = meta; + pair_meta.config_fingerprint = graphMetricConfigFingerprint(pair); + try self.updateGraphMetricBuildLeaseProgress(metric_name, .publish_generation, result.iterations_completed); + try self.publishGraphMetricScorePair(metric_name, scores, pair.name, pair_scores, target_generation, meta, pair_meta); + return try self.graphMetricStatus(metric_name); + } + try self.updateGraphMetricBuildLeaseProgress(metric_name, .publish_generation, result.iterations_completed); + return try self.publishGraphMetricScores(metric_name, target_generation, scores, meta); + } + + pub fn graphMetricStatus(self: *GraphIndex, metric_name: []const u8) !GraphMetricStatus { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + return try self.graphMetricStatusInTxn(metric_name, &txn); + } + + pub fn graphMetricSchedulerStatus(self: *GraphIndex, metric_name: []const u8, now_ms: ?u64) !GraphMetricSchedulerStatus { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const lifecycle_cfg = self.graphMetricLifecycleOwnerConfig(cfg); + const lifecycle_name = lifecycle_cfg.name; + const published_generation = try self.metricPublishedGeneration(&txn, metric_name); + const dirty_generation = try self.metricDirtyGeneration(&txn, metric_name); + const current_generation = try self.graphMetricCurrentGenerationInTxn(&txn, metric_name); + const target_edge_generation = @max(dirty_generation, current_generation); + const maintenance_paused = try self.metricMaintenancePaused(&txn, lifecycle_name); + const disabled = try self.metricDisabled(&txn, metric_name); + const maybe_lease = try self.metricBuildLease(&txn, lifecycle_name); + const current_ms = now_ms orelse @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + const active_lease = if (maybe_lease) |lease| lease.lease_expires_at_ms > current_ms else false; + const last_event = try self.graphMetricLastEvent(&txn, metric_name); + + var published_edge_generation: u64 = published_generation; + var config_fingerprint_stale = false; + if (published_generation != 0) { + const meta_key = try self.graphMetricMetaKeyAlloc(metric_name, published_generation); + defer self.alloc.free(meta_key); + if (txn.get(meta_key)) |raw| { + if (decodeGraphMetricMeta(raw)) |meta| { + if (meta.target_edge_generation != 0) published_edge_generation = meta.target_edge_generation; + if (meta.schema_version >= 3) { + config_fingerprint_stale = meta.config_fingerprint != graphMetricConfigFingerprint(cfg); + } + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + } + const failed_target_generation = if (last_event != null and last_event.?.kind == .failed) + last_event.?.target_edge_generation + else + 0; + const base_state: GraphMetricState = if (disabled) + .disabled + else if (failed_target_generation == target_edge_generation and failed_target_generation != 0) + .failed + else if (published_generation == 0) + .not_ready + else if (dirty_generation > published_edge_generation or current_generation > published_edge_generation or config_fingerprint_stale) + .stale + else + .fresh; + const state: GraphMetricState = if (active_lease) .building else base_state; + return .{ + .state = state, + .phase = if (active_lease) maybe_lease.?.phase else if (state == .fresh) .complete else .idle, + .maintenance_paused = maintenance_paused, + .target_edge_generation = target_edge_generation, + .failed_target_generation = failed_target_generation, + }; + } + + fn graphMetricStatusInTxn(self: *GraphIndex, metric_name: []const u8, txn: anytype) !GraphMetricStatus { + return self.graphMetricSnapshotStatusInTxn(metric_name, txn, .operator); + } + + // Queries need publication/freshness identity, not operator history or + // scans of worker checkpoints. Both views use the caller's read snapshot. + const MetricStatusDetail = enum { query, operator }; + + fn graphMetricSnapshotStatusInTxn(self: *GraphIndex, metric_name: []const u8, txn: anytype, comptime detail: MetricStatusDetail) !GraphMetricStatus { + return self.graphMetricSnapshotStatusInTxnAlloc(self.alloc, metric_name, txn, detail); + } + + fn graphMetricSnapshotStatusInTxnAlloc(self: *GraphIndex, result_alloc: Allocator, metric_name: []const u8, txn: anytype, comptime detail: MetricStatusDetail) !GraphMetricStatus { + const cfg = self.metricConfig(metric_name) orelse return error.MetricNotReady; + const lifecycle_cfg = self.graphMetricLifecycleOwnerConfig(cfg); + const lifecycle_name = lifecycle_cfg.name; + const published_generation = try self.metricPublishedGeneration(txn, metric_name); + const dirty_generation = try self.metricDirtyGeneration(txn, metric_name); + const maintenance_paused = try self.metricMaintenancePaused(txn, lifecycle_name); + const disabled = try self.metricDisabled(txn, metric_name); + const maybe_build_lease = try self.metricBuildLease(txn, lifecycle_name); + const maybe_build_job = try self.metricBuildJob(txn, lifecycle_name); + const maybe_failure_detail = try self.metricFailureDetail(txn, metric_name); + defer if (maybe_failure_detail) |failure| failure.deinit(self.alloc); + const recent_events = if (detail == .operator) try self.graphMetricRecentEvents(txn, metric_name, graph_metric_recent_event_limit) else &.{}; + errdefer if (recent_events.len > 0) self.alloc.free(recent_events); + const recent_failures = if (detail == .operator) try self.graphMetricRecentFailureRecords(txn, metric_name, graph_metric_recent_event_limit) else &.{}; + errdefer { + for (recent_failures) |*failure| failure.deinit(self.alloc); + if (recent_failures.len > 0) self.alloc.free(recent_failures); + } + const last_event = if (detail == .query) try self.graphMetricLastEvent(txn, metric_name) else if (recent_events.len > 0) recent_events[0] else null; + var meta = GraphMetricMeta{ .schema_version = 0 }; + if (published_generation != 0) { + const meta_key = try self.graphMetricMetaKeyAlloc(metric_name, published_generation); + defer self.alloc.free(meta_key); + if (txn.get(meta_key)) |raw| { + meta = decodeGraphMetricMeta(raw) orelse .{ .schema_version = 0 }; + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + } + // The pointer addresses an internal score namespace. Preserve the + // public generation contract as the edge snapshot that produced it; + // pre-v4 materializations used the same value for both. + const published_edge_generation = if (published_generation == 0) + 0 + else if (meta.target_edge_generation != 0) + meta.target_edge_generation + else + published_generation; + const name = try result_alloc.dupe(u8, metric_name); + errdefer result_alloc.free(name); + var edge_filter = try cfg.edge_filter.cloneAlloc(result_alloc); + errdefer edge_filter.deinit(result_alloc); + var has_stored_edge_filter = false; + if (published_generation != 0) { + const edge_filter_key = try self.graphMetricMetaEdgeFilterKeyAlloc(metric_name, published_generation); + defer self.alloc.free(edge_filter_key); + if (txn.get(edge_filter_key)) |raw| { + if (try decodeGraphMetricEdgeFilterAlloc(result_alloc, raw)) |stored_edge_filter| { + edge_filter.deinit(result_alloc); + edge_filter = stored_edge_filter; + has_stored_edge_filter = true; + } + } else |err| switch (err) { + error.NotFound => {}, + else => return err, + } + } + var has_stored_config_fingerprint = false; + var stored_config_fingerprint: u64 = 0; + if (published_generation != 0) { + const config_fingerprint_key = try self.graphMetricMetaConfigFingerprintKeyAlloc(metric_name, published_generation); + defer self.alloc.free(config_fingerprint_key); + if (txn.get(config_fingerprint_key)) |raw| { + if (raw.len == 8) { + stored_config_fingerprint = std.mem.readInt(u64, raw[0..8], .little); + has_stored_config_fingerprint = true; + } + } else |err| switch (err) { + error.NotFound => { + if (meta.schema_version >= 3) { + stored_config_fingerprint = meta.config_fingerprint; + has_stored_config_fingerprint = true; + } + }, + else => return err, + } + } + const current_generation = try self.graphMetricCurrentGenerationInTxn(txn, metric_name); + const target_edge_generation = @max(dirty_generation, current_generation); + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + const active_build_lease = if (maybe_build_lease) |lease| lease.lease_expires_at_ms > now_ms else false; + const building_generation = if (active_build_lease) maybe_build_lease.?.target_generation else 0; + const build_job_id = if (active_build_lease) maybe_build_lease.?.job_id else 0; + const build_started_at_ms = if (active_build_lease) maybe_build_lease.?.started_at_ms else 0; + const build_iteration = if (active_build_lease) maybe_build_lease.?.iteration else 0; + const build_lease_expires_at_ms = if (active_build_lease) maybe_build_lease.?.lease_expires_at_ms else 0; + const active_build_worker_id = if (active_build_lease) + try result_alloc.dupe(u8, maybe_build_lease.?.worker_id) + else + ""; + errdefer if (active_build_worker_id.len > 0) result_alloc.free(active_build_worker_id); + const active_build_job = if (detail == .operator and active_build_lease and maybe_build_job != null and maybe_build_job.?.job_id == build_job_id) + maybe_build_job.? + else + null; + var active_build_progress = if (active_build_job) |job| + try self.graphMetricActiveBuildProgressAggregate(txn, lifecycle_name, job) + else + GraphMetricBuildProgressAggregate{}; + errdefer active_build_progress.deinit(self.alloc); + const active_build_cursor = active_build_progress.cursor; + active_build_progress.cursor = ""; + errdefer if (active_build_cursor.len > 0) self.alloc.free(active_build_cursor); + var active_build_page_statuses = if (active_build_job) |job| + try self.graphMetricActiveBuildPageStatuses(txn, lifecycle_name, job, graph_metric_status_page_limit) + else + GraphMetricBuildPageStatusList{}; + errdefer active_build_page_statuses.deinit(self.alloc); + const edge_filter_stale = has_stored_edge_filter and !graphMetricEdgeFiltersEqual(edge_filter, cfg.edge_filter); + const config_fingerprint_stale = has_stored_config_fingerprint and stored_config_fingerprint != graphMetricConfigFingerprint(cfg); + const base_state: GraphMetricState = if (disabled) + .disabled + else if (last_event != null and last_event.?.kind == .failed and last_event.?.target_edge_generation == target_edge_generation) + .failed + else if (published_generation == 0) + .not_ready + else if (dirty_generation > published_edge_generation or current_generation > published_edge_generation or edge_filter_stale or config_fingerprint_stale) + .stale + else + .fresh; + const failure_applies = base_state == .failed and maybe_failure_detail != null; + const last_error = if (failure_applies) + try result_alloc.dupe(u8, maybe_failure_detail.?.last_error) + else + ""; + errdefer if (last_error.len > 0) result_alloc.free(last_error); + const state: GraphMetricState = if (active_build_lease) .building else base_state; + const explicitly_queued = try self.metricBuildRequestedInTxn(txn, lifecycle_name); + const queued_generation: u64 = if (active_build_lease) + if (target_edge_generation > building_generation) target_edge_generation else 0 + else if (explicitly_queued) target_edge_generation else switch (base_state) { + .not_ready, .stale, .failed => target_edge_generation, + else => 0, + }; + const phase: GraphMetricBuildPhase = if (state == .fresh) .complete else .idle; + const active_phase = if (active_build_lease) maybe_build_lease.?.phase else phase; + return .{ + .name = name, + .state = state, + .phase = active_phase, + .edge_filter = edge_filter, + .metadata_version = meta.schema_version, + .config_fingerprint = stored_config_fingerprint, + .maintenance_paused = maintenance_paused, + .build_queued = explicitly_queued or queued_generation != 0, + .published_generation = published_generation, + .published_edge_generation = published_edge_generation, + .edge_generation = current_generation, + .target_edge_generation = target_edge_generation, + .queued_generation = queued_generation, + .building_generation = building_generation, + .build_job_id = build_job_id, + .build_started_at_ms = build_started_at_ms, + .build_iteration = build_iteration, + .build_lease_expires_at_ms = build_lease_expires_at_ms, + .build_worker_id = active_build_worker_id, + .build_cursor = active_build_cursor, + .build_completed_units = active_build_progress.completed_units, + .build_total_units = active_build_progress.total_units, + .build_pages = active_build_page_statuses.pages, + .build_pages_truncated = active_build_page_statuses.truncated, + .retry_count = if (failure_applies) maybe_failure_detail.?.retry_count else 0, + .last_error = last_error, + .progress = if (active_build_lease) graphMetricActiveBuildProgress( + lifecycle_cfg, + active_phase, + build_iteration, + if (active_build_progress.total_units == 0) + 0.0 + else + @as(f64, @floatFromInt(active_build_progress.completed_units)) / + @as(f64, @floatFromInt(active_build_progress.total_units)), + ) else if (phase == .complete) 1.0 else 0.0, + .converged = meta.converged, + .iterations_completed = meta.iterations_completed, + .delta = meta.delta, + .computed_at_ms = meta.computed_at_ms, + .last_event = last_event, + .recent_events = recent_events, + .recent_failures = recent_failures, + }; + } + + fn graphMetricNodeFromScoreKeyAlloc(self: *GraphIndex, key: []const u8, prefix: []const u8) !?[]u8 { + if (!std.mem.startsWith(u8, key, prefix)) return null; + const pos = prefix.len; + const term = internal_keys.findComponentTerminator(key, pos) orelse return null; + if (term + 2 != key.len) return null; + return try internal_keys.decodeBodyAlloc(self.alloc, key[pos..term]); + } + + fn graphMetricNodeFromRankKeyAlloc(self: *GraphIndex, key: []const u8, prefix: []const u8) !?[]u8 { + if (!std.mem.startsWith(u8, key, prefix)) return null; + const score_pos = prefix.len; + const score_term = internal_keys.findComponentTerminator(key, score_pos) orelse return null; + const node_pos = score_term + 2; + const node_term = internal_keys.findComponentTerminator(key, node_pos) orelse return null; + if (node_term + 2 != key.len) return null; + return try internal_keys.decodeBodyAlloc(self.alloc, key[node_pos..node_term]); + } + + pub fn graphMetricTopK(self: *GraphIndex, metric_name: []const u8, limit: usize) ![]GraphMetricScore { + if (limit == 0) return try self.alloc.alloc(GraphMetricScore, 0); + if (limit > graph_metric_rank_entry_limit) return error.InvalidGraphMetricTopK; + _ = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const generation = try self.metricPublishedGeneration(&txn, metric_name); + if (generation == 0) return error.MetricNotReady; + + return self.graphMetricTopKInTxnAlloc(&txn, metric_name, generation, limit); + } + + pub fn graphMetricTopKSnapshotAlloc( + self: *GraphIndex, + metric_name: []const u8, + limit: usize, + ) !GraphMetricTopKSnapshot { + if (limit > graph_metric_rank_entry_limit) return error.InvalidGraphMetricTopK; + _ = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var status = try self.graphMetricSnapshotStatusInTxn(metric_name, &txn, .query); + errdefer status.deinit(self.alloc); + if (status.published_generation == 0) return error.MetricNotReady; + const scores = if (limit == 0) + try self.alloc.alloc(GraphMetricScore, 0) + else + try self.graphMetricTopKInTxnAlloc(&txn, metric_name, status.published_generation, limit); + return .{ .status = status, .scores = scores }; + } + + fn graphMetricTopKInTxnAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + generation: u64, + limit: usize, + ) ![]GraphMetricScore { + // Current materializations maintain a score-ordered secondary keyspace, + // so ordinary top-K reads touch only K entries. Keep the node-keyed scan + // below as an upgrade fallback for materializations written by older + // releases; the next refresh replaces them with ranked keys. + const rank_prefix = try self.graphMetricRankPrefixAlloc(metric_name, generation); + defer self.alloc.free(rank_prefix); + var ranked = std.ArrayListUnmanaged(GraphMetricScore).empty; + defer ranked.deinit(self.alloc); + errdefer for (ranked.items) |*score| score.deinit(self.alloc); + try ranked.ensureTotalCapacity(self.alloc, limit); + { + var rank_cur = try txn.openCursor(); + defer rank_cur.close(); + var rank_entry_opt = try rank_cur.seekAtOrAfter(rank_prefix); + while (rank_entry_opt) |entry| : (rank_entry_opt = try rank_cur.next()) { + if (!std.mem.startsWith(u8, entry.key, rank_prefix)) break; + const score_value = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(score_value)) return error.InvalidGraphMetricScore; + const node = (try self.graphMetricNodeFromRankKeyAlloc(entry.key, rank_prefix)) orelse return error.InvalidGraphMetricRankEntry; + try ranked.append(self.alloc, .{ .node = node, .score = score_value }); + if (ranked.items.len == limit) break; + } + } + if (ranked.items.len > 0) { + const out = try self.alloc.dupe(GraphMetricScore, ranked.items); + ranked.items.len = 0; + return out; + } + + return try self.selectGraphMetricTopKFromScoresInTxnAlloc(txn, metric_name, generation, limit); + } + + fn selectGraphMetricTopKFromScoresInTxnAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + generation: u64, + limit: usize, + ) ![]GraphMetricScore { + if (limit == 0) return try self.alloc.alloc(GraphMetricScore, 0); + const prefix = try self.graphMetricScorePrefixAlloc(metric_name, generation); + defer self.alloc.free(prefix); + var scores = std.PriorityQueue(GraphMetricScore, void, graphMetricScoreWorstFirst).initContext({}); + defer scores.deinit(self.alloc); + errdefer for (scores.items) |*score| score.deinit(self.alloc); + try scores.ensureTotalCapacity(self.alloc, limit); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.seekAtOrAfter(prefix); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + const score_value = decodeF64(entry.value) orelse return error.InvalidGraphMetricScore; + if (!std.math.isFinite(score_value)) return error.InvalidGraphMetricScore; + // Once the heap is full, a strictly lower score cannot enter it; + // avoid decoding and allocating that node key on the common path. + if (scores.peek()) |worst| { + if (scores.items.len == limit and score_value < worst.score) continue; + } + const node = (try self.graphMetricNodeFromScoreKeyAlloc(entry.key, prefix)) orelse + return error.InvalidGraphMetricRankEntry; + errdefer self.alloc.free(node); + const candidate = GraphMetricScore{ .node = node, .score = score_value }; + if (scores.items.len < limit) { + try scores.push(self.alloc, candidate); + continue; + } + const worst = scores.peek().?; + if (!graphMetricScoreComesBefore(candidate, worst)) { + self.alloc.free(node); + continue; + } + var removed = scores.pop().?; + removed.deinit(self.alloc); + try scores.push(self.alloc, candidate); + } + std.mem.sort(GraphMetricScore, scores.items, {}, graphMetricScoreLessThan); + const out = try self.alloc.dupe(GraphMetricScore, scores.items); + scores.items.len = 0; + return out; + } + + fn selectGraphMetricTopKFromScoresAlloc( + self: *GraphIndex, + metric_name: []const u8, + generation: u64, + limit: usize, + ) ![]GraphMetricScore { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + return try self.selectGraphMetricTopKFromScoresInTxnAlloc(&txn, metric_name, generation, limit); + } + + fn graphMetricScoreComesBefore(a: GraphMetricScore, b: GraphMetricScore) bool { + if (a.score == b.score) return std.mem.lessThan(u8, a.node, b.node); + return a.score > b.score; + } + + fn graphMetricScoreLessThan(_: void, a: GraphMetricScore, b: GraphMetricScore) bool { + return graphMetricScoreComesBefore(a, b); + } + + fn graphMetricScoreWorstFirst(_: void, a: GraphMetricScore, b: GraphMetricScore) std.math.Order { + if (graphMetricScoreComesBefore(a, b)) return .gt; + if (graphMetricScoreComesBefore(b, a)) return .lt; + return .eq; + } + + pub fn graphMetricScore(self: *GraphIndex, metric_name: []const u8, node: []const u8) !?f64 { + _ = self.metricConfig(metric_name) orelse return error.MetricNotReady; + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + const generation = try self.metricPublishedGeneration(&txn, metric_name); + if (generation == 0) return null; + const score_key = try self.graphMetricScoreKeyAlloc(metric_name, generation, node); + defer self.alloc.free(score_key); + const raw = txn.get(score_key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + return decodeF64(raw); + } + + /// Batches point lookups with status under one stable read snapshot. The + /// generation is intentionally not accepted from the caller: doing so can + /// race publication cleanup and silently turn valid scores into misses. + pub fn graphMetricScoreSnapshotAlloc( + self: *GraphIndex, + metric_name: []const u8, + nodes: []const []const u8, + ) !GraphMetricScoreSnapshot { + return self.graphMetricScoreSnapshotWithPolicyAlloc(metric_name, nodes, .{}); + } + + pub fn graphMetricScoreSnapshotWithPolicyAlloc( + self: *GraphIndex, + metric_name: []const u8, + nodes: []const []const u8, + policy: GraphMetricColumnReadPolicy, + ) !GraphMetricScoreSnapshot { + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var status = try self.graphMetricSnapshotStatusInTxn(metric_name, &txn, .query); + errdefer status.deinit(self.alloc); + if ((policy.require_published or policy.require_fresh) and status.published_generation == 0) return error.MetricNotReady; + if (policy.require_fresh and status.state != .fresh) return error.MetricStale; + const scores = try self.graphMetricScoresInTxnAlloc(&txn, metric_name, status.published_generation, nodes); + return .{ .status = status, .scores = scores }; + } + + /// Synthetic published columns for read benchmarks. No topology or metric + /// execution is included in fixture setup or measured query work. + pub fn benchmarkSeedScoreColumns(self: *GraphIndex, names: []const []const u8, nodes: []const []const u8) !void { + for (names) |name| { + var offset: usize = 0; + while (offset < nodes.len) { + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const end = @min(nodes.len, offset + 4096); + for (nodes[offset..end], offset..) |node, i| { + const key = try self.graphMetricScoreKeyAlloc(name, 1, node); + defer self.alloc.free(key); + try putF64(&batch, key, @floatFromInt(i)); + } + try batch.commit(); + offset = end; + } + var batch = try self.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try self.graphMetricPublishedGenerationKeyAlloc(name); + defer self.alloc.free(key); + try putU64(&batch, key, 1); + try batch.commit(); + } + } + + /// Benchmark oracle for the former operator-status query path. It shares + /// the exact score reader and read transaction with production snapshots. + pub fn benchmarkScoreSnapshotAlloc(self: *GraphIndex, metric_name: []const u8, nodes: []const []const u8, operator_details: bool) !GraphMetricScoreSnapshot { + if (!operator_details) return self.graphMetricScoreSnapshotAlloc(metric_name, nodes); + var txn = try self.beginReadReverseTxn(); + defer txn.abort(); + var status = try self.graphMetricStatusInTxn(metric_name, &txn); + errdefer status.deinit(self.alloc); + const scores = try self.graphMetricScoresInTxnAlloc(&txn, metric_name, status.published_generation, nodes); + return .{ .status = status, .scores = scores }; + } + + /// One publication snapshot spanning filter, order and projection reads. + /// Resolve every policy before reading scores, including dependencies whose + /// eventual row selection is empty. Never reopen between query stages. + pub const GraphMetricReadSession = struct { + alloc: Allocator, + txn: backend_erased.ReadTxn, + statuses: []GraphMetricStatus, + prefixes: []?[]const u8, + reads: score_read.Stats = .{}, + + pub fn deinit(self: *@This()) void { + self.txn.abort(); + for (self.statuses) |*status| status.deinit(self.alloc); + self.alloc.free(self.statuses); + for (self.prefixes) |prefix| if (prefix) |value| self.alloc.free(value); + self.alloc.free(self.prefixes); + self.* = undefined; + } + + pub fn readColumns(self: *@This(), alloc: Allocator, names: []const []const u8, nodes: []const []const u8, columns: []const []?f64) !void { + if (names.len != columns.len) return error.InvalidQueryRequest; + for (columns) |column| if (column.len != nodes.len) return error.InvalidQueryRequest; + const prefixes = try alloc.alloc(?[]const u8, names.len); + defer alloc.free(prefixes); + for (names, prefixes) |name, *prefix| { + prefix.* = for (self.statuses, self.prefixes) |status, value| { + if (std.mem.eql(u8, status.name, name)) break value; + } else return error.InvalidQueryRequest; + } + const read_stats = try score_read.populate(alloc, &self.txn, prefixes, nodes, columns); + self.reads.keys += read_stats.keys; + self.reads.batches += read_stats.batches; + } + }; + + pub fn openGraphMetricReadSession(self: *GraphIndex, metric_names: []const []const u8, policies: []const GraphMetricColumnReadPolicy) !GraphMetricReadSession { + return self.openGraphMetricReadSessionAlloc(self.alloc, metric_names, policies); + } + + pub fn openGraphMetricReadSessionAlloc(self: *GraphIndex, alloc: Allocator, metric_names: []const []const u8, policies: []const GraphMetricColumnReadPolicy) !GraphMetricReadSession { + if (metric_names.len != policies.len) return error.InvalidQueryRequest; + var txn = try self.beginReadReverseTxn(); + errdefer txn.abort(); + const statuses = try alloc.alloc(GraphMetricStatus, metric_names.len); + var initialized: usize = 0; + errdefer { + for (statuses[0..initialized]) |*status| status.deinit(alloc); + alloc.free(statuses); + } + for (metric_names, policies, 0..) |name, policy, i| { + statuses[i] = try self.graphMetricSnapshotStatusInTxnAlloc(alloc, name, &txn, .query); + initialized += 1; + if ((policy.require_published or policy.require_fresh) and statuses[i].published_generation == 0) return error.MetricNotReady; + if (policy.require_fresh and statuses[i].state != .fresh) return error.MetricStale; + } + const prefixes = try alloc.alloc(?[]const u8, metric_names.len); + @memset(prefixes, null); + errdefer { + for (prefixes) |prefix| if (prefix) |value| alloc.free(value); + alloc.free(prefixes); + } + for (metric_names, statuses, prefixes) |name, status, *prefix| { + if (status.published_generation != 0) { + var generation_buf: [20]u8 = undefined; + const generation = try std.fmt.bufPrint(&generation_buf, "{d}", .{status.published_generation}); + prefix.* = try graphMetricKeyWithAllocator(alloc, &.{ name, "score", generation }); + } + } + return .{ .alloc = alloc, .txn = txn, .statuses = statuses, .prefixes = prefixes }; + } + + pub fn graphMetricColumnsSnapshotAlloc( + self: *GraphIndex, + metric_names: []const []const u8, + nodes: []const []const u8, + policies: []const GraphMetricColumnReadPolicy, + ) !GraphMetricColumnsSnapshot { + var session = try self.openGraphMetricReadSession(metric_names, policies); + defer session.deinit(); + const score_columns = try self.alloc.alloc([]?f64, metric_names.len); + var initialized: usize = 0; + errdefer { + for (score_columns[0..initialized]) |column| self.alloc.free(column); + self.alloc.free(score_columns); + } + for (score_columns) |*column| { + column.* = try self.alloc.alloc(?f64, nodes.len); + initialized += 1; + } + try session.readColumns(self.alloc, metric_names, nodes, score_columns); + const statuses = session.statuses; + session.statuses = &.{}; + return .{ .statuses = statuses, .score_columns = score_columns }; + } + + fn graphMetricScoresInTxnAlloc( + self: *GraphIndex, + txn: anytype, + metric_name: []const u8, + generation: u64, + nodes: []const []const u8, + ) ![]?f64 { + const scores = try self.alloc.alloc(?f64, nodes.len); + errdefer self.alloc.free(scores); + const prefix = if (generation != 0) try self.graphMetricScorePrefixAlloc(metric_name, generation) else null; + defer if (prefix) |value| self.alloc.free(value); + _ = try score_read.populate(self.alloc, txn, &.{prefix}, nodes, &.{scores}); + return scores; + } + + fn edgeOwnedBytes(edge: Edge) usize { + var total: usize = @sizeOf(Edge); + total = std.math.add(usize, total, edge.source.len) catch return std.math.maxInt(usize); + total = std.math.add(usize, total, edge.target.len) catch return std.math.maxInt(usize); + total = std.math.add(usize, total, edge.edge_type.len) catch return std.math.maxInt(usize); + return std.math.add(usize, total, edge.metadata.len) catch std.math.maxInt(usize); + } + + fn edgeScanCursorFromPhysicalKey(alloc: Allocator, direction: EdgeDirection, type_index: u32, edge: Edge) !EdgeScanCursor { + const edge_type = try alloc.dupe(u8, edge.edge_type); + errdefer alloc.free(edge_type); + return .{ + .direction = direction, + .type_index = type_index, + .edge_type = edge_type, + .adjacent_key = try alloc.dupe(u8, if (direction == .out) edge.target else edge.source), + }; + } + + fn edgeScanStartCursor(alloc: Allocator, direction: EdgeDirection, type_index: u32, edge_type_name: []const u8) !EdgeScanCursor { + const edge_type = try alloc.dupe(u8, edge_type_name); + errdefer alloc.free(edge_type); + return .{ + .direction = direction, + .type_index = type_index, + .edge_type = edge_type, + .adjacent_key = try alloc.alloc(u8, 0), + .at_phase_start = true, + }; + } + + /// Free an edge's allocated fields. + pub fn freeEdge(alloc: Allocator, edge: Edge) void { + alloc.free(edge.source); + alloc.free(edge.target); + alloc.free(edge.edge_type); + if (edge.metadata.len > 0) alloc.free(edge.metadata); + } + + /// Free a slice of edges returned by getEdges. + pub fn freeEdges(alloc: Allocator, edges: []Edge) void { + for (edges) |e| freeEdge(alloc, e); + alloc.free(edges); + } +}; + +const RuntimeStoreHandle = struct { + store: backend_erased.Store, + owned: bool, +}; + +fn initRuntimeStore(alloc: Allocator, store: anytype) !RuntimeStoreHandle { + const T = @TypeOf(store); + if (T == backend_erased.Store) return .{ .store = store, .owned = true }; + if (T == *backend_erased.Store) return .{ .store = store.*, .owned = false }; + + switch (@typeInfo(T)) { + .pointer => |ptr| { + if (@typeInfo(ptr.child) == .@"struct" and @hasDecl(ptr.child, "backendStore")) { + return .{ + .store = try backend_erased.storeFrom(alloc, store.backendStore()), + .owned = true, + }; + } + }, + .@"struct" => { + if (@hasDecl(T, "backendStore")) { + return .{ + .store = try backend_erased.storeFrom(alloc, store.backendStore()), + .owned = true, + }; + } + }, + else => {}, + } + + return .{ + .store = try backend_erased.storeFrom(alloc, store), + .owned = true, + }; +} + +// ============================================================================ +// Tests +// ============================================================================ + +fn openTestGraphIndex(alloc: Allocator, store: anytype, path: [*:0]const u8, name: []const u8, opts: GraphIndexOptions) !GraphIndex { + var index = try GraphIndex.open(alloc, store, path, name, opts); + index.test_partition_target_units = graph_metric_build_target_scan_page_units; + return index; +} + +fn tmpPath(buf: []u8, label: []const u8) [*:0]const u8 { + const ns = platform_time.monotonicNs(); + const slice = std.fmt.bufPrint(buf, "/tmp/antfly-graph-{s}-{d}\x00", .{ label, ns }) catch unreachable; + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().createDirPath(io_impl.io(), std.mem.span(@as([*:0]const u8, @ptrCast(slice.ptr)))) catch {}; + return @ptrCast(slice.ptr); +} + +fn cleanupTmp(path: [*:0]const u8) void { + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), std.mem.span(path)) catch {}; +} + +test "graph addEdge and getEdges out" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); + defer graph.close(); + + try graph.addEdge("doc1", "doc2", "cites", 0.9, 1000, 1001, "{}"); + try graph.addEdge("doc1", "doc3", "cites", 0.5, 1000, 1001, ""); + + const edges = try graph.getEdges(alloc, "doc1", "cites", .out); + defer GraphIndex.freeEdges(alloc, edges); + + try std.testing.expectEqual(@as(usize, 2), edges.len); + try std.testing.expectEqualStrings("doc1", edges[0].source); + try std.testing.expectApproxEqAbs(@as(f64, 0.9), edges[0].weight, 0.001); +} + +test "graph pagerank metric publishes top-k scores and status" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .max_iterations = 40, + .tolerance = 0.0000001, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-d", "cites", 1.0, 0, 0, ""); + + var status = try graph.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, status.state); + try std.testing.expectEqual(@as(u32, 0), status.metadata_version); + + var published = try graph.runPageRankMetric("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(@as(u32, GraphIndex.graph_metric_meta_schema_version), published.metadata_version); + try std.testing.expect(published.published_generation > 0); + try std.testing.expect(published.iterations_completed > 0); + + const top = try graph.graphMetricTopK("pagerank", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("doc-d", top[0].node); + try std.testing.expect(top[0].score >= top[1].score); + + try graph.addEdge("doc-d", "doc-a", "cites", 1.0, 0, 0, ""); + var stale = try graph.graphMetricStatus("pagerank"); + defer stale.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, stale.state); +} + +test "graph metric dirty marker survives reopen and rebuilds later generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-dirty-reopen"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-dirty-reopen"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .max_iterations = 40, + .tolerance = 0.0000001, + .refresh = .manual, + }}; + + var first_generation: u64 = 0; + var dirty_generation: u64 = 0; + var expected_top_node = try alloc.alloc(u8, 0); + defer alloc.free(expected_top_node); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + first_generation = published.published_generation; + + const top = try graph.graphMetricTopK("pagerank", 1); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 1), top.len); + alloc.free(expected_top_node); + expected_top_node = try alloc.dupe(u8, top[0].node); + + try graph.addEdge("doc-c", "doc-a", "cites", 1.0, 0, 0, ""); + dirty_generation = graph.edge_generation; + var stale = try graph.graphMetricStatus("pagerank"); + defer stale.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, stale.state); + try std.testing.expect(stale.build_queued); + try std.testing.expectEqual(dirty_generation, stale.queued_generation); + try std.testing.expectEqual(first_generation, stale.published_generation); + } + store.close(); + + store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var reopened_stale = try graph.graphMetricStatus("pagerank"); + defer reopened_stale.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, reopened_stale.state); + try std.testing.expect(reopened_stale.build_queued); + try std.testing.expectEqual(dirty_generation, reopened_stale.edge_generation); + try std.testing.expectEqual(dirty_generation, reopened_stale.queued_generation); + try std.testing.expectEqual(first_generation, reopened_stale.published_generation); + + const stale_top = try graph.graphMetricTopK("pagerank", 1); + defer { + for (stale_top) |*score| score.deinit(alloc); + alloc.free(stale_top); + } + try std.testing.expectEqual(@as(usize, 1), stale_top.len); + try std.testing.expectEqualStrings(expected_top_node, stale_top[0].node); + + var rebuilt = try graph.runGraphMetric("pagerank"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expectEqual(dirty_generation, rebuilt.published_generation); + try std.testing.expect(!rebuilt.build_queued); +} + +test "graph metric status marks algorithm config drift stale" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-config-drift"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-config-drift"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .damping = 0.85, + .max_iterations = 40, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + + @constCast(graph.metric_configs)[0].damping = 0.90; + var stale = try graph.graphMetricStatus("pagerank"); + defer stale.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, stale.state); + try std.testing.expect(stale.build_queued); + try std.testing.expectEqual(stale.edge_generation, stale.queued_generation); +} + +test "graph metric metadata preserves score epoch input and decodes v3" { + const current = GraphIndex.GraphMetricMeta{ + .target_edge_generation = 41, + .converged = true, + .iterations_completed = 7, + .delta = 0.125, + .computed_at_ms = 1234, + .config_fingerprint = 0x1122334455667788, + }; + var encoded: [GraphIndex.graph_metric_meta_encoded_len]u8 = undefined; + GraphIndex.encodeGraphMetricMeta(current, &encoded); + const decoded = GraphIndex.decodeGraphMetricMeta(&encoded) orelse return error.TestExpectedGraphMetricMeta; + try std.testing.expectEqual(GraphIndex.graph_metric_meta_schema_version, decoded.schema_version); + try std.testing.expectEqual(current.target_edge_generation, decoded.target_edge_generation); + try std.testing.expectEqual(current.converged, decoded.converged); + try std.testing.expectEqual(current.iterations_completed, decoded.iterations_completed); + try std.testing.expectEqual(current.delta, decoded.delta); + try std.testing.expectEqual(current.computed_at_ms, decoded.computed_at_ms); + try std.testing.expectEqual(current.config_fingerprint, decoded.config_fingerprint); + + // V3 encoded the same convergence fields and fingerprint but used its key + // generation as the edge snapshot identity. + var v3: [GraphIndex.graph_metric_meta_v3_encoded_len]u8 = undefined; + var offset: usize = 0; + inline for (.{ + @as(u64, 3), + @as(u64, 1), + @as(u64, 5), + @as(u64, @bitCast(@as(f64, 0.25))), + @as(u64, 5678), + @as(u64, 0x8877665544332211), + }) |value| { + std.mem.writeInt(u64, v3[offset..][0..8], value, .little); + offset += 8; + } + const legacy = GraphIndex.decodeGraphMetricMeta(&v3) orelse return error.TestExpectedGraphMetricMeta; + try std.testing.expectEqual(@as(u32, 3), legacy.schema_version); + try std.testing.expectEqual(@as(u64, 0), legacy.target_edge_generation); + try std.testing.expectEqual(@as(u64, 0x8877665544332211), legacy.config_fingerprint); +} + +test "graph metric failed rebuild preserves published generation and records event" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-failure"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-failure"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + @constCast(graph.metric_configs)[0].damping = std.math.nan(f64); + try std.testing.expectError(error.InvalidGraphMetricOptions, graph.runGraphMetric("pagerank")); + + var failed = try graph.graphMetricStatus("pagerank"); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricOptions", failed.last_error); + try std.testing.expect(failed.last_event != null); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, failed.last_event.?.kind); + try std.testing.expect(failed.recent_events.len >= 2); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, failed.recent_events[0].kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, failed.recent_events[1].kind); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const failed_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricOptions", failed_job.last_error); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, failed_job.phase); + try std.testing.expect(failed_job.score_generation != failed_job.target_generation); + try std.testing.expect(failed_job.score_generation != failed.published_generation); + } + + const top = try graph.graphMetricTopK("pagerank", 1); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 1), top.len); + + @constCast(graph.metric_configs)[0].damping = 0.85; + var recovered = try graph.runGraphMetric("pagerank"); + defer recovered.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, recovered.state); + try std.testing.expectEqual(@as(u64, 0), recovered.retry_count); + try std.testing.expectEqualStrings("", recovered.last_error); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const recovered_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, recovered_job.phase); + try std.testing.expectEqual(@as(u64, 0), recovered_job.retry_count); + try std.testing.expectEqualStrings("", recovered_job.last_error); + } +} + +test "graph metric rebuild at unchanged edge generation publishes an isolated score epoch" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-same-edge-rebuild"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-same-edge-rebuild"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ .name = "degree", .kind = .degree, .refresh = .manual }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-a", "doc-c", "cites", 1.0, 0, 0, ""); + + var first = try graph.runGraphMetric("degree"); + defer first.deinit(alloc); + const edge_generation = first.published_edge_generation; + const first_score_generation = first.published_generation; + + var rebuilt = try graph.runGraphMetric("degree"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(edge_generation, rebuilt.published_edge_generation); + try std.testing.expect(rebuilt.published_generation > first_score_generation); + try std.testing.expectEqual(@as(usize, 3), try graph.countGraphMetricScoreGeneration("degree", rebuilt.published_generation)); + + const top = try graph.graphMetricTopK("degree", 3); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 3), top.len); + try std.testing.expectEqualStrings("doc-a", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top[0].score, 0.001); + + var cleanup_steps: usize = 0; + while (try graph.cleanupRetiredGraphMetricScoreGenerationPage("degree")) { + cleanup_steps += 1; + if (cleanup_steps > 8) return error.TestGraphMetricCleanupDidNotConverge; + } + try std.testing.expect(cleanup_steps > 0); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("degree", first_score_generation)); +} + +test "graph metric failed rebuild preserves published generation across reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-failure-reopen"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-failure-reopen"); + defer cleanupTmp(rev_path); + + var metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + }}; + + var published_generation: u64 = 0; + var expected_top_node = try alloc.alloc(u8, 0); + defer alloc.free(expected_top_node); + var expected_top_score: f64 = 0.0; + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("pagerank", 1); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 1), before_failure_top.len); + alloc.free(expected_top_node); + expected_top_node = try alloc.dupe(u8, before_failure_top[0].node); + expected_top_score = before_failure_top[0].score; + + @constCast(graph.metric_configs)[0].damping = std.math.nan(f64); + try std.testing.expectError(error.InvalidGraphMetricOptions, graph.runGraphMetric("pagerank")); + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const failed_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricOptions", failed_job.last_error); + } + store.close(); + + metrics[0].damping = 0.85; + store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var failed = try graph.graphMetricStatus("pagerank"); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricOptions", failed.last_error); + try std.testing.expect(failed.build_queued); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, failed.last_event.?.kind); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const failed_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricOptions", failed_job.last_error); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, failed_job.phase); + } + + const after_reopen_top = try graph.graphMetricTopK("pagerank", 1); + defer { + for (after_reopen_top) |*score| score.deinit(alloc); + alloc.free(after_reopen_top); + } + try std.testing.expectEqual(@as(usize, 1), after_reopen_top.len); + try std.testing.expectEqualStrings(expected_top_node, after_reopen_top[0].node); + try std.testing.expectApproxEqAbs(expected_top_score, after_reopen_top[0].score, 0.0000000001); + + var recovered = try graph.runGraphMetric("pagerank"); + defer recovered.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, recovered.state); + try std.testing.expectEqual(@as(u64, 0), recovered.retry_count); + try std.testing.expectEqualStrings("", recovered.last_error); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const recovered_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, recovered_job.phase); + try std.testing.expectEqual(@as(u64, 0), recovered_job.retry_count); + try std.testing.expectEqualStrings("", recovered_job.last_error); + } +} + +test "graph metric status exposes queued and active local build lease" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-lease"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-lease"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + + var queued = try graph.graphMetricStatus("degree"); + defer queued.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, queued.state); + try std.testing.expect(queued.build_queued); + try std.testing.expectEqual(graph.edge_generation, queued.queued_generation); + try std.testing.expectEqual(@as(u64, 0), queued.building_generation); + + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var building = try graph.graphMetricStatus("degree"); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, building.phase); + try std.testing.expectEqual(graph.edge_generation, building.building_generation); + try std.testing.expect(building.build_job_id != 0); + try std.testing.expect(building.build_started_at_ms != 0); + try std.testing.expectEqual(@as(u32, 0), building.build_iteration); + try std.testing.expectEqual(@as(u64, 0), building.queued_generation); + try std.testing.expectEqualStrings(graph_metric_local_build_worker_id, building.build_worker_id); + try std.testing.expect(building.build_lease_expires_at_ms > 0); + try std.testing.expectApproxEqAbs(@as(f64, 0.01), building.progress, 0.0001); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, active_job.job_id); + try std.testing.expectEqual(graph.edge_generation, active_job.target_generation); + try std.testing.expectEqual(graph.edge_generation, active_job.score_generation); + try std.testing.expectEqual(building.build_started_at_ms, active_job.started_at_ms); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, active_job.phase); + try std.testing.expectEqual(@as(u32, 0), active_job.iteration); + try std.testing.expectEqualStrings(graph_metric_local_build_worker_id, active_job.worker_id); + const cached_raw = try job_txn.get(graph_metric_partition_plan_key); + var cached_plan = (try graph.decodeGraphMetricPartitionPlanAlloc(cached_raw)) orelse return error.TestExpectedGraphMetricBuildManifest; + defer cached_plan.deinit(alloc); + try std.testing.expectEqual(graph.edge_generation, cached_plan.edge_generation); + try std.testing.expectEqual(graph.edge_count, cached_plan.edge_count); + try std.testing.expectEqual(graph.node_count, cached_plan.node_count); + + const checksum_corrupt = try alloc.dupe(u8, cached_raw); + defer alloc.free(checksum_corrupt); + checksum_corrupt[checksum_corrupt.len - 1] ^= 0x01; + try std.testing.expect((try graph.decodeGraphMetricPartitionPlanAlloc(checksum_corrupt)) == null); + + const malformed_count = try alloc.dupe(u8, cached_raw); + defer alloc.free(malformed_count); + std.mem.writeInt(u32, malformed_count[28..32], 0, .little); + const checksum_offset = malformed_count.len - 8; + std.mem.writeInt( + u64, + malformed_count[checksum_offset..][0..8], + std.hash.Wyhash.hash(graph_metric_partition_plan_checksum_seed, malformed_count[0..checksum_offset]), + .little, + ); + try std.testing.expect((try graph.decodeGraphMetricPartitionPlanAlloc(malformed_count)) == null); + + try std.testing.expectEqual(@as(usize, 76), cached_raw.len); + const control_copy = try alloc.dupe(u8, cached_raw); + defer alloc.free(control_copy); + try graph.materializeGraphMetricPartitionPlan(&job_txn, graph_metric_partition_plan_key, &cached_plan); + try std.testing.expect(cached_plan.edge_boundaries.items.len > 0); + cached_plan.boundary_digest[0] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.materializeGraphMetricPartitionPlan(&job_txn, graph_metric_partition_plan_key, &cached_plan)); + // Control validation remains allocation-free even with persisted data. + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + const saved_allocator = graph.alloc; + graph.alloc = failing.allocator(); + defer graph.alloc = saved_allocator; + var header_only = (try graph.decodeGraphMetricPartitionPlanAlloc(control_copy)).?; + defer header_only.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), header_only.edge_boundaries.items.len); + } + + try graph.updateGraphMetricBuildLeaseProgressWithCursor("degree", .computing, 5, "edge-page:0007", 7, 20); + var iterating = try graph.graphMetricStatus("degree"); + defer iterating.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, iterating.state); + try std.testing.expectEqual(building.build_job_id, iterating.build_job_id); + try std.testing.expectEqual(building.build_started_at_ms, iterating.build_started_at_ms); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.computing, iterating.phase); + try std.testing.expectEqual(@as(u32, 5), iterating.build_iteration); + try std.testing.expectEqualStrings("edge-page:0007", iterating.build_cursor); + try std.testing.expectEqual(@as(u64, 7), iterating.build_completed_units); + try std.testing.expectEqual(@as(u64, 20), iterating.build_total_units); + try std.testing.expectApproxEqAbs(@as(f64, 0.1), iterating.progress, 0.0001); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(iterating.build_job_id, active_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.computing, active_job.phase); + try std.testing.expectEqual(@as(u32, 5), active_job.iteration); + try std.testing.expectEqualStrings("edge-page:0007", active_job.cursor); + try std.testing.expectEqual(@as(u64, 7), active_job.completed_units); + try std.testing.expectEqual(@as(u64, 20), active_job.total_units); + try std.testing.expect(active_job.updated_at_ms >= active_job.started_at_ms); + } + + try std.testing.expectError(error.GraphMetricBuildAlreadyRunning, graph.runGraphMetric("degree")); + + try graph.releaseGraphMetricBuildLease("degree"); + + const custom_started_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + const custom_lease = GraphIndex.GraphMetricBuildLease{ + .job_id = 99, + .target_generation = graph.edge_generation, + .started_at_ms = custom_started_at_ms, + .lease_expires_at_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms) + graph_metric_local_build_lease_ms, + .phase = .publishing, + .iteration = 17, + .worker_id = "worker-a", + }; + const custom_lease_key = try graph.graphMetricBuildLeaseKeyAlloc("degree"); + defer alloc.free(custom_lease_key); + const custom_lease_encoded = try alloc.alloc(u8, GraphIndex.graphMetricBuildLeaseEncodedLen(custom_lease)); + defer alloc.free(custom_lease_encoded); + GraphIndex.encodeGraphMetricBuildLease(custom_lease, custom_lease_encoded); + { + var custom_lease_batch = try graph.beginWriteReverseBatch(); + errdefer custom_lease_batch.abort(); + try custom_lease_batch.put(custom_lease_key, custom_lease_encoded); + try custom_lease_batch.commit(); + } + + var custom_building = try graph.graphMetricStatus("degree"); + defer custom_building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, custom_building.state); + try std.testing.expectEqual(@as(u64, 99), custom_building.build_job_id); + try std.testing.expectEqual(custom_started_at_ms, custom_building.build_started_at_ms); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publishing, custom_building.phase); + try std.testing.expectEqual(@as(u32, 17), custom_building.build_iteration); + try std.testing.expectEqualStrings("worker-a", custom_building.build_worker_id); + try std.testing.expectApproxEqAbs(@as(f64, 0.95), custom_building.progress, 0.0001); + + try graph.releaseGraphMetricBuildLease("degree"); + var published = try graph.runGraphMetric("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expect(!published.build_queued); + try std.testing.expectEqual(@as(u64, 0), published.building_generation); + try std.testing.expectEqual(@as(u64, 0), published.build_job_id); + try std.testing.expectEqual(@as(u64, 0), published.build_started_at_ms); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const completed_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expectEqual(graph.edge_generation, completed_job.target_generation); + try std.testing.expectEqual(try graph.metricPublishedGeneration(&job_txn, "degree"), completed_job.score_generation); + try std.testing.expectEqual(@as(u64, 0), completed_job.lease_expires_at_ms); + try std.testing.expect(completed_job.updated_at_ms >= completed_job.started_at_ms); + } +} + +test "graph metric planned progress is monotonic across pages phases and iterations" { + const pagerank = GraphMetricConfig{ .name = "pagerank", .kind = .pagerank, .max_iterations = 10 }; + const first_contribution_start = GraphIndex.graphMetricActiveBuildProgress(pagerank, .iterate_contributions, 0, 0); + const first_contribution_half = GraphIndex.graphMetricActiveBuildProgress(pagerank, .iterate_contributions, 0, 0.5); + const first_convergence_end = GraphIndex.graphMetricActiveBuildProgress(pagerank, .check_convergence, 0, 1); + const second_contribution_start = GraphIndex.graphMetricActiveBuildProgress(pagerank, .reduce_ranks, 1, 0); + try std.testing.expect(first_contribution_start < first_contribution_half); + try std.testing.expect(first_contribution_half < first_convergence_end); + try std.testing.expectApproxEqAbs(first_convergence_end, second_contribution_start, 0.0000001); + + const hits = GraphMetricConfig{ .name = "hits", .kind = .hits_authority, .max_iterations = 10 }; + const first_hits_end = GraphIndex.graphMetricActiveBuildProgress(hits, .check_convergence, 0, 1); + const second_hits_start = GraphIndex.graphMetricActiveBuildProgress(hits, .reduce_ranks, 1, 0); + try std.testing.expectApproxEqAbs(first_hits_end, second_hits_start, 0.0000001); + + const degree = GraphMetricConfig{ .name = "degree", .kind = .degree }; + try std.testing.expect( + GraphIndex.graphMetricActiveBuildProgress(degree, .scan_edges_and_out_degree, 0, 1) <= + GraphIndex.graphMetricActiveBuildProgress(degree, .reduce_ranks, 0, 0), + ); +} + +test "graph metric build lease survives reopen and expired lease can be reclaimed" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-lease-reopen"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-lease-reopen"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + const target_generation = graph.edge_generation; + try graph.acquireGraphMetricBuildLease("degree", target_generation); + try graph.updateGraphMetricBuildLeaseProgressWithCursor("degree", .computing, 5, "edge-page:0013", 13, 20); + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + var active = try graph.graphMetricStatus("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, active.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.computing, active.phase); + try std.testing.expect(active.build_job_id != 0); + const active_job_id = active.build_job_id; + try std.testing.expect(active.build_started_at_ms != 0); + const active_started_at_ms = active.build_started_at_ms; + try std.testing.expectEqual(target_generation, active.building_generation); + try std.testing.expectEqual(@as(u32, 5), active.build_iteration); + try std.testing.expectEqualStrings("edge-page:0013", active.build_cursor); + try std.testing.expectEqual(@as(u64, 13), active.build_completed_units); + try std.testing.expectEqual(@as(u64, 20), active.build_total_units); + try std.testing.expectEqualStrings(graph_metric_local_build_worker_id, active.build_worker_id); + try std.testing.expectApproxEqAbs(@as(f64, 0.1), active.progress, 0.0001); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active.build_job_id, active_job.job_id); + try std.testing.expectEqual(active.build_started_at_ms, active_job.started_at_ms); + try std.testing.expectEqual(target_generation, active_job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.computing, active_job.phase); + try std.testing.expectEqual(@as(u32, 5), active_job.iteration); + try std.testing.expectEqualStrings("edge-page:0013", active_job.cursor); + try std.testing.expectEqual(@as(u64, 13), active_job.completed_units); + try std.testing.expectEqual(@as(u64, 20), active_job.total_units); + try std.testing.expectEqualStrings(graph_metric_local_build_worker_id, active_job.worker_id); + } + active.deinit(alloc); + + const now_ms = @divTrunc(platform_time.realtimeNs(), std.time.ns_per_ms); + const expired_lease = GraphIndex.GraphMetricBuildLease{ + .target_generation = target_generation, + .started_at_ms = now_ms - 1, + .lease_expires_at_ms = now_ms - 1, + .phase = .computing, + .iteration = 9, + .worker_id = "expired-worker", + }; + const expired_lease_key = try graph.graphMetricBuildLeaseKeyAlloc("degree"); + defer alloc.free(expired_lease_key); + const expired_lease_encoded = try alloc.alloc(u8, GraphIndex.graphMetricBuildLeaseEncodedLen(expired_lease)); + defer alloc.free(expired_lease_encoded); + GraphIndex.encodeGraphMetricBuildLease(expired_lease, expired_lease_encoded); + var expired_batch = try graph.beginWriteReverseBatch(); + errdefer expired_batch.abort(); + try expired_batch.put(expired_lease_key, expired_lease_encoded); + try expired_batch.commit(); + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + var expired = try graph.graphMetricStatus("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, expired.state); + try std.testing.expect(expired.build_queued); + try std.testing.expectEqual(@as(u64, 0), expired.building_generation); + try std.testing.expectEqual(@as(u64, 0), expired.build_job_id); + try std.testing.expectEqual(@as(u64, 0), expired.build_started_at_ms); + try std.testing.expectEqual(@as(u32, 0), expired.build_iteration); + try std.testing.expectEqualStrings("", expired.build_worker_id); + expired.deinit(alloc); + + var rebuilt = try graph.runGraphMetric("degree"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expect(!rebuilt.build_queued); + try std.testing.expectEqual(@as(u64, 0), rebuilt.building_generation); + try std.testing.expectEqual(@as(u64, 0), rebuilt.build_job_id); + try std.testing.expectEqual(@as(u64, 0), rebuilt.build_started_at_ms); + try std.testing.expect(active_job_id != 0); + try std.testing.expect(active_started_at_ms != 0); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + const completed_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expectEqual(target_generation, completed_job.target_generation); + try std.testing.expectEqual(@as(u64, 0), completed_job.lease_expires_at_ms); + } +} + +test "graph metric edge filter equality and fingerprint treat types as set" { + const left_types = [_][]const u8{ "cites", "mentions" }; + const right_types = [_][]const u8{ "mentions", "cites" }; + const left_filter = GraphMetricEdgeFilter{ .mode = .types, .types = &left_types }; + const right_filter = GraphMetricEdgeFilter{ .mode = .types, .types = &right_types }; + + try std.testing.expect(left_filter.equivalent(right_filter)); + try std.testing.expect(right_filter.equivalent(left_filter)); + + const left = GraphMetricConfig{ + .name = "pagerank", + .kind = .pagerank, + .edge_filter = left_filter, + }; + const right = GraphMetricConfig{ + .name = "pagerank", + .kind = .pagerank, + .edge_filter = right_filter, + }; + const fingerprint = GraphIndex.graphMetricConfigFingerprint(left); + try std.testing.expectEqual(fingerprint, GraphIndex.graphMetricConfigFingerprint(right)); + try std.testing.expect(fingerprint > 0); + try std.testing.expect(fingerprint <= std.math.maxInt(i64)); +} + +test "graph metric build manifest is durable and idempotent across reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-manifest"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-manifest"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + const target_generation = graph.edge_generation; + try graph.acquireGraphMetricBuildLease("pagerank", target_generation); + + var active_job = GraphIndex.GraphMetricBuildJob{}; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + active_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(GraphIndex.graph_metric_build_execution_schema_version, manifest.execution_schema_version); + try std.testing.expectEqual(active_job.job_id, manifest.job_id); + try std.testing.expectEqual(target_generation, manifest.target_generation); + try std.testing.expectEqual(target_generation, manifest.score_generation); + try std.testing.expectEqual(GraphIndex.graphMetricConfigFingerprint(metrics[0]), manifest.config_fingerprint); + try std.testing.expectEqual(@as(u64, 2), manifest.edge_count); + try std.testing.expectEqual(@as(u64, 3), manifest.node_count); + try std.testing.expectEqual(GraphIndex.graph_metric_iterative_build_phases.len, manifest.phase_count); + try std.testing.expectEqual(GraphIndex.graph_metric_iterative_build_phases.len + 2, manifest.page_count); + + const scan_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, scan_page.state); + try std.testing.expectEqual(@as(u64, 2), scan_page.total_units); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, scan_page.range_kind); + try std.testing.expect(scan_page.range_lower.len > 0); + try std.testing.expectEqualStrings("", scan_page.range_upper); + const initialize_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 3), initialize_page.total_units); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, initialize_page.range_kind); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(target_generation, manifest.target_generation); + const scan_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, scan_page.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, scan_page.range_kind); + try std.testing.expect(scan_page.range_lower.len > 0); + } + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildPageInBatch(&batch, "pagerank", .{ + .job_id = active_job.job_id, + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 1, + .state = .complete, + .completed_units = 2, + .total_units = 2, + .output_fingerprint = 1234, + }); + try batch.commit(); + } + + try graph.ensureGraphMetricBuildManifestForJob("pagerank", active_job); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(GraphIndex.graph_metric_iterative_build_phases.len + 2, manifest.page_count); + const scan_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, scan_page.state); + try std.testing.expectEqual(@as(u64, 2), scan_page.completed_units); + try std.testing.expectEqual(@as(u64, 1234), scan_page.output_fingerprint); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var legacy_manifest = try graph.metricBuildManifest(&batch, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + legacy_manifest.execution_schema_version = 1; + try graph.putGraphMetricBuildManifestInBatch(&batch, "pagerank", legacy_manifest); + try batch.commit(); + } + try std.testing.expectError( + error.InvalidGraphMetricBuildManifest, + graph.ensureGraphMetricBuildManifestForJob("pagerank", active_job), + ); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const lease_key = try graph.graphMetricBuildLeaseKeyAlloc("pagerank"); + defer alloc.free(lease_key); + try batch.delete(lease_key); + try batch.commit(); + } + try graph.acquireGraphMetricBuildLease("pagerank", target_generation); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const replacement_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expect(replacement_job.job_id != active_job.job_id); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + const failure = try graph.metricFailureDetail(&txn, "pagerank") orelse return error.TestExpectedGraphMetricFailureDetail; + defer failure.deinit(alloc); + try std.testing.expectEqualStrings("GraphMetricBuildSupersededByLeaseTakeover", failure.last_error); + } +} + +test "graph metric build pagerank manifest partitions iterative phase pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-manifest-partitions"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-manifest-partitions"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var target_buf: [64]u8 = undefined; + const target = try std.fmt.bufPrint(&target_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge("hub", target, "cites", 1.0, 0, 0, ""); + } + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + // Eight data-partition pages plus two dependency-summary pages. + try std.testing.expectEqual(GraphIndex.graph_metric_iterative_build_phases.len + 10, manifest.page_count); + + const first_scan = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, first_scan.range_kind); + try std.testing.expect(first_scan.range_lower.len > 0); + try std.testing.expect(first_scan.range_upper.len > 0); + try std.testing.expectEqual(@as(u64, 33), first_scan.total_units); + const second_scan = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqualStrings(first_scan.range_upper, second_scan.range_lower); + try std.testing.expectEqualStrings("", second_scan.range_upper); + try std.testing.expectEqual(@as(u64, 32), second_scan.total_units); + + const first_init = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, first_init.range_kind); + try std.testing.expect(first_init.range_lower.len > 0); + try std.testing.expect(first_init.range_upper.len > 0); + try std.testing.expectEqual(@as(u64, 33), first_init.total_units); + const second_init = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .initialize_ranks, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqualStrings(first_init.range_upper, second_init.range_lower); + try std.testing.expectEqualStrings("", second_init.range_upper); + try std.testing.expectEqual(@as(u64, 33), second_init.total_units); + + const contribution = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, contribution.range_kind); + const reduce = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, reduce.range_kind); + const convergence = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, convergence.range_kind); + + const out_degree_cleanup = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, 0) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, out_degree_cleanup.range_kind); + try std.testing.expect(out_degree_cleanup.output_prefix.len > 0); + const node_cleanup = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, node_cleanup.range_kind); + try std.testing.expect(node_cleanup.output_prefix.len > 0); + const final_cleanup = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, final_cleanup.range_kind); + try std.testing.expect(final_cleanup.output_prefix.len > 0); + try std.testing.expect(!std.mem.eql(u8, out_degree_cleanup.output_prefix, node_cleanup.output_prefix)); + try std.testing.expect(!std.mem.eql(u8, node_cleanup.output_prefix, final_cleanup.output_prefix)); + try std.testing.expect((try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, 3)) == null); +} + +test "graph metric partition spans remain balanced at production cardinality" { + const total_units: usize = 10_000_003; + var cursor: usize = 0; + var min_len: usize = std.math.maxInt(usize); + var max_len: usize = 0; + for (0..graph_metric_build_max_partition_pages) |page_index| { + const span = GraphIndex.graphMetricPartitionSpan( + total_units, + graph_metric_build_max_partition_pages, + page_index, + ); + try std.testing.expectEqual(cursor, span.start); + cursor += span.len; + min_len = @min(min_len, span.len); + max_len = @max(max_len, span.len); + } + try std.testing.expectEqual(total_units, cursor); + try std.testing.expect(max_len - min_len <= 1); +} + +test "graph metric floating page aggregates are deterministic across adoption order" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-deterministic-pages"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-deterministic-pages"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); + defer graph.close(); + + const packed_entries = try graph.encodePackedF64EntriesAlloc(&.{ + .{ .node = "doc-a", .value = 1.25 }, + .{ .node = "doc-b", .value = -2.5 }, + }); + defer alloc.free(packed_entries); + const unpacked = try graph.decodePackedF64EntriesAlloc(packed_entries); + defer alloc.free(unpacked); + try std.testing.expectEqual(@as(usize, 2), unpacked.len); + try std.testing.expectEqualStrings("doc-a", unpacked[0].node); + try std.testing.expectEqual(@as(f64, 1.25), unpacked[0].value); + const corrupted = try alloc.dupe(u8, packed_entries); + defer alloc.free(corrupted); + corrupted[corrupted.len - 1] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.decodePackedF64EntriesAlloc(corrupted)); + + var forward: ?[]u8 = null; + defer if (forward) |value| alloc.free(value); + for ([_]struct { page_id: u64, value: f64 }{ + .{ .page_id = 2, .value = 1.0e16 }, + .{ .page_id = 0, .value = 1.0 }, + .{ .page_id = 1, .value = -1.0e16 }, + }) |entry| { + const next = try graph.replaceDeterministicF64PageValueAlloc(forward, null, entry.page_id, entry.value); + if (forward) |value| alloc.free(value); + forward = next; + } + + var reverse: ?[]u8 = null; + defer if (reverse) |value| alloc.free(value); + for ([_]struct { page_id: u64, value: f64 }{ + .{ .page_id = 1, .value = -1.0e16 }, + .{ .page_id = 0, .value = 1.0 }, + .{ .page_id = 2, .value = 1.0e16 }, + }) |entry| { + const next = try graph.replaceDeterministicF64PageValueAlloc(reverse, null, entry.page_id, entry.value); + if (reverse) |value| alloc.free(value); + reverse = next; + } + + try std.testing.expectEqualSlices(u8, forward.?, reverse.?); + const forward_sum = try GraphIndex.decodeDeterministicF64Sum(forward.?); + const reverse_sum = try GraphIndex.decodeDeterministicF64Sum(reverse.?); + try std.testing.expectEqual(@as(u64, @bitCast(forward_sum)), @as(u64, @bitCast(reverse_sum))); + try std.testing.expectEqual(@as(f64, 1.0), forward_sum); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildManifestInBatch(&batch, "hits-a", .{ .job_id = 1, .node_count = 1 }); + try graph.putGraphMetricBuildManifestInBatch(&batch, "hits-b", .{ .job_id = 2, .node_count = 1 }); + for ([_]struct { page_id: u64, value: f64 }{ + .{ .page_id = 2, .value = 1.0e16 }, + .{ .page_id = 0, .value = 1.0 }, + .{ .page_id = 1, .value = -1.0e16 }, + }) |entry| { + const key = try graph.graphMetricBuildHitsHubRawKeyAlloc("hits-a", 1, 0, "hub", entry.page_id); + defer alloc.free(key); + try GraphIndex.putF64(&batch, key, entry.value); + } + for ([_]struct { page_id: u64, value: f64 }{ + .{ .page_id = 1, .value = -1.0e16 }, + .{ .page_id = 0, .value = 1.0 }, + .{ .page_id = 2, .value = 1.0e16 }, + }) |entry| { + const key = try graph.graphMetricBuildHitsHubRawKeyAlloc("hits-b", 2, 0, "hub", entry.page_id); + defer alloc.free(key); + try GraphIndex.putF64(&batch, key, entry.value); + } + try batch.commit(); + } + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const hits_a = try graph.aggregateHitsHubRawForNode(&txn, "hits-a", 1, 0, "hub"); + const hits_b = try graph.aggregateHitsHubRawForNode(&txn, "hits-b", 2, 0, "hub"); + try std.testing.expectEqual(@as(u64, @bitCast(hits_a)), @as(u64, @bitCast(hits_b))); + try std.testing.expectEqual(@as(f64, 1.0), hits_a); +} + +test "graph metric column snapshots preserve order across chunks and reject stale reads before scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-column-snapshot"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-column-snapshot"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ .name = "degree", .kind = .degree, .refresh = .manual }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const small_nodes = [_][]const u8{ "doc-b", "doc-a", "doc-b" }; + try std.testing.expectError( + error.MetricNotReady, + graph.graphMetricColumnsSnapshotAlloc(&.{"degree"}, &small_nodes, &.{.{ .require_published = true }}), + ); + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runGraphMetric("degree"); + defer published.deinit(alloc); + + const node_count = 4100; + const nodes = try alloc.alloc([]const u8, node_count); + defer alloc.free(nodes); + const node_buffers = try alloc.alloc([32]u8, node_count); + defer alloc.free(node_buffers); + for (nodes, 0..) |*node, i| node.* = if (i < 3) small_nodes[i] else try std.fmt.bufPrint(&node_buffers[i], "missing-{d:0>6}", .{i}); + var snapshot = try graph.graphMetricColumnsSnapshotAlloc( + &.{ "degree", "degree" }, + nodes, + &.{ .{ .require_published = true, .require_fresh = true }, .{ .require_published = true } }, + ); + defer snapshot.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), snapshot.score_columns.len); + for (snapshot.score_columns) |column| for (column, 0..) |score, i| { + try std.testing.expectEqual(@as(?f64, if (i < 3) 1.0 else null), score); + }; + var single = try graph.graphMetricScoreSnapshotAlloc("degree", nodes); + defer single.deinit(alloc); + try std.testing.expectEqualSlices(?f64, snapshot.score_columns[0], single.scores); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const PointOnlyTxn = struct { + inner: *@TypeOf(txn), + reads: usize = 0, + // Deliberately no cursor API: query metadata must be point-only. + pub fn get(self: *@This(), key: []const u8) ![]const u8 { + self.reads += 1; + return self.inner.get(key); + } + }; + var point = PointOnlyTxn{ .inner = &txn }; + var status = try graph.graphMetricSnapshotStatusInTxn("degree", &point, .query); + defer status.deinit(alloc); + try std.testing.expectEqual(published.published_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 0), status.recent_events.len); + try std.testing.expectEqual(@as(usize, 0), status.recent_failures.len); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + try std.testing.expect(point.reads <= 16); + } + + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + try std.testing.expectError( + error.MetricStale, + graph.graphMetricColumnsSnapshotAlloc(&.{"degree"}, nodes, &.{.{ .require_fresh = true }}), + ); + // Poison a published value to prove freshness rejection precedes score I/O. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try graph.graphMetricScoreKeyAlloc("degree", published.published_generation, "doc-a"); + defer alloc.free(key); + try batch.put(key, "invalid-score"); + try batch.commit(); + } + try std.testing.expectError(error.MetricStale, graph.graphMetricScoreSnapshotWithPolicyAlloc("degree", nodes, .{ .require_fresh = true })); +} + +test "graph metric partition census bounds steps resumes after reopen and fences generation changes" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-partition-census"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-partition-census"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank" }}; + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + try graph.addEdge("c", "a", "cites", 1, 0, 0, ""); + try std.testing.expect(!try graph.prepareGraphMetricPartitionStep(2)); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var census = (try partition_census.State.decodeAlloc(alloc, try txn.get(graph_metric_partition_census_key))).?; + defer census.deinit(alloc); + try std.testing.expectEqual(@as(u64, 2), census.edges_seen); + } + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try std.testing.expect(!try graph.prepareGraphMetricPartitionStep(2)); + try graph.addEdge("d", "a", "cites", 1, 0, 0, ""); + try std.testing.expect(!try graph.prepareGraphMetricPartitionStep(2)); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var census = (try partition_census.State.decodeAlloc(alloc, try batch.get(graph_metric_partition_census_key))).?; + defer census.deinit(alloc); + try std.testing.expectEqual(graph.edge_generation, census.generation); + try std.testing.expectEqual(@as(u64, 2), census.edges_seen); + // This namespace must be skipped by a seek, not counted as census work. + for (0..1000) |i| { + var key: [64]u8 = undefined; + try batch.put(try std.fmt.bufPrint(&key, "meta:metric_fixture:{d:0>6}", .{i}), "score"); + } + try batch.commit(); + } + var steps: usize = 0; + while (!try graph.prepareGraphMetricPartitionStep(2)) { + steps += 1; + if (steps > 6) return error.TestUnexpectedResult; + } + var plan = try graph.cachedGraphMetricPartitionPlan(); + defer plan.deinit(alloc); + try std.testing.expectEqual(graph.edge_generation, plan.edge_generation); + try std.testing.expectEqual(graph.edge_count, plan.edge_count); + try std.testing.expectEqual(graph.node_count, plan.node_count); + try std.testing.expect(try graph.prepareGraphMetricPartitionStep(1)); +} + +fn installGraphMetricPlanningNodeRefsForTest(graph: *GraphIndex, count: usize) !void { + if (count < 2) return error.TestUnexpectedResult; + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (2..count) |i| { + var node_buf: [64]u8 = undefined; + const node = try std.fmt.bufPrint(&node_buf, "doc-{d:0>4}", .{i}); + const key = try GraphIndex.graphNodeRefKeyAlloc(graph.alloc, node); + defer graph.alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + graph.node_count = @intCast(count); + try graph.persistGraphCounters(&batch); + try batch.commit(); +} + +test "graph metric large-build summary pages resume and gate dependent partitions" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-summary-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-summary-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics, .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + defer graph.close(); + + try graph.addEdge("doc-0000", "doc-0001", "cites", 1.0, 0, 0, ""); + try installGraphMetricPlanningNodeRefsForTest( + &graph, + graph_metric_build_checkpoint_reduce_units + 1, + ); + var building = try graph.ensureGraphMetricPlannedBuild("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer building.deinit(alloc); + + var job_txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..65) |i| { + const node = try std.fmt.allocPrint(alloc, "doc-{d:0>4}", .{i}); + defer alloc.free(node); + const key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", job.job_id, node, 1); + defer alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + try batch.commit(); + } + + const first_claim = try graph.claimNextGraphMetricBuildPageAt("pagerank", job.job_id, .initialize_ranks, 0, "summary-worker-a", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(graph_metric_build_summary_leaf_base, first_claim.page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.summary, first_claim.range_kind); + try std.testing.expectEqual(@as(usize, 32), try graph.executeGraphMetricReduceSummaryBuildPageWithLimit("pagerank", metrics[0], job, first_claim, 32)); + const second_claim = (try graph.claimNextGraphMetricBuildPageAt("pagerank", job.job_id, .initialize_ranks, 0, "summary-worker-b", 1001)).?; + try std.testing.expect(second_claim.page_id != first_claim.page_id); + _ = try graph.executeGraphMetricReduceSummaryBuildPage("pagerank", metrics[0], job, second_claim); + + const renewed = try graph.claimNextGraphMetricBuildPageAt("pagerank", job.job_id, .initialize_ranks, 0, "summary-worker-a", 1001) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeGraphMetricReduceSummaryBuildPage("pagerank", metrics[0], job, renewed); + for (0..graph_metric_build_max_partition_pages + 1) |_| { + const claimed = (try graph.claimNextGraphMetricBuildPageAt("pagerank", job.job_id, .initialize_ranks, 0, "summary-worker-a", 1002)).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.summary, claimed.range_kind); + _ = try graph.executeGraphMetricReduceSummaryBuildPage("pagerank", metrics[0], job, claimed); + if (claimed.page_id == 0) break; + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPage(&txn, "pagerank", job.job_id, .initialize_ranks, 0, 0) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, summary.state); + try std.testing.expectApproxEqAbs(@as(f64, 65.0), summary.rank_sum, 0.0); + } + try std.testing.expectEqual(@as(usize, 65), try graph.graphMetricBuildNodeCount("pagerank", job.job_id)); +} + +test "graph metric ordinal progress owns cursor before storage iteration advances" { + const alloc = std.testing.allocator; + var graph: GraphIndex = undefined; + graph.alloc = alloc; + const job = GraphIndex.GraphMetricBuildJob{ .job_id = 1, .phase = .reduce_ranks }; + const page = GraphIndex.GraphMetricBuildPage{ .job_id = 1, .phase = .reduce_ranks, .state = .leased, .cursor = "durable-checkpoint", .completed_units = 2, .total_units = 5 }; + const encoded = try alloc.alloc(u8, GraphIndex.graphMetricBuildPageEncodedLen(page)); + defer alloc.free(encoded); + GraphIndex.encodeGraphMetricBuildPage(page, encoded); + const prefix = try graph.graphMetricBuildPagePrefixAlloc("rank", 1, .reduce_ranks, 0); + defer alloc.free(prefix); + const BorrowedTxn = struct { + key: []const u8, + value: []u8, + const Entry = struct { key: []const u8, value: []const u8 }; + const Cursor = struct { + txn: *ThisOuter, + fn seekAtOrAfter(self: *@This(), _: []const u8) !?Entry { + return .{ .key = self.txn.key, .value = self.txn.value }; + } + fn next(self: *@This()) !?Entry { + @memset(self.txn.value, 0xdd); + return null; + } + fn close(_: *@This()) void {} + }; + const ThisOuter = @This(); + fn openCursor(self: *@This()) !Cursor { + return .{ .txn = self }; + } + }; + var txn = BorrowedTxn{ .key = prefix, .value = encoded }; + var progress = try graph.graphMetricActiveBuildProgressAggregate(&txn, "rank", job); + defer progress.deinit(alloc); + try std.testing.expectEqualStrings("durable-checkpoint", progress.cursor); + try std.testing.expectEqual(@as(u64, 2), progress.completed_units); + try std.testing.expectEqual(@as(u64, 5), progress.total_units); +} + +test "graph metric ordinal shuffle selects winning attempts across changed checkpoint boundaries" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-ordinal-shuffle"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-ordinal-shuffle"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + defer graph.close(); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const nodes = try temp.alloc([]const u8, 300); + const values = try temp.alloc(ordinal_blocks.Value, nodes.len); + const page = GraphIndex.GraphMetricBuildPage{ .job_id = 1, .phase = .iterate_contributions, .iteration = 0, .page_id = 3, .attempt = 1, .worker_id = "first", .state = .leased, .lease_expires_at_ms = 100, .total_units = 300 }; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildManifestInBatch(&batch, "rank", .{ .job_id = 1, .node_count = 4097 }); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", page); + for (nodes, values, 0..) |*node, *value, i| { + node.* = try std.fmt.allocPrint(temp, "node-{d:0>4}", .{i}); + value.* = .{ .ordinal = (@as(u64, 1) << 32) + i, .value = 9 }; + const key = try GraphIndex.graphMetricNodeSlotKey(temp, "rank", 1, node.*); + try GraphIndex.putU64(&batch, key, value.ordinal); + } + try graph.writeOrdinalContributionsInBatch(&batch, "rank", 1, page, 0, values); + try batch.commit(); + } + // The first attempt crashed after a checkpoint but before completion. Its + // replacement uses different checkpoint boundaries and different values. + const replacement = (try graph.claimGraphMetricBuildPageAt("rank", 1, page.phase, 0, 3, "second", 101)).?; + try std.testing.expectEqual(@as(u64, 2), replacement.attempt); + for (values) |*value| value.value = 2; + for ([_]struct { start: usize, end: usize }{ .{ .start = 0, .end = 200 }, .{ .start = 200, .end = 300 } }) |span| { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.writeOrdinalContributionsInBatch(&batch, "rank", 1, replacement, span.start, values[span.start..span.end]); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writeOrdinalContributionsInBatch(&batch, "rank", 1, page, span.start, values[span.start..span.end])); + _ = try graph.updateGraphMetricBuildPageProgressInBatch(&batch, "rank", 1, page.phase, 0, 3, replacement.worker_id, replacement.attempt, "", span.end, 300); + try batch.commit(); + } + _ = try graph.completeGraphMetricBuildPageForAttempt("rank", 1, page.phase, 0, 3, "second", replacement.attempt, 300, 42); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const actual = try graph.ordinalContributionsForNodesAlloc(&txn, "rank", 1, .iterate_contributions, 0, nodes); + defer alloc.free(actual); + for (actual) |value| try std.testing.expectEqual(@as(f64, 2), value); +} + +test "graph metric ordinal iterations reuse immutable adjacency without numeric shuffle" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-adjacency-reuse"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-adjacency-reuse"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const cfg = GraphMetricConfig{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 3, .tolerance = 1e-20 }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &.{cfg} }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + try graph.acquireGraphMetricBuildLease(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + defer graph.releaseGraphMetricBuildLease(cfg.name) catch {}; + const job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk (try graph.metricBuildJob(&txn, cfg.name)).?; + }; + try drainGraphMetricBuildToPublishForTest(&graph, cfg.name, cfg, "worker", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + { + const key = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.topologyKey(&txn, cfg.name, job.job_id, try graph.graphMetricMembershipKey(cfg.name, job.job_id, 0, 0)); + }; + defer alloc.free(key); + const saved = blk: { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const raw = try alloc.dupe(u8, try batch.get(key)); + errdefer alloc.free(raw); + try batch.delete(key); + try batch.commit(); + break :blk raw; + }; + defer alloc.free(saved); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| alloc.free(node); + nodes.deinit(alloc); + } + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.collectGraphMetricInitializedNodesInRange(&txn, cfg.name, job.job_id, "", "", "", null, &nodes)); + } + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(key, saved); + try batch.commit(); + } + { + const missing_key = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.topologyKey(&txn, cfg.name, job.job_id, try GraphIndex.graphMetricNodeSlotKey(alloc, cfg.name, job.job_id, "b")); + }; + defer alloc.free(missing_key); + var saved: [8]u8 = undefined; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + @memcpy(&saved, try batch.get(missing_key)); + try batch.delete(missing_key); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| alloc.free(node); + nodes.deinit(alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(alloc); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.collectGraphMetricOrdinalNodesInRange(&txn, cfg.name, job.job_id, "", "", "", null, &nodes, &slots)); + } + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(missing_key, &saved); + try batch.commit(); + } + const namespace = try graph.graphMetricBuildJobNamespacePrefixAlloc(cfg.name, job.job_id); + defer alloc.free(namespace); + const prefix = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.topologyKey(&txn, cfg.name, job.job_id, try std.fmt.allocPrint(alloc, "{s}adjacency-packed/", .{namespace})); + }; + defer alloc.free(prefix); + var first_digest: ?u64 = null; + for (0..3) |iteration| { + if (iteration == 0) try drainGraphMetricBuildToPublishForTest(&graph, cfg.name, cfg, "worker", &.{.iterate_contributions}); + try drainGraphMetricBuildToPublishForTest(&graph, cfg.name, cfg, "worker", &.{.reduce_ranks}); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const shuffle = try graph.graphMetricBuildPageRankContributionPrefixAlloc(cfg.name, job.job_id, @intCast(iteration)); + defer alloc.free(shuffle); + try std.testing.expectEqual(@as(usize, 0), try GraphIndex.countKeysWithPrefix(&txn, shuffle)); + const staged = try std.fmt.allocPrint(alloc, "{s}adjacency/", .{namespace}); + defer alloc.free(staged); + try std.testing.expectEqual(@as(usize, 0), try GraphIndex.countKeysWithPrefix(&txn, staged)); + var cursor = try txn.openCursor(); + defer cursor.close(); + var next = try cursor.seekAtOrAfter(prefix); + var hash = std.hash.Wyhash.init(0); + var count: usize = 0; + while (next) |entry| : (next = try cursor.next()) { + if (!std.mem.startsWith(u8, entry.key, prefix)) break; + hash.update(entry.key); + hash.update(entry.value); + count += 1; + } + try std.testing.expect(count != 0); + if (first_digest) |digest| try std.testing.expectEqual(digest, hash.final()) else first_digest = hash.final(); + } + try drainGraphMetricBuildToPublishForTest(&graph, cfg.name, cfg, "worker", &.{.check_convergence}); + } +} + +test "graph metric ordinal fold bounds hot shards and resumes across reopen and takeover" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-ordinal-fold"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-ordinal-fold"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const cfg = GraphMetricConfig{ .name = "rank", .kind = .eigenvector, .refresh = .manual }; + const job = GraphIndex.GraphMetricBuildJob{ .job_id = 1, .phase = .reduce_ranks }; + const page = GraphIndex.GraphMetricBuildPage{ .job_id = 1, .phase = .reduce_ranks, .page_id = graph_metric_build_summary_leaf_base, .state = .leased, .range_kind = .summary, .worker_id = "original", .attempt = 1, .lease_expires_at_ms = 100, .total_units = 2 }; + const options = GraphIndexOptions{ .metric_configs = &.{cfg}, .reverse_lsm_options = .{ .flush_threshold = 8192 } }; + const nodes = [_][]const u8{ "a", "z" }; + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildManifestInBatch(&batch, "rank", .{ .job_id = 1, .node_count = 2 }); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .initialize_ranks, .page_id = graph_metric_build_summary_leaf_base, .range_kind = .summary, .state = .complete, .completed_units = 2, .total_units = 2 }); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .initialize_ranks, .page_id = 0, .range_kind = .summary, .state = .complete, .completed_units = 2, .total_units = 2 }); + try graph.sealGraphMetricActivePlan(&batch, "rank", cfg, job); + try graph.writeGraphMetricMembership(&batch, "rank", 1, 0, 0, &nodes); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", page); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .iterate_contributions, .page_id = 3, .state = .complete, .attempt = 2 }); + for (nodes, 0..) |node, i| { + const slot = (@as(u64, 1) << 32) + i; + const key = try GraphIndex.graphMetricNodeSlotKey(temp, "rank", 1, node); + try GraphIndex.putU64(&batch, key, slot); + const scanned_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("rank", 1, node, 0); + defer alloc.free(scanned_key); + try GraphIndex.putU64(&batch, scanned_key, 1); + var vector: vector_chunk.Chunk = @splat(0); + try vector_chunk.put(&vector, 0, 2); + try vector_chunk.put(&vector, 1, 3); + try batch.put(try GraphIndex.graphMetricVectorChunkKey(temp, "rank", 1, "rank", 0, slot / 256), &vector); + const prefix = try graph.ordinalAdjacencyPrefixAlloc("rank", 1, .iterate_contributions, slot / 256); + defer alloc.free(prefix); + for (0..6) |shard| { + const canonical = try std.fmt.allocPrint(temp, "{s}{d:0>20}:{d:0>20}:{d:0>20}", .{ prefix, @as(u64, 3), @as(u64, if (shard == 0) 1 else 2), i * 6 + shard }); + var edges: [256]ordinal_blocks.Edge = @splat(.{ .source = if (shard == 0) 999 else slot, .target = slot }); + try batch.put(canonical, try ordinal_blocks.encodeTopology(temp, .{ .edges = &edges, .cursor = @constCast(""), .scanned = edges.len, .complete = true })); + } + } + try batch.commit(); + } + // Packing has its own restart/takeover regression below. Here the + // durable checkpoint must hold partial numeric folds of dense tiles. + const chunk = (@as(u64, 1) << 32) / 256; + try std.testing.expectEqual(@as(usize, 12), try graph.compactOrdinalAdjacencyChunk("rank", job, page, .iterate_contributions, chunk, 512)); + try std.testing.expectEqual(@as(usize, 2), try graph.executeOrdinalReduceSummary("rank", cfg, job, page, 256, 2)); + } + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + // A checkpoint also pins its node group when the requested limit changes. + try std.testing.expectEqual(@as(usize, 2), try graph.executeOrdinalReduceSummary("rank", cfg, job, page, 1, 2)); + const replacement = (try graph.claimGraphMetricBuildPageAt("rank", 1, .reduce_ranks, 0, page.page_id, "replacement", 101)).?; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executeOrdinalReduceSummary("rank", cfg, job, page, 256, 2)); + var done = false; + var visited: usize = 0; + for (0..12) |_| { + const count = try graph.executeOrdinalReduceSummary("rank", cfg, job, replacement, 256, 2); + try std.testing.expect(count <= 2); + visited += count; + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const current = (try graph.metricBuildPage(&txn, "rank", 1, .reduce_ranks, 0, page.page_id)).?; + if (current.state == .complete) { + try std.testing.expectEqual(@as(f64, 325 * 256 * 256), current.rank_sum); + try std.testing.expectEqual(@as(u64, 2), current.completed_units); + done = true; + break; + } + try std.testing.expectEqual(@as(u64, 0), current.completed_units); + } + try std.testing.expect(done); + // Takeover recomputes ten dense tiles without mixing the prior sums. + try std.testing.expectEqual(@as(usize, 10), visited); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const raw = try graph.pageRankContributionsForNodesAlloc(&txn, "rank", 1, 0, &nodes); + defer alloc.free(raw); + try std.testing.expectEqualSlices(f64, &.{ 10 * 256, 15 * 256 }, raw); +} + +test "graph metric shared topology ordinal cursors bind sealed coverage without reading node dictionaries" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-ordinal-cursor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-ordinal-cursor"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }}; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.benchmarkPrepareOrdinalCursor("rank"); + try std.testing.expectEqual(try graph.benchmarkOrdinalCursorRead("rank", true), try graph.benchmarkOrdinalCursorRead("rank", false)); + const job_id = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk (try graph.metricBuildJob(&txn, "rank")).?.job_id; + }; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try graph.topologyKey(&batch, "rank", job_id, try GraphIndex.graphMetricNodeSlotKey(alloc, "rank", job_id, "b")); + defer alloc.free(key); + try batch.delete(key); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var page = (try graph.metricBuildPage(&txn, "rank", job_id, .reduce_ranks, 0, graph_metric_build_summary_leaf_base)).?; + // Claim strings are not valid outside their transaction. Only scalar + // identity is needed: durable bounds are read from the new snapshot. + page.range_lower = "invalid-claim-boundary"; + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(alloc); + var total: u64 = 0; + try std.testing.expect(!try graph.collectGraphMetricPageSlots(&txn, "rank", job_id, page, 0, 1, &total, &slots)); + try std.testing.expectEqual(@as(u64, 2), total); + try std.testing.expect(try graph.collectGraphMetricPageSlots(&txn, "rank", job_id, page, 1, 1, &total, &slots)); + try std.testing.expectEqualSlices(u64, &.{ @as(u64, 1) << 32, (@as(u64, 1) << 32) + 1 }, slots.items); + try std.testing.expectError(error.InvalidGraphMetricBuildProgress, graph.collectGraphMetricPageSlots(&txn, "rank", job_id, page, 3, 1, &total, &slots)); + // The publication boundary still rejects the corrupt dictionary. + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.validateGraphMetricOrdinalDictionary(&txn, "rank", job_id, &.{ "a", "b" }, slots.items)); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const key = try graph.graphMetricActivePlanKey("rank", job_id); + defer alloc.free(key); + const raw = try alloc.dupe(u8, try batch.get(key)); + defer alloc.free(raw); + raw[8] ^= 1; + try batch.put(key, raw); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.graphMetricActivePlan(&txn, "rank", job_id)); + } + // Inline drains must return the terminal failure, not reinterpret it as + // an idle scheduler and overwrite its root cause with NoEligiblePage. + var failed = try graph.runGraphMetricPlannedActive("rank", configs[0]); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expect(std.mem.startsWith(u8, failed.last_error, "GraphMetricBuildPageAttemptsExhausted:")); + try std.testing.expect(std.mem.endsWith(u8, failed.last_error, "cause=InvalidGraphMetricBuildManifest")); +} + +test "graph metric ordinal topology checkpoints stop at byte admission for long typed keys" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-long-type-checkpoint"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-long-type-checkpoint"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const kind = "x" ** edge_type_mod.max_bytes; + const config = GraphMetricConfig{ .name = "rank", .kind = .pagerank, .edge_filter = .{ .mode = .types, .types = &.{kind} }, .max_iterations = 2 }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &.{config} }); + defer graph.close(); + try graph.batchApply(&.{ + .{ .source = "a", .target = "b", .edge_type = kind }, + .{ .source = "b", .target = "a", .edge_type = kind }, + }, &.{}); + var started = try graph.ensureGraphMetricPlannedBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + started.deinit(alloc); + try drainGraphMetricBuildToPublishForTest(&graph, "rank", config, "worker", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, "rank")).?; + const page = (try graph.metricBuildPage(&txn, "rank", job.job_id, .iterate_contributions, 0, 3)).?; + var topology = try graph.ordinalTopologyAlloc(&txn, "rank", config, job.job_id, page, 4096); + defer topology.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), topology.edges.len); + try std.testing.expect(!topology.complete); + try std.testing.expect(topology.cursor.len > kind.len * 2); +} + +test "graph metric membership private store serializes read modify write across concurrent views" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-serialized-writes"); + defer cleanupTmp(store_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var io_impl = std.Io.Threaded.init(alloc, .{}); + defer io_impl.deinit(); + const io = io_impl.io(); + const Worker = struct { + fn run(runtime: std.Io, view: backend_erased.Store, start: *std.Io.Event) !void { + var shared_view = view; + try start.wait(runtime); + for (0..32) |_| { + var batch = try shared_view.beginBatch(); + errdefer batch.abort(); + const current = try GraphIndex.readU64OrZero(&batch, "meta:test_serialized_counter"); + try runtime.sleep(std.Io.Duration.fromNanoseconds(100_000), .awake); + try GraphIndex.putU64(&batch, "meta:test_serialized_counter", current + 1); + try batch.commit(); + var txn = try shared_view.beginWrite(); + errdefer txn.abort(); + const value = try GraphIndex.readU64OrZero(&txn, "meta:test_serialized_counter"); + try runtime.sleep(std.Io.Duration.fromNanoseconds(100_000), .awake); + try GraphIndex.putU64(&txn, "meta:test_serialized_counter", value + 1); + try txn.commit(); + } + } + }; + for ([_]ReverseBackend{ .mem, .lsm_memory, .lsm }) |kind| { + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-serialized-writes"); + defer cleanupTmp(rev_path); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .reverse_backend = kind, .reverse_lsm_options = .{ .flush_threshold = 64 * 1024 } }); + defer graph.close(); + const gate = graph.reverseStore().write_gate orelse return error.TestExpectedWriteGate; + { + var batch = try graph.beginWriteReverseBatch(); + defer batch.abort(); + try std.testing.expect(!gate.tryLock()); + // A write gate must not block snapshot reads. + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + try std.testing.expectEqual(@as(u64, 0), try GraphIndex.readU64OrZero(&read, "meta:test_serialized_counter")); + } + var start: std.Io.Event = .unset; + var first = try io.concurrent(Worker.run, .{ io, graph.reverseStore().*, &start }); + defer _ = first.cancel(io) catch {}; + var second = try io.concurrent(Worker.run, .{ io, graph.reverseStore().*, &start }); + defer _ = second.cancel(io) catch {}; + start.set(io); + try first.await(io); + try second.await(io); + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + try std.testing.expectEqual(@as(u64, 128), try GraphIndex.readU64OrZero(&read, "meta:test_serialized_counter")); + } +} + +test "graph metric membership deltas match immediate updates for duplicates self loops and reversals" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-membership-deltas"); + defer cleanupTmp(store_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-membership-deltas"); + defer cleanupTmp(rev_path); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); + defer graph.close(); + const Operation = struct { source: []const u8, target: []const u8, present: bool }; + const operations = [_]Operation{ + .{ .source = "a", .target = "a", .present = true }, + .{ .source = "a", .target = "a", .present = true }, + .{ .source = "a", .target = "b", .present = true }, + .{ .source = "b", .target = "c", .present = true }, + .{ .source = "b", .target = "c", .present = false }, + .{ .source = "b", .target = "a", .present = true }, + }; + for ([_]bool{ true, false }) |reference| { + var batch = try graph.beginWriteReverseBatch(); + defer batch.abort(); + var updates = typed_edges.Updates.init(alloc); + defer updates.deinit(); + for (operations) |op| { + const key = try reverseEdgeKeyAlloc(alloc, op.target, "links", "cites", op.source); + defer alloc.free(key); + if (reference) try typed_edges.update(alloc, &batch, "cites", key, op.source, op.target, op.present) else try updates.stage(&batch, "cites", key, op.source, op.target, op.present); + } + if (!reference) { + try updates.flush(&batch); + try updates.flush(&batch); // A second flush is a no-op. + } + for ([_][]const u8{ "a", "b", "c" }, [_]u64{ 4, 2, 0 }) |node, expected| { + const key = try std.mem.concat(alloc, u8, &.{ typed_edges.node_prefix, "cites\x00\x00", node }); + defer alloc.free(key); + try std.testing.expectEqual(expected, try GraphIndex.readU64OrZero(&batch, key)); + } + } +} + +test "graph metric coalesced global counters preserve duplicate self-loop and replacement semantics" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-global-deltas"); + defer cleanupTmp(store_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-global-deltas"); + defer cleanupTmp(rev_path); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); + defer graph.close(); + const writes = [_]BatchWrite{ + .{ .source = "a", .target = "a", .edge_type = "link" }, + .{ .source = "a", .target = "b", .edge_type = "link" }, + .{ .source = "a", .target = "b", .edge_type = "link", .weight = 2 }, + }; + try graph.batchApply(&writes, &.{}); + const generation = graph.edge_generation; + const deletes = [_]BatchDelete{ + .{ .source = "a", .target = "a", .edge_type = "link" }, + .{ .source = "a", .target = "b", .edge_type = "link" }, + .{ .source = "a", .target = "b", .edge_type = "link" }, + }; + try graph.batchApply(&writes, &deletes); + try std.testing.expectEqual(generation, graph.edge_generation); + try std.testing.expectEqual(@as(u64, 2), graph.edge_count); + try std.testing.expectEqual(@as(u64, 2), graph.node_count); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(@as(u64, 3), try GraphIndex.readU64OrZero(&txn, "meta:node_ref:a")); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, "meta:node_ref:b")); + } + try graph.batchApply(&.{}, &deletes); + try std.testing.expectEqual(@as(u64, 0), graph.edge_count); + try std.testing.expectEqual(@as(u64, 0), graph.node_count); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get("meta:node_ref:a")); + try std.testing.expectError(error.NotFound, txn.get("meta:node_ref:b")); +} + +test "graph metric filtered postings activate lazily and retain mutation coverage across reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-lazy-postings"); + defer cleanupTmp(store_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-lazy-postings"); + defer cleanupTmp(rev_path); + const configs = [_]GraphMetricConfig{.{ .name = "all", .kind = .degree }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + while (!try graph.prepareGraphMetricPartitionStep(1)) {} + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqualStrings("0", try txn.get(typed_edges.active_key)); + try std.testing.expectError(error.NotFound, txn.get(typed_edges.ready_key)); + var cursor = try txn.openCursor(); + defer cursor.close(); + for ([_][]const u8{ typed_edges.prefix, typed_edges.node_prefix }) |prefix| { + if (try cursor.seekAtOrAfter(prefix)) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, prefix)); + } + } + const filtered = GraphMetricConfig{ .name = "filtered", .kind = .degree, .edge_filter = .{ .mode = .types, .types = &.{"cites"} } }; + try std.testing.expect(!try graph.prepareGraphMetricPartitionForConfigStep(filtered, 1)); + // Only an unfiltered metric remains configured, but the started covering + // index must continue tracking writes behind its backfill cursor. + try graph.addEdge("z", "a", "cites", 1, 0, 0, ""); + try graph.deleteEdge("a", "b", "cites"); + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + while (!try graph.prepareGraphMetricPartitionForConfigStep(filtered, 1)) {} + const full = try graph.benchmarkMetricEdgeScan(filtered.edge_filter, true); + const selected = try graph.benchmarkMetricEdgeScan(filtered.edge_filter, false); + try std.testing.expectEqual(@as(usize, 2), selected.matched); + try std.testing.expectEqual(full.checksum, selected.checksum); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + // Older partial indexes had neither an activation marker nor ready + // flag. Their persisted postings still require foreground maintenance. + try batch.delete(typed_edges.active_key); + try batch.delete(typed_edges.ready_key); + try batch.commit(); + } + try graph.deleteEdge("b", "c", "cites"); + while (!try graph.prepareGraphMetricPartitionForConfigStep(filtered, 1)) {} + const remaining = try graph.benchmarkMetricEdgeScan(filtered.edge_filter, false); + try std.testing.expectEqual(@as(usize, 1), remaining.matched); + var plan = try graph.cachedGraphMetricPartitionPlanForConfig(filtered); + defer plan.deinit(alloc); + try std.testing.expectEqual(@as(u64, 2), plan.node_count); +} + +test "graph metric filtered census completes during unrelated churn and deduplicates shared endpoints" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-filter-census-churn"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-filter-census-churn"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .edge_filter = .{ .mode = .types, .types = &.{ "links", "cites" } }, .max_iterations = 2 }}; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + graph.test_partition_target_units = 1; + try graph.batchApply(&.{ + .{ .source = "a", .target = "b", .edge_type = "cites" }, + .{ .source = "b", .target = "c", .edge_type = "links" }, + }, &.{}); + const epoch = try graph.graphMetricCurrentGeneration("rank"); + var steps: usize = 0; + while (!try graph.prepareGraphMetricPartitionForConfigStep(configs[0], 1)) { + steps += 1; + try std.testing.expect(steps < 32); + var id_buf: [32]u8 = undefined; + const id = try std.fmt.bufPrint(&id_buf, "unrelated-{d}", .{steps}); + try graph.addEdge(id, "unrelated-target", "other", 1, 0, 0, ""); + if (steps == 3) { + graph.close(); + graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + graph.test_partition_target_units = 1; + } + } + var plan = try graph.cachedGraphMetricPartitionPlanForConfig(configs[0]); + defer plan.deinit(alloc); + try std.testing.expectEqual(@as(u64, 2), plan.edge_count); + try std.testing.expectEqual(@as(u64, 3), plan.node_count); + try std.testing.expectEqual(epoch, plan.edge_generation); + var published = try graph.runGraphMetricPlannedDrain("rank", epoch, .{ .worker_ids = &.{"worker"}, .max_steps = 2048 }); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try graph.deleteEdge("a", "b", "cites"); + while (!try graph.prepareGraphMetricPartitionForConfigStep(configs[0], 1)) {} + var changed = try graph.cachedGraphMetricPartitionPlanForConfig(configs[0]); + defer changed.deinit(alloc); + try std.testing.expectEqual(@as(u64, 1), changed.edge_count); + try std.testing.expectEqual(@as(u64, 2), changed.node_count); +} + +test "graph metric shared topology dependency epochs survive attribute writes unrelated types and reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-type-epochs"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-type-epochs"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter = GraphMetricEdgeFilter{ .mode = .types, .types = &.{"cites"} }; + const configs = [_]GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .edge_filter = filter, .max_iterations = 2 }, + .{ .name = "eigen", .kind = .eigenvector, .edge_filter = filter, .max_iterations = 2 }, + }; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.batchApply(&.{ + .{ .source = "a", .target = "b", .edge_type = "cites" }, + .{ .source = "b", .target = "a", .edge_type = "cites" }, + }, &.{}); + const epoch = try graph.graphMetricCurrentGeneration("rank"); + var started = try graph.ensureGraphMetricPlannedBuild("rank", epoch); + started.deinit(alloc); + try graph.addEdge("x", "y", "unrelated", 1, 0, 0, ""); + const global_epoch = graph.edge_generation; + try graph.batchApply(&.{.{ .source = "a", .target = "b", .edge_type = "cites", .weight = 2, .metadata_json = "{\"changed\":true}" }}, &.{.{ .source = "a", .target = "b", .edge_type = "cites" }}); + try graph.deleteEdge("missing", "edge", "cites"); + try std.testing.expectEqual(global_epoch, graph.edge_generation); + try std.testing.expectEqual(epoch, try graph.graphMetricCurrentGeneration("rank")); + var published = try graph.runGraphMetricPlannedDrain("rank", epoch, .{ .worker_ids = &.{"worker"}, .max_steps = 2048 }); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(epoch, published.edge_generation); + const full = try graph.benchmarkMetricEdgeScan(filter, true); + const selected = try graph.benchmarkMetricEdgeScan(filter, false); + try std.testing.expectEqual(@as(usize, 3), full.visited); + try std.testing.expectEqual(@as(usize, 2), selected.visited); + try std.testing.expectEqual(full.checksum, selected.checksum); + graph.close(); + graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + var eigen_started = try graph.ensureGraphMetricPlannedBuild("eigen", epoch); + eigen_started.deinit(alloc); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, "eigen")).?; + try std.testing.expect((try graph.topologyBinding(&txn, "eigen", job.job_id)).?.adopted); + } + var eigen = try graph.runGraphMetricPlannedDrain("eigen", epoch, .{ .worker_ids = &.{"worker"}, .max_steps = 2048 }); + defer eigen.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, eigen.state); + try std.testing.expectEqual(epoch, eigen.edge_generation); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + var stale = try graph.graphMetricStatus("rank"); + defer stale.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, stale.state); +} + +test "graph metric shared topology preparation is independent durable and numerical-free" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-independent-preparation"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-independent-preparation"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 2 }, + .{ .name = "eigen", .kind = .eigenvector, .refresh = .manual, .max_iterations = 2 }, + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual, .max_iterations = 2 }, + }; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + // Production scheduling spans amortize control commits; test-only tiny + // spans elsewhere continue to exercise multiple boundaries cheaply. + try std.testing.expectEqual(@as(usize, 4), graph.graphMetricDegreeScanPageCount(16384)); + try std.testing.expectEqual(@as(usize, 1), graph.graphMetricDegreeReducePageCount(1024)); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + try graph.addEdge("c", "a", "cites", 1, 0, 0, ""); + for (configs) |cfg| { + var queued = try graph.queueGraphMetricBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + defer queued.deinit(alloc); + try std.testing.expectEqual(graph.edge_generation, queued.queued_generation); + try std.testing.expectEqual(@as(u64, 0), queued.build_job_id); + } + while (!try graph.prepareGraphMetricPartitionStep(4096)) {} + // A reverse-capable task satisfies all three numerical kinds. + try std.testing.expectEqual(GraphIndex.TopologyPreparationAdmission.queued, try graph.prepareGraphMetricTopologyDetailed(configs[0], try graph.graphMetricCurrentGeneration(configs[0].name))); + try std.testing.expectEqual(GraphIndex.TopologyPreparationAdmission.waiting, try graph.prepareGraphMetricTopologyDetailed(configs[1], try graph.graphMetricCurrentGeneration(configs[1].name))); + try std.testing.expectEqual(GraphIndex.TopologyPreparationAdmission.waiting, try graph.prepareGraphMetricTopologyDetailed(configs[2], try graph.graphMetricCurrentGeneration(configs[2].name))); + var ready = false; + for (0..256) |i| { + // A partial page stays leased to its owner; another worker may have + // no eligible page until the first worker resumes its checkpoint. + _ = try graph.runGraphMetricTopologyPreparationStep(if (i % 2 == 0) "first" else "second"); + if (i == 7) { + graph.close(); + graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + } + _ = try graph.cleanupGraphMetricTopologyPage(); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + for (configs) |cfg| { + try std.testing.expect(try graph.metricBuildLease(&txn, cfg.name) == null); + try std.testing.expect(try graph.metricBuildJob(&txn, cfg.name) == null); + try std.testing.expectEqual(@as(u64, 0), try graph.metricPublishedGeneration(&txn, cfg.name)); + } + var cur = try txn.openCursor(); + defer cur.close(); + var entry = try cur.seekAtOrAfter(topology_task_prefix); + while (entry) |record| : (entry = try cur.next()) { + if (!std.mem.startsWith(u8, record.key, topology_task_prefix)) break; + const name = try GraphIndex.topologyTaskNameAlloc(alloc, record.key, record.value); + defer alloc.free(name); + if (try graph.metricBuildJob(&txn, name)) |job| { + var job_buf: [20]u8 = undefined; + const vectors = try graph.graphMetricControlKeyAlloc(&.{ name, "job", try std.fmt.bufPrint(&job_buf, "{d}", .{job.job_id}), "vector" }); + defer alloc.free(vectors); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, vectors)); + } + try std.testing.expectEqual(@as(u64, 0), try graph.metricPublishedGeneration(&txn, name)); + } + } + if (try graph.prepareGraphMetricTopology(configs[2], graph.edge_generation)) { + ready = true; + break; + } + } + if (!ready) return error.TopologyPreparationDidNotSeal; + for (0..32) |_| if (!try graph.runGraphMetricTopologyPreparationStep("cleanup")) break; + for (configs) |cfg| { + try std.testing.expect(try graph.prepareGraphMetricTopology(cfg, graph.edge_generation)); + var started = try graph.ensureGraphMetricPlannedBuildFromCachedPlan(cfg.name, graph.edge_generation); + started.deinit(alloc); + var duplicate = try graph.queueGraphMetricBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + duplicate.deinit(alloc); + if (try graph.graphMetricBuildRequested(cfg.name)) return error.TopologyDependentRequestNotConsumed; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + try std.testing.expect((try graph.topologyBinding(&txn, cfg.name, job.job_id)).?.adopted); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, topology_task_prefix)); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, topology_task_control_prefix)); + } + var result = try graph.runGraphMetricPlannedActive(cfg.name, cfg); + defer result.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, result.state); + const expected: f64 = if (cfg.kind == .pagerank) 1.0 / 3.0 else 1.0 / @sqrt(@as(f64, 3)); + for ([_][]const u8{ "a", "b", "c" }) |node| try std.testing.expectApproxEqAbs(expected, (try graph.graphMetricScore(cfg.name, node)).?, 1e-12); + } + for (0..16) |_| _ = try graph.cleanupGraphMetricTopologyPage(); + const backend = graph.reverse_owner.lsm.backend; + const before = backend.snapshotWriteStats(); + for (0..128) |_| try std.testing.expect(!try graph.cleanupGraphMetricTopologyPage()); + const after = backend.snapshotWriteStats(); + try std.testing.expectEqual(before.wal_append_records, after.wal_append_records); + try std.testing.expectEqual(before.wal_append_bytes, after.wal_append_bytes); +} + +test "graph metric shared topology preparation failure retires durably and preserves dependent root cause" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-preparation-failure"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-preparation-failure"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .refresh = .background }, + .{ .name = "eigen", .kind = .eigenvector, .refresh = .manual }, + .{ .name = "degree", .kind = .degree, .refresh = .background }, + }; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("source", "sink", "cites", 1, 0, 0, ""); + var queued = try graph.queueGraphMetricBuild("eigen", try graph.graphMetricCurrentGeneration("eigen")); + queued.deinit(alloc); + while (!try graph.prepareGraphMetricPartitionStep(4096)) {} + try std.testing.expect(!try graph.prepareGraphMetricTopology(configs[0], graph.edge_generation)); + try std.testing.expect(try graph.runGraphMetricTopologyPreparationStep("worker")); + { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var cur = try batch.openCursor(); + const entry = (try cur.seekAtOrAfter(topology_task_prefix)).?; + const name = try GraphIndex.topologyTaskNameAlloc(temp, entry.key, entry.value); + cur.close(); + var job = (try graph.metricBuildJob(&batch, name)).?; + job.last_error = "InjectedTopologyStorageFailure"; + job.retry_count = 1; + try graph.putGraphMetricBuildJobInBatch(&batch, name, job); + // Force several retirement pages, including recovery after the job + // pointer itself has already been deleted. + for (0..1100) |i| { + const suffix = try std.fmt.allocPrint(temp, "retirement-{d:0>5}", .{i}); + const key = try GraphIndex.graphMetricControlKeyWithAllocator(temp, &.{ name, suffix }); + try batch.put(key, "x"); + } + try batch.commit(); + } + try std.testing.expect(try graph.runGraphMetricTopologyPreparationStep("retire")); + graph.close(); + graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + for (0..8) |_| if (!try graph.runGraphMetricTopologyPreparationStep("reopened")) break; + for (configs[0..2]) |cfg| { + var status = try graph.graphMetricStatus(cfg.name); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqualStrings("InjectedTopologyStorageFailure", status.last_error); + try std.testing.expect(!try graph.graphMetricBuildRequested(cfg.name)); + } + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, (try graph.graphMetricSchedulerStatus("degree", null)).state); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, topology_task_prefix)); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, topology_task_control_prefix)); +} + +test "graph metric shared topology retry incarnation fences delayed failure cleanup and admission" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-preparation-incarnation"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-preparation-incarnation"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual }}; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("source", "sink", "cites", 1, 0, 0, ""); + var queued = try graph.queueGraphMetricBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + queued.deinit(alloc); + while (!try graph.prepareGraphMetricPartitionStep(4096)) {} + try std.testing.expect(!try graph.prepareGraphMetricTopology(configs[0], graph.edge_generation)); + const old = read: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + const entry = (try cur.seekAtOrAfter(topology_task_prefix)).?; + break :read .{ .key = try temp.dupe(u8, entry.key), .raw = try temp.dupe(u8, entry.value), .name = try GraphIndex.topologyTaskNameAlloc(temp, entry.key, entry.value) }; + }; + const old_cfg = try GraphIndex.topologyTaskConfigAlloc(temp, old.name, old.raw); + try graph.propagateTopologyTaskFailure(old_cfg, graph.edge_generation, "FirstFailure"); + try std.testing.expect(!try graph.graphMetricBuildRequested("rank")); + var retry = try graph.queueGraphMetricBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + retry.deinit(alloc); + // A retry can arrive before the prior task has finished retirement. + try graph.propagateTopologyTaskFailure(old_cfg, graph.edge_generation, "DelayedDuplicateFailure"); + try std.testing.expect(try graph.graphMetricBuildRequested("rank")); + while (try graph.retireTopologyTaskPage(old.key, old.name, old.raw)) {} + graph.close(); + graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + try std.testing.expectEqual(GraphIndex.TopologyPreparationAdmission.queued, try graph.prepareGraphMetricTopologyDetailed(configs[0], try graph.graphMetricCurrentGeneration(configs[0].name))); + try graph.propagateTopologyTaskFailure(old_cfg, graph.edge_generation, "DelayedOldIncarnationFailure"); + try std.testing.expect(!try graph.retireTopologyTaskPage(old.key, old.name, old.raw)); + try std.testing.expect(try graph.graphMetricBuildRequested("rank")); + var old_view = graph; + old_view.metric_configs = &.{old_cfg}; + old_view.topology_preparation_only = true; + old_view.sealed_vectors = .{ .capacity = 0 }; + try std.testing.expectError(error.GraphMetricBuildSuperseded, old_view.ensureGraphMetricPlannedBuildFromCachedPlan(old.name, graph.edge_generation)); + try std.testing.expect(try graph.runGraphMetricTopologyPreparationStep("replacement")); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const current = try txn.get(old.key); + const current_name = try GraphIndex.topologyTaskNameAlloc(temp, old.key, current); + try std.testing.expect(try GraphIndex.topologyTaskIncarnation(current) > try GraphIndex.topologyTaskIncarnation(old.raw)); + try std.testing.expect(!std.mem.eql(u8, old.name, current_name)); + try std.testing.expect(try graph.metricBuildJob(&txn, current_name) != null); + try std.testing.expect(try graph.metricBuildJob(&txn, old.name) == null); + var status = try graph.graphMetricStatus("rank"); + defer status.deinit(alloc); + try std.testing.expectEqualStrings("FirstFailure", status.last_error); + // An abandoned manual numerical job must accept a new durable request; + // its stale job pointer alone is not proof of an active lease. + var started = try graph.ensureGraphMetricPlannedBuildFromCachedPlan("rank", graph.edge_generation); + started.deinit(alloc); + try std.testing.expect(!try graph.graphMetricBuildRequested("rank")); + try graph.releaseGraphMetricBuildLease("rank"); + var resumed = try graph.queueGraphMetricBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + resumed.deinit(alloc); + try std.testing.expect(try graph.graphMetricBuildRequested("rank")); +} + +test "graph metric shared topology paired failure delivery owns one canonical retry" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-paired-preparation"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-paired-preparation"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "hub", .kind = .hits_hub, .refresh = .manual }, + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual }, + }; + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + var queued = try graph.queueGraphMetricBuild("hub", try graph.graphMetricCurrentGeneration("hub")); + queued.deinit(alloc); + while (!try graph.prepareGraphMetricPartitionStep(4096)) {} + _ = try graph.prepareGraphMetricTopology(configs[0], graph.edge_generation); + const task = read: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + const entry = (try cur.seekAtOrAfter(topology_task_prefix)).?; + const name = try GraphIndex.topologyTaskNameAlloc(temp, entry.key, entry.value); + break :read try GraphIndex.topologyTaskConfigAlloc(temp, name, entry.value); + }; + // Stop exactly after the first alias reports, then accept a user retry. + try graph.recordGraphMetricFailureReasonAtGeneration("hub", "FirstFailure", graph.edge_generation, task.name); + try std.testing.expect(!try graph.graphMetricBuildRequested("authority")); + var retry = try graph.queueGraphMetricBuild("hub", try graph.graphMetricCurrentGeneration("hub")); + retry.deinit(alloc); + try graph.propagateTopologyTaskFailure(task, graph.edge_generation, "DelayedFailure"); + try std.testing.expect(try graph.graphMetricBuildRequested("authority")); + for (configs) |cfg| { + var status = try graph.graphMetricStatus(cfg.name); + defer status.deinit(alloc); + try std.testing.expectEqualStrings("FirstFailure", status.last_error); + } +} + +test "graph metric shared topology admission is bounded and checkpoints rotate across filters" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-fair-preparation"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-fair-preparation"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var configs: [GraphIndex.max_pending_topology_tasks + 1]GraphMetricConfig = undefined; + for (&configs, 0..) |*cfg, i| { + const name = try std.fmt.allocPrint(temp, "filter-{d}", .{i}); + const types = try temp.alloc([]const u8, 1); + types[0] = name; + cfg.* = .{ .name = name, .kind = .pagerank, .edge_filter = .{ .mode = .types, .types = types } }; + } + var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + for (configs) |cfg| try graph.addEdge("a", "b", cfg.name, 1, 0, 0, ""); + while (!try graph.prepareGraphMetricPartitionStep(4096)) {} + for (configs, 0..) |cfg, i| try std.testing.expectEqual( + if (i < GraphIndex.max_pending_topology_tasks) GraphIndex.TopologyPreparationAdmission.queued else .waiting, + try graph.prepareGraphMetricTopologyDetailed(cfg, try graph.graphMetricCurrentGeneration(cfg.name)), + ); + for (0..GraphIndex.max_pending_topology_tasks) |_| try std.testing.expect(try graph.runGraphMetricTopologyPreparationStep("fair-worker")); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry = try cur.seekAtOrAfter(topology_task_prefix); + var started: usize = 0; + while (entry) |item| : (entry = try cur.next()) { + if (!std.mem.startsWith(u8, item.key, topology_task_prefix)) break; + const name = try GraphIndex.topologyTaskNameAlloc(temp, item.key, item.value); + try std.testing.expect(try graph.metricBuildJob(&txn, name) != null); + started += 1; + } + try std.testing.expectEqual(@as(usize, GraphIndex.max_pending_topology_tasks), started); +} + +test "graph metric shared topology survives producer cleanup and reopen across numerical kinds" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-shared-topology"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-shared-topology"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual, .max_iterations = 2 }, + .{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 2 }, + .{ .name = "eigen", .kind = .eigenvector, .refresh = .manual, .max_iterations = 2 }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + try graph.addEdge("c", "a", "cites", 1, 0, 0, ""); + var started = try graph.ensureGraphMetricPlannedBuild("authority", try graph.graphMetricCurrentGeneration("authority")); + started.deinit(alloc); + const owner = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, "authority")).?; + const binding = (try graph.topologyBinding(&txn, "authority", job.job_id)).?; + try std.testing.expect(!binding.adopted); + break :blk binding.id; + }; + var published = try graph.runGraphMetricPlannedActive("authority", configs[0]); + published.deinit(alloc); + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + for (configs[1..]) |cfg| { + var building = try graph.ensureGraphMetricPlannedBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + building.deinit(alloc); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + const binding = (try graph.topologyBinding(&txn, cfg.name, job.job_id)).?; + try std.testing.expect(binding.adopted); + try std.testing.expectEqualSlices(u8, &owner, &binding.id); + for ([_]GraphIndex.GraphMetricBuildPhase{ .scan_edges_and_out_degree, .iterate_contributions }) |phase| { + const page = (try graph.metricBuildPage(&txn, cfg.name, job.job_id, phase, 0, GraphIndex.graphMetricBuildPhasePageIdBase(cfg.kind, phase))).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 0), page.attempt); + } + } + var result = try graph.runGraphMetricPlannedActive(cfg.name, cfg); + result.deinit(alloc); + const expected: f64 = if (cfg.kind == .pagerank) 1.0 / 3.0 else 1.0 / @sqrt(@as(f64, 3)); + for ([_][]const u8{ "a", "b", "c" }) |node| try std.testing.expectApproxEqAbs(expected, (try graph.graphMetricScore(cfg.name, node)).?, 1e-12); + } + // Empty configuration must not leave the owner or stale job pins behind. + const saved_configs = graph.metric_configs; + graph.metric_configs = &.{}; + defer graph.metric_configs = saved_configs; + for (0..8) |_| _ = try graph.cleanupGraphMetricTopologyPage(); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const key = try topology_owner.catalogKey(alloc, owner); + defer alloc.free(key); + try std.testing.expectError(error.NotFound, txn.get(key)); + const pins = try topology_owner.pinsPrefix(alloc, owner); + defer alloc.free(pins); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, pins)); +} + +test "graph metric shared topology isolates concurrent producers and fences bounded reclamation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-topology-fencing"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-topology-fencing"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "first", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }, + .{ .name = "second", .kind = .pagerank, .refresh = .manual, .max_iterations = 1, .damping = 0.5 }, + .{ .name = "consumer", .kind = .eigenvector, .refresh = .manual, .max_iterations = 1 }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + var owners: [2]topology_owner.Id = undefined; + for (configs[0..2], 0..) |cfg, i| { + var started = try graph.ensureGraphMetricPlannedBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + started.deinit(alloc); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + const binding = (try graph.topologyBinding(&txn, cfg.name, job.job_id)).?; + try std.testing.expect(!binding.adopted); + owners[i] = binding.id; + } + try std.testing.expect(!std.mem.eql(u8, &owners[0], &owners[1])); + for (configs[0..2]) |cfg| { + var result = try graph.runGraphMetricPlannedActive(cfg.name, cfg); + result.deinit(alloc); + } + var started = try graph.ensureGraphMetricPlannedBuild("consumer", try graph.graphMetricCurrentGeneration("consumer")); + started.deinit(alloc); + const consumer = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, "consumer")).?; + const binding = (try graph.topologyBinding(&txn, "consumer", job.job_id)).?; + try std.testing.expect(binding.adopted); + try std.testing.expectEqualSlices(u8, &owners[0], &binding.id); + break :blk try graph.cloneGraphMetricBuildJobAlloc(job); + }; + defer graph.deinitClonedGraphMetricBuildJob(consumer); + // A graph mutation invalidates reuse but cannot reclaim an active pin. + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + for (0..8) |_| _ = try graph.cleanupGraphMetricTopologyPage(); + const owner_key = try topology_owner.catalogKey(alloc, owners[0]); + defer alloc.free(owner_key); + const data = try topology_owner.dataPrefix(alloc, owners[0]); + defer alloc.free(data); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + _ = try batch.get(owner_key); + var completed = consumer; + completed.phase = .complete; + try graph.putGraphMetricBuildJobInBatch(&batch, "consumer", completed); + for (0..5000) |i| { + const key = try std.fmt.allocPrint(alloc, "{s}gc-fixture/{d:0>8}", .{ data, i }); + defer alloc.free(key); + try batch.put(key, "x"); + } + try batch.commit(); + } + graph.topology_gc_cursor = null; + try std.testing.expect(try graph.cleanupGraphMetricTopologyPage()); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const record = try topology_owner.Record.decode(try txn.get(owner_key)); + try std.testing.expectEqual(topology_owner.State.deleting, record.state); + try std.testing.expect(try graph.hasKeysWithPrefixInBatch(&txn, data)); + try std.testing.expectError(error.GraphMetricBuildSuperseded, graph.topologyBinding(&txn, "consumer", consumer.job_id)); + } + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + for (0..32) |_| _ = try graph.cleanupGraphMetricTopologyPage(); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(owner_key)); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, data)); +} + +test "graph metric shared topology canonicalizes filters and retires removed filters without graph writes" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-topology-filters"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-topology-filters"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "first", .kind = .pagerank, .refresh = .manual, .max_iterations = 1, .edge_filter = .{ .mode = .types, .types = &.{ "mentions", "cites" } } }, + .{ .name = "alias", .kind = .pagerank, .refresh = .manual, .max_iterations = 2, .damping = 0.5, .edge_filter = .{ .mode = .types, .types = &.{ "cites", "mentions" } } }, + .{ .name = "changed", .kind = .pagerank, .refresh = .manual, .max_iterations = 1, .edge_filter = .{ .mode = .types, .types = &.{"cites"} } }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "a", "mentions", 1, 0, 0, ""); + try graph.addEdge("c", "a", "excluded", 1, 0, 0, ""); + var owners: [3]topology_owner.Id = undefined; + for (configs, 0..) |cfg, i| { + var started = try graph.ensureGraphMetricPlannedBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + started.deinit(alloc); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + const binding = (try graph.topologyBinding(&txn, cfg.name, job.job_id)).?; + try std.testing.expectEqual(i == 1, binding.adopted); + owners[i] = binding.id; + } + var result = try graph.runGraphMetricPlannedActive(cfg.name, cfg); + result.deinit(alloc); + try std.testing.expect((try graph.graphMetricScore(cfg.name, "c")) == null); + } + try std.testing.expectEqualSlices(u8, &owners[0], &owners[1]); + try std.testing.expect(!std.mem.eql(u8, &owners[0], &owners[2])); + try std.testing.expectApproxEqAbs(@as(f64, 0.5), (try graph.graphMetricScore("alias", "a")).?, 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 0.2875), (try graph.graphMetricScore("changed", "a")).?, 1e-12); + const saved_configs = graph.metric_configs; + graph.metric_configs = saved_configs[2..]; + defer graph.metric_configs = saved_configs; + for (0..16) |_| _ = try graph.cleanupGraphMetricTopologyPage(); + // Retained-owner census work must not keep a worker pool busy forever. + for (0..4) |_| try std.testing.expect(!(try graph.cleanupGraphMetricTopologyPageDetailed()).progressed); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const obsolete = try topology_owner.catalogKey(alloc, owners[0]); + defer alloc.free(obsolete); + try std.testing.expectError(error.NotFound, txn.get(obsolete)); + const retained = try topology_owner.catalogKey(alloc, owners[2]); + defer alloc.free(retained); + _ = try topology_owner.Record.decode(try txn.get(retained)); +} + +test "graph metric membership initializes all vector lanes without producer rediscovery" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-membership-initialize"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-membership-initialize"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .refresh = .manual }, + .{ .name = "eigen", .kind = .eigenvector, .refresh = .manual }, + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual }, + // Keep this independent from the compatible authority job so both + // public entry kinds exercise initialization as lifecycle owners. + .{ .name = "hub", .kind = .hits_hub, .refresh = .manual, .max_iterations = 2 }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs, .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("b", "c", "cites", 1, 0, 0, ""); + for (configs) |cfg| { + var building = try graph.ensureGraphMetricPlannedBuild(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + defer building.deinit(alloc); + var ready = false; + for (0..64) |_| { + _ = try graph.runGraphMetricPlannedWorkerPageStep(cfg.name, cfg, "worker"); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + if (job.phase == .initialize_ranks) { + if (try graph.metricBuildPage(&txn, cfg.name, job.job_id, .initialize_ranks, 0, 0)) |root| { + if (root.state == .complete) { + ready = true; + break; + } + } + } + } + _ = try graph.runGraphMetricPlannedCoordinatorStep(cfg.name, cfg); + } + try std.testing.expect(ready); + var txn = try graph.beginReadReverseTxn(); + const job = (try graph.metricBuildJob(&txn, cfg.name)).?; + txn.abort(); + // A post-seal consumer must not touch this discovery-only namespace. + const partial = try graph.graphMetricBuildPageRankNodePartialKeyAlloc(cfg.name, job.job_id, "a", 0); + defer alloc.free(partial); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(partial, "invalid-discovery-row"); + try batch.commit(); + } + const page = (try graph.claimNextGraphMetricBuildPage(cfg.name, job.job_id, .initialize_ranks, 0, "worker")).?; + for (0..3) |_| { + const count = switch (cfg.kind) { + .pagerank => try graph.executePageRankInitializeBuildPageWithLimit(cfg.name, job, page, 1), + .eigenvector => try graph.executeEigenvectorInitializeBuildPageWithLimit(cfg.name, job, page, 1), + .hits_authority, .hits_hub => try graph.executeHitsInitializeBuildPageWithLimit(cfg.name, job, page, 1), + else => unreachable, + }; + try std.testing.expectEqual(@as(usize, 1), count); + } + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + const lanes: []const []const u8 = if (cfg.kind == .pagerank) &.{ "rank", "factor" } else if (cfg.kind == .eigenvector) &.{"rank"} else &.{ "authority", "hub" }; + for (lanes) |lane| { + const values = try graph.readGraphMetricVectorAlloc(&read, cfg.name, job.job_id, lane, 0, &.{ "a", "b", "c" }, true); + defer alloc.free(values); + try std.testing.expectEqual(@as(usize, 3), values.len); + for (values, 0..) |value, i| { + const expected: f64 = if (cfg.kind != .pagerank) 1.0 / @sqrt(@as(f64, 3)) else if (std.mem.eql(u8, lane, "factor") and i == 2) 0 else 1.0 / 3.0; + try std.testing.expectApproxEqAbs(expected, value, 1e-12); + } + } + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, (try graph.metricBuildPage(&read, cfg.name, job.job_id, .initialize_ranks, 0, page.page_id)).?.state); + } +} + +test "graph metric membership resumes across sealed blocks and rejects corrupt resume ordinals" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-membership-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-membership-resume"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); + defer graph.close(); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var ids: [300][]const u8 = undefined; + for (&ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(temp, "node-{d:0>4}", .{i}); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + // A filtered graph with 256 planning leaves and one populated leaf. + try graph.putGraphMetricBuildManifestInBatch(&batch, "rank", .{ .job_id = 1, .node_count = 131072 }); + for (0..256) |i| try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ + .job_id = 1, + .phase = .initialize_ranks, + .page_id = graph_metric_build_summary_leaf_base + i, + .range_kind = .summary, + .state = .complete, + .range_lower = if (i == 0) "" else try std.fmt.allocPrint(temp, "z-{d:0>4}", .{i - 1}), + .range_upper = if (i == 255) "" else try std.fmt.allocPrint(temp, "z-{d:0>4}", .{i}), + .completed_units = if (i == 0) ids.len else 0, + .total_units = 512, + }); + try graph.writeGraphMetricMembership(&batch, "rank", 1, 0, 0, ids[0..270]); + // Retry with a different checkpoint boundary, then append. + try graph.writeGraphMetricMembership(&batch, "rank", 1, 0, 200, ids[200..280]); + try graph.writeGraphMetricMembership(&batch, "rank", 1, 0, 280, ids[280..]); + for (ids, 0..) |id, i| try GraphIndex.putU64(&batch, try GraphIndex.graphMetricNodeSlotKey(temp, "rank", 1, id), (@as(u64, 1) << 32) | i); + try batch.commit(); + } + // Publication admission is independent of the 64-node planning unit. + // Test the exact byte boundary, resume, oversized-first-row progress, and + // a whole sealed leaf larger than the former publication checkpoint. + for ([_]usize{ 0, 9, 18, 1024 * 1024 }) |byte_limit| { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var admitted = std.ArrayListUnmanaged([]u8).empty; + defer { + for (admitted.items) |node| alloc.free(node); + admitted.deinit(alloc); + } + var slots = std.ArrayListUnmanaged(u64).empty; + defer slots.deinit(alloc); + const complete = try graph.collectSealedGraphMetricMembership(&txn, "rank", 1, "", "z-0000", "", 4096, byte_limit, &admitted, &slots); + const expected = @min(ids.len, @max(@as(usize, 1), byte_limit / ids[0].len)); + try std.testing.expectEqual(expected, admitted.items.len); + try std.testing.expectEqual(expected == ids.len, complete); + try graph.validateGraphMetricOrdinalDictionary(&txn, "rank", 1, admitted.items, slots.items); + } + for ([_]bool{ false, true }) |corrupt| { + if (corrupt) { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putU64(&batch, try GraphIndex.graphMetricNodeSlotKey(temp, "rank", 1, ids[255]), (@as(u64, 1) << 32) | 299); + try batch.commit(); + } + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| alloc.free(node); + nodes.deinit(alloc); + } + if (corrupt) { + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.collectGraphMetricInitializedNodesInRange(&txn, "rank", 1, "", "", ids[255], 2, &nodes)); + } else { + try std.testing.expect(!try graph.collectGraphMetricInitializedNodesInRange(&txn, "rank", 1, "", "", ids[255], 2, &nodes)); + try std.testing.expectEqual(@as(usize, 2), nodes.items.len); + try std.testing.expectEqualStrings(ids[256], nodes.items[0]); + try std.testing.expectEqualStrings(ids[257], nodes.items[1]); + } + } +} + +test "graph metric ordinal borrowed folds match the allocating path without warm tile allocations" { + const alloc = std.testing.allocator; + const expected = try GraphIndex.benchmarkOrdinalFold(alloc, true, 32); + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + const actual = try GraphIndex.benchmarkOrdinalFold(failing.allocator(), false, 32); + try std.testing.expectEqual(expected, actual); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); +} + +test "graph metric ordinal fold target scratch fences chunk-local aliases" { + var scratch = GraphIndex.OrdinalFoldScratch{ .arena = std.heap.ArenaAllocator.init(std.testing.allocator) }; + defer scratch.arena.deinit(); + const slots = [_]u64{ 1, 255, 257, 511, 800 }; + scratch.prepareTargets(&slots, 0); + try std.testing.expectEqual(@as(u16, 0), scratch.targets[1]); + try std.testing.expectEqual(@as(u16, 1), scratch.targets[255]); + try std.testing.expectEqual(std.math.maxInt(u16), scratch.targets[32]); + scratch.prepareTargets(&slots, 1); + try std.testing.expectEqual(@as(u16, 2), scratch.targets[1]); + try std.testing.expectEqual(@as(u16, 3), scratch.targets[255]); + scratch.prepareTargets(&slots, 3); + try std.testing.expectEqual(std.math.maxInt(u16), scratch.targets[1]); + try std.testing.expectEqual(@as(u16, 4), scratch.targets[32]); +} + +test "graph metric vector chunks cache gathers across a bounded fold" { + const alloc = std.testing.allocator; + var graph: GraphIndex = undefined; + graph.alloc = alloc; + const Txn = struct { + chunk: vector_chunk.Chunk = @splat(0), + calls: usize = 0, + pub fn getManySorted(self: *@This(), keys: []const []const u8, values: []?[]const u8) !void { + self.calls += 1; + try std.testing.expectEqual(@as(usize, 1), keys.len); + values[0] = &self.chunk; + } + }; + var txn = Txn{}; + try vector_chunk.put(&txn.chunk, 1, 3); + try vector_chunk.put(&txn.chunk, 2, 5); + var cache = GraphIndex.VectorReadCache.empty; + defer cache.deinit(alloc); + for (0..16) |_| { + const values = try graph.readGraphMetricVectorSlotsCachedAlloc(&txn, "rank", 1, "rank", 0, &.{ 1, 2, 1 }, true, &cache); + defer alloc.free(values); + try std.testing.expectEqualSlices(f64, &.{ 3, 5, 3 }, values); + } + try std.testing.expectEqual(@as(usize, 1), txn.calls); + try std.testing.expectEqual(@as(u32, 1), cache.count()); + try std.testing.expectError(error.InvalidGraphMetricScore, graph.readGraphMetricVectorSlotsCachedAlloc(&txn, "rank", 1, "rank", 0, &.{3}, true, &cache)); +} + +test "graph metric vector chunks sealed cache survives checkpoints without borrowing transactions" { + const alloc = std.testing.allocator; + var graph: GraphIndex = undefined; + graph.alloc = alloc; + var sealed = @import("sealed_vector_cache.zig").Cache{}; + defer sealed.deinit(alloc); + const Txn = struct { + chunk: vector_chunk.Chunk = @splat(0), + calls: usize = 0, + pub fn getManySorted(self: *@This(), keys: []const []const u8, values: []?[]const u8) !void { + self.calls += keys.len; + for (values) |*value| value.* = &self.chunk; + } + }; + var txn = Txn{}; + try vector_chunk.put(&txn.chunk, 1, 3); + for (0..16) |_| { + var cache = GraphIndex.VectorReadCache{ .sealed = &sealed }; + defer cache.deinit(alloc); + const values = try graph.readGraphMetricVectorSlotsCachedAlloc(&txn, "rank", 1, "rank", 0, &.{1}, true, &cache); + defer alloc.free(values); + try std.testing.expectEqualSlices(f64, &.{3}, values); + // The next transaction reuses/invalidates the previous read buffer. + @memset(&txn.chunk, 0); + } + try std.testing.expectEqual(@as(usize, 1), txn.calls); + var next_epoch = GraphIndex.VectorReadCache{ .sealed = &sealed }; + defer next_epoch.deinit(alloc); + try std.testing.expectError(error.InvalidGraphMetricScore, graph.readGraphMetricVectorSlotsCachedAlloc(&txn, "rank", 1, "rank", 1, &.{1}, true, &next_epoch)); + try std.testing.expectEqual(@as(usize, 2), txn.calls); +} + +test "graph metric ordinal publication checkpoints ordered runs and resumes before pointer swap" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-rank-checkpoint"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-rank-checkpoint"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const cfg = GraphMetricConfig{ .name = "degree", .kind = .degree, .refresh = .manual }; + const options = GraphIndexOptions{ .metric_configs = &.{cfg} }; + const job = GraphIndex.GraphMetricBuildJob{ .job_id = 1, .score_generation = 1, .phase = .publish_generation }; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + var scores: [600]GraphIndex.GraphMetricScore = undefined; + for (&scores, 0..) |*score, i| score.* = .{ .node = try std.fmt.allocPrint(temp, "node-{d:0>4}", .{i}), .score = @floatFromInt(i % 17) }; + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildJobInBatch(&batch, cfg.name, job); + try graph.putGraphMetricBuildManifestInBatch(&batch, cfg.name, .{ .job_id = 1, .score_generation = 1, .config_fingerprint = GraphIndex.graphMetricConfigFingerprint(cfg) }); + try graph.putPlannedGraphMetricScorePageInBatch(&batch, cfg.name, job, cfg.name, &scores); + // A replacement attempt updates one row without leaving a second + // ordered key for the node in the unpublished staging run. + scores[0].score = 100; + try graph.putPlannedGraphMetricScorePageInBatch(&batch, cfg.name, job, cfg.name, scores[0..1]); + try batch.commit(); + try std.testing.expect(!try graph.checkpointGraphMetricRankPrefix(cfg.name, cfg, job, cfg.name, scores.len)); + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + const prefix = try graph.graphMetricRankPrefixAlloc(cfg.name, job.score_generation); + defer alloc.free(prefix); + try std.testing.expectEqual(@as(usize, GraphIndex.graph_metric_rank_checkpoint_entries), try GraphIndex.countKeysWithPrefix(&read, prefix)); + const stage = try graph.graphMetricRankStagePrefixAlloc(cfg.name, job.job_id, cfg.name, "rank-run"); + defer alloc.free(stage); + try std.testing.expectEqual(scores.len, try GraphIndex.countKeysWithPrefix(&read, stage)); + try std.testing.expectEqual(@as(u64, 0), try graph.metricPublishedGeneration(&read, cfg.name)); + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.verifyGraphMetricRankReady(&read, cfg.name, job, cfg.name, scores.len)); + } + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + try std.testing.expect(!try graph.checkpointGraphMetricRankPrefix(cfg.name, cfg, job, cfg.name, scores.len)); + try std.testing.expect(try graph.checkpointGraphMetricRankPrefix(cfg.name, cfg, job, cfg.name, scores.len)); + // A duplicate coordinator observes the durable completion receipt. + try std.testing.expect(try graph.checkpointGraphMetricRankPrefix(cfg.name, cfg, job, cfg.name, scores.len)); + var stale = job; + stale.job_id += 1; + try std.testing.expectError(error.GraphMetricBuildJobMismatch, graph.checkpointGraphMetricRankPrefix(cfg.name, cfg, stale, cfg.name, scores.len)); + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + try graph.verifyGraphMetricRankReady(&read, cfg.name, job, cfg.name, scores.len); + const actual = try graph.graphMetricTopKInTxnAlloc(&read, cfg.name, job.score_generation, scores.len); + defer { + for (actual) |*score| score.deinit(alloc); + alloc.free(actual); + } + std.mem.sort(GraphIndex.GraphMetricScore, &scores, {}, GraphIndex.graphMetricScoreLessThan); + try std.testing.expectEqual(scores.len, actual.len); + for (scores, actual) |expected, score| { + try std.testing.expectEqualStrings(expected.node, score.node); + try std.testing.expectEqual(expected.score, score.score); + } + } +} + +test "graph metric ordinal packing densifies fragmented output and fences incomplete attempts" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-adjacency-packing"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-adjacency-packing"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const cfg = GraphMetricConfig{ .name = "rank", .kind = .eigenvector, .refresh = .manual }; + const options = GraphIndexOptions{ .metric_configs = &.{cfg}, .reverse_lsm_options = .{ .flush_threshold = 8192 } }; + const job = GraphIndex.GraphMetricBuildJob{ .job_id = 1, .phase = .reduce_ranks }; + const page = GraphIndex.GraphMetricBuildPage{ .job_id = 1, .phase = .reduce_ranks, .page_id = graph_metric_build_summary_leaf_base, .state = .leased, .range_kind = .summary, .worker_id = "original", .attempt = 1, .lease_expires_at_ms = 100, .total_units = 1 }; + const owner_id: topology_owner.Id = @splat('c'); + const slot: u64 = @as(u64, 1) << 32; + const chunk = slot / vector_chunk.entries; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const owner = topology_owner.Record{ .generation = 0, .identity = @splat(3), .filter = try topology_owner.filterDigest(temp, cfg.edge_filter), .bidirectional = false }; + try batch.put(try topology_owner.catalogKey(temp, owner_id), &owner.encode()); + const binding_key = try graph.topologyBindingKey("rank", 1); + defer alloc.free(binding_key); + try batch.put(binding_key, &(topology_owner.Binding{ .id = owner_id, .adopted = false }).encode()); + try graph.putGraphMetricBuildManifestInBatch(&batch, "rank", .{ .job_id = 1, .node_count = 1 }); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .initialize_ranks, .page_id = graph_metric_build_summary_leaf_base, .range_kind = .summary, .state = .complete, .completed_units = 1, .total_units = 1 }); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .initialize_ranks, .page_id = 0, .range_kind = .summary, .state = .complete, .completed_units = 1, .total_units = 1 }); + try graph.sealGraphMetricActivePlan(&batch, "rank", cfg, job); + try graph.writeGraphMetricMembership(&batch, "rank", 1, 0, 0, &.{"a"}); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", page); + try graph.putGraphMetricBuildPageInBatch(&batch, "rank", .{ .job_id = 1, .phase = .iterate_contributions, .page_id = 3, .state = .complete, .attempt = 2 }); + const slot_key = try graph.topologyKey(&batch, "rank", 1, try GraphIndex.graphMetricNodeSlotKey(alloc, "rank", 1, "a")); + defer alloc.free(slot_key); + try GraphIndex.putU64(&batch, slot_key, slot); + const node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("rank", 1, "a", 0); + defer alloc.free(node_key); + try GraphIndex.putU64(&batch, node_key, 1); + var vector: vector_chunk.Chunk = @splat(0); + try vector_chunk.put(&vector, 0, 2); + try batch.put(try GraphIndex.graphMetricVectorChunkKey(temp, "rank", 1, "rank", 0, chunk), &vector); + const prefix = try graph.ordinalAdjacencyPrefixAlloc("rank", 1, .iterate_contributions, chunk); + defer alloc.free(prefix); + const edge = [_]ordinal_blocks.Edge{.{ .source = slot, .target = slot }}; + const encoded = try ordinal_blocks.encodeTopology(temp, .{ .edges = @constCast(&edge), .cursor = @constCast(""), .scanned = 1, .complete = true }); + // The abandoned attempt must not be decoded, even if malformed. + try batch.put(try std.fmt.allocPrint(temp, "{s}{d:0>20}:{d:0>20}:{d:0>20}", .{ prefix, 3, 1, 0 }), "invalid"); + for (0..1025) |i| try batch.put(try std.fmt.allocPrint(temp, "{s}{d:0>20}:{d:0>20}:{d:0>20}", .{ prefix, 3, 2, i }), encoded); + try batch.commit(); + } + try std.testing.expectEqual(@as(usize, 512), try graph.compactOrdinalAdjacencyChunk("rank", job, page, .iterate_contributions, chunk, 512)); + } + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + const base = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.topologyKey(&txn, "rank", 1, try graph.packedAdjacencyBaseAlloc("rank", 1, .iterate_contributions, chunk)); + }; + defer alloc.free(base); + const receipt_key = try std.fmt.allocPrint(temp, "{s}complete", .{base}); + try std.testing.expectEqual(@as(usize, 512), try graph.compactOrdinalAdjacencyChunk("rank", job, page, .iterate_contributions, chunk, 512)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(receipt_key)); + const state = try adjacency_blocks.State.decode(try txn.get(try std.fmt.allocPrint(temp, "{s}state", .{base}))); + try std.testing.expectEqual(@as(u64, 3), state.blocks); + try std.testing.expectEqual(@as(usize, 255), state.count); + } + const replacement = (try graph.claimGraphMetricBuildPageAt("rank", 1, .reduce_ranks, 0, page.page_id, "replacement", 101)).?; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.compactOrdinalAdjacencyChunk("rank", job, page, .iterate_contributions, chunk, 512)); + for ([_]usize{ 512, 512, 2 }) |count| try std.testing.expectEqual(count, try graph.compactOrdinalAdjacencyChunk("rank", job, replacement, .iterate_contributions, chunk, 512)); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const abandoned = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/", .{ base, page.attempt }); + const winning = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/", .{ base, replacement.attempt }); + const data = try topology_owner.dataPrefix(temp, owner_id); + var digest: topology_owner.Digest = undefined; + std.crypto.hash.sha2.Sha256.hash(abandoned, &digest, .{}); + const task = try std.fmt.allocPrint(temp, "{s}retired/{s}", .{ data, std.fmt.bytesToHex(digest, .lower) }); + try batch.put(task, winning); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.cleanupTopologyAttemptInBatch(&batch, temp, owner_id)); + try batch.put(task, abandoned); + // Three abandoned tiles plus their queue marker, never the winner. + try std.testing.expectEqual(@as(usize, 4), try graph.cleanupTopologyAttemptInBatch(&batch, temp, owner_id)); + try std.testing.expectEqual(@as(usize, 0), try GraphIndex.countKeysWithPrefix(&batch, abandoned)); + try std.testing.expectEqual(@as(usize, 5), try GraphIndex.countKeysWithPrefix(&batch, winning)); + try batch.commit(); + } + const missing_key = try std.fmt.allocPrint(temp, "{s}data/{d:0>20}/{d:0>20}", .{ base, replacement.attempt, 2 }); + var saved: []const u8 = undefined; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const receipt = try adjacency_blocks.Receipt.decode(try batch.get(receipt_key)); + try std.testing.expectEqual(replacement.attempt, receipt.attempt); + try std.testing.expectEqual(@as(u64, 1025), receipt.edges); + try std.testing.expectEqual(@as(u64, 5), receipt.blocks()); + saved = try temp.dupe(u8, try batch.get(missing_key)); + try batch.delete(missing_key); + try batch.commit(); + } + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.executeOrdinalReduceSummary("rank", cfg, job, replacement, 256, 512)); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(missing_key, saved); + try batch.commit(); + } + try std.testing.expectEqual(@as(usize, 5), try graph.executeOrdinalReduceSummary("rank", cfg, job, replacement, 256, 512)); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const values = try graph.pageRankContributionsForNodesAlloc(&txn, "rank", 1, 0, &.{"a"}); + defer alloc.free(values); + try std.testing.expectEqualSlices(f64, &.{2050}, values); +} + +test "graph metric consumer barrier retires bounded input pages and resumes after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-input-gc"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-input-gc"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }}; + const options = GraphIndexOptions{ .metric_configs = &configs, .reverse_lsm_options = .{ .flush_threshold = 8192 } }; + var job_id: u64 = 0; + var retirement_key: []u8 = ""; + defer if (retirement_key.len != 0) alloc.free(retirement_key); + var first_cursor: []u8 = ""; + defer if (first_cursor.len != 0) alloc.free(first_cursor); + { + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + var building = try graph.ensureGraphMetricPlannedBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + defer building.deinit(alloc); + var complete = false; + for (0..32) |_| { + const step = try graph.runGraphMetricPlannedWorkerPageStep("rank", configs[0], "worker"); + if (step.phase == .reduce_ranks) { + var read = try graph.beginWriteReverseBatch(); + defer read.abort(); + const active = (try graph.metricBuildJob(&read, "rank")).?; + const summary = try graph.summarizeGraphMetricBuildPhaseInBatch(&read, "rank", configs[0], active, .reduce_ranks, 0); + if (summary.state == .complete) { + complete = true; + break; + } + } else _ = try graph.runGraphMetricPlannedCoordinatorStep("rank", configs[0]); + } + try std.testing.expect(complete); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + job_id = (try graph.metricBuildJob(&batch, "rank")).?.job_id; + // Additional temporary vectors exercise GC cardinality independently + // of the numerical kernel and its small fixture topology. + for (0..2 * graph_metric_build_adoption_page_units + 3) |i| { + const key = try GraphIndex.graphMetricVectorChunkKey(alloc, "rank", job_id, "raw_rank", 0, i); + defer alloc.free(key); + try GraphIndex.putF64(&batch, key, 0); + } + try batch.commit(); + } + const cleanup = try graph.runGraphMetricPlannedCoordinatorStep("rank", configs[0]); + try std.testing.expect(!cleanup.advanced_phase); + try std.testing.expectEqual(graph_metric_build_adoption_page_units, cleanup.retired_input_records); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, (try graph.metricBuildJob(&txn, "rank")).?.phase); + const job_id_text = try std.fmt.allocPrint(alloc, "{d}", .{job_id}); + defer alloc.free(job_id_text); + const prefix = try graph.graphMetricControlKeyAlloc(&.{ "rank", "job", job_id_text, "vector", "raw_rank", "0" }); + defer alloc.free(prefix); + try std.testing.expectEqual(graph_metric_build_adoption_page_units + 4, try GraphIndex.countKeysWithPrefix(&txn, prefix)); + const job_text = try std.fmt.allocPrint(alloc, "{d}", .{job_id}); + defer alloc.free(job_text); + const retirement = try graph.graphMetricControlKeyAlloc(&.{ "rank", "job", job_text, "retirement", "reduce_ranks", "0" }); + defer alloc.free(retirement); + retirement_key = try std.fmt.allocPrint(alloc, "{s}{s}", .{ retirement, prefix }); + first_cursor = try alloc.dupe(u8, try txn.get(retirement_key)); + try std.testing.expect(std.mem.startsWith(u8, first_cursor, prefix)); + } + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", options); + defer graph.close(); + const resumed = try graph.runGraphMetricPlannedWorkerStep("rank", configs[0], "replacement"); + try std.testing.expect(!resumed.advanced_phase and !resumed.claimed_page); + try std.testing.expectEqual(graph_metric_build_adoption_page_units, resumed.retired_input_records); + { + var read = try graph.beginReadReverseTxn(); + defer read.abort(); + try std.testing.expect(std.mem.order(u8, first_cursor, try read.get(retirement_key)) == .lt); + } + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("rank", job_id, .reduce_ranks, 0)); + try std.testing.expect(!try graph.advanceGraphMetricBuildPhaseIfReady("rank", job_id, .reduce_ranks, 0)); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job_id_text = try std.fmt.allocPrint(alloc, "{d}", .{job_id}); + defer alloc.free(job_id_text); + const prefix = try graph.graphMetricControlKeyAlloc(&.{ "rank", "job", job_id_text, "vector", "raw_rank", "0" }); + defer alloc.free(prefix); + try std.testing.expect(!try graph.hasKeysWithPrefixInBatch(&txn, prefix)); + try std.testing.expectEqualStrings("\x00", try txn.get(retirement_key)); +} + +test "graph metric vector chunks checkpoint across blocks preserve caller order and reject missing output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-vector-checkpoint"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-vector-checkpoint"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + defer graph.close(); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const scores = try temp.alloc(GraphIndex.GraphMetricScore, 300); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (scores, 0..) |*score, i| { + score.* = .{ .node = try std.fmt.allocPrint(temp, "node-{d:0>4}", .{i}), .score = @floatFromInt(i) }; + const key = try GraphIndex.graphMetricNodeSlotKey(temp, "rank", 1, score.node); + try GraphIndex.putU64(&batch, key, (@as(u64, 1) << 32) + i); + } + try graph.writeGraphMetricVector(&batch, "rank", 1, "rank", 0, scores[0..257], null, null); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.InvalidGraphMetricScore, graph.readGraphMetricVectorAlloc(&txn, "rank", 1, "rank", 0, &.{scores[299].node}, true)); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + // Replay a checkpoint, then fill the remaining slots of its last block. + try graph.writeGraphMetricVector(&batch, "rank", 1, "rank", 0, scores[0..257], null, null); + try graph.writeGraphMetricVector(&batch, "rank", 1, "rank", 0, scores[257..], null, null); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const values = try graph.readGraphMetricVectorAlloc(&txn, "rank", 1, "rank", 0, &.{ scores[299].node, scores[0].node, scores[256].node, scores[299].node }, true); + defer alloc.free(values); + try std.testing.expectEqualSlices(f64, &.{ 299, 0, 256, 299 }, values); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.writeGraphMetricVector(&batch, "rank", 1, "rank", 2, scores, null, 0); + try batch.commit(); + } + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.InvalidGraphMetricScore, graph.readGraphMetricVectorAlloc(&txn, "rank", 1, "rank", 0, &.{scores[0].node}, true)); + const fresh = try graph.readGraphMetricVectorAlloc(&txn, "rank", 1, "rank", 2, &.{scores[0].node}, true); + defer alloc.free(fresh); + try std.testing.expectEqual(@as(f64, 0), fresh[0]); +} + +test "graph metric vector chunks publish pagerank eigenvector and hits with sparse filtered ordinals" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-vector-chunks"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-vector-chunks"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 3 }, + .{ .name = "eigen", .kind = .eigenvector, .refresh = .manual, .max_iterations = 3 }, + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual, .max_iterations = 3 }, + .{ .name = "hub", .kind = .hits_hub, .refresh = .manual, .max_iterations = 3 }, + .{ .name = "degree", .kind = .degree, .refresh = .manual }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs, .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + defer graph.close(); + const nodes = [_][]const u8{ "doc-0000", "doc-0001", "doc-4096" }; + try graph.addEdge(nodes[0], nodes[1], "cites", 1, 0, 0, ""); + try graph.addEdge(nodes[1], nodes[2], "cites", 1, 0, 0, ""); + try graph.addEdge(nodes[2], nodes[1], "cites", 1, 0, 0, ""); + // The projection is sparse in a large immutable node dictionary. This + // exercises empty producer ranges and slots in distant vector chunks. + try installGraphMetricPlanningNodeRefsForTest(&graph, graph_metric_build_checkpoint_reduce_units + 1); + const edges = [_]metric_kernels.Edge{ .{ .source = 0, .target = 1 }, .{ .source = 1, .target = 2 }, .{ .source = 2, .target = 1 } }; + var rank = try metric_kernels.pageRankAlloc(alloc, nodes.len, &edges, .{ .max_iterations = 3 }); + defer rank.deinit(alloc); + var eigen = try metric_kernels.eigenvectorAlloc(alloc, nodes.len, &edges, .{ .max_iterations = 3 }); + defer eigen.deinit(alloc); + var hits = try metric_kernels.hitsAlloc(alloc, nodes.len, &edges, .{ .max_iterations = 3 }); + defer hits.deinit(alloc); + var degree = try metric_kernels.degreeAlloc(alloc, nodes.len, &edges, .{}); + defer degree.deinit(alloc); + { + var building = try graph.ensureGraphMetricPlannedBuild("rank", try graph.graphMetricCurrentGeneration("rank")); + building.deinit(alloc); + var reached_iteration = false; + for (0..2000) |_| { + _ = try graph.runGraphMetricPlannedWorkerPageStepForMetric("rank", "plan-worker"); + _ = try graph.runGraphMetricPlannedCoordinatorStepForMetric("rank"); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = (try graph.metricBuildJob(&txn, "rank")).?; + if (job.iteration != 1) continue; + const plan = try graph.graphMetricActivePlan(&txn, "rank", job.job_id); + try std.testing.expectEqual(@as(usize, 65), plan.count); + try std.testing.expectEqual(@as(u64, 3), plan.total()); + var active_count: usize = 0; + for (plan.counts[0..plan.count], 0..) |count, i| { + const id = GraphIndex.graphMetricBuildPhasePageIdBase(.pagerank, .reduce_ranks) + i; + const page = try graph.metricBuildPage(&txn, "rank", job.job_id, .reduce_ranks, 1, id); + const leaf = try graph.metricBuildPage(&txn, "rank", job.job_id, .reduce_ranks, 1, graph_metric_build_summary_leaf_base + i); + if (count == 0) { + try std.testing.expect(page == null and leaf == null); + const initial = (try graph.metricBuildPage(&txn, "rank", job.job_id, .reduce_ranks, 0, id)).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, initial.state); + try std.testing.expectEqual(@as(u32, 0), initial.attempt); + } else { + active_count += 1; + try std.testing.expectEqual(count, page.?.total_units); + try std.testing.expectEqual(count, leaf.?.total_units); + } + } + try std.testing.expectEqual(@as(usize, 2), active_count); + reached_iteration = true; + break; + } + try std.testing.expect(reached_iteration); + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs, .reverse_lsm_options = .{ .flush_threshold = 8192 } }); + } + for (configs, 0..) |cfg, index| { + // A compatible HITS pair shares the owner's one build. + if (cfg.kind != .hits_hub) { + var status = try graph.runGraphMetricPlannedDrain(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name), .{ .worker_ids = &.{ "chunk-a", "chunk-b" }, .max_steps = 10000 }); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + } + const expected = switch (index) { + 0 => rank.scores, + 1 => eigen.scores, + 2 => hits.authorities, + 3 => hits.hubs, + 4 => degree.scores, + else => unreachable, + }; + for (nodes, expected) |node, score| try std.testing.expectApproxEqAbs(score, (try graph.graphMetricScore(cfg.name, node)).?, 1e-12); + } + try graph.addEdge(nodes[0], nodes[2], "cites", 1, 0, 0, ""); + const rebuilt_edges = edges ++ [_]metric_kernels.Edge{.{ .source = 0, .target = 2 }}; + var warm = try metric_kernels.pageRankAlloc(alloc, nodes.len, &rebuilt_edges, .{ .max_iterations = 3, .initial_scores = rank.scores }); + defer warm.deinit(alloc); + var rebuilt = try graph.runGraphMetricPlannedDrain("rank", try graph.graphMetricCurrentGeneration("rank"), .{ .worker_ids = &.{ "warm-a", "warm-b" }, .max_steps = 10000 }); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + for (nodes, warm.scores) |node, score| try std.testing.expectApproxEqAbs(score, (try graph.graphMetricScore("rank", node)).?, 1e-12); +} + +test "graph degree large-build summary counts filtered materialization without coordinator scan" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-summary-count"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-summary-count"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-0000", "doc-0001", "cites", 1.0, 0, 0, ""); + try installGraphMetricPlanningNodeRefsForTest( + &graph, + graph_metric_build_target_reduce_page_units + 1, + ); + var building = try graph.ensureGraphMetricPlannedBuild("degree", try graph.graphMetricCurrentGeneration("degree")); + defer building.deinit(alloc); + + var job_txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..65) |i| { + const node = try std.fmt.allocPrint(alloc, "doc-{d:0>4}", .{i}); + defer alloc.free(node); + const key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", job.job_id, node, 1); + defer alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + try batch.commit(); + } + + const first_claim = try graph.claimNextGraphMetricBuildPageAt("degree", job.job_id, .reduce_ranks, 0, "degree-summary-a", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 0), first_claim.page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.summary, first_claim.range_kind); + try std.testing.expectEqual(@as(usize, 64), try graph.executeGraphMetricReduceSummaryBuildPageWithLimit("degree", metrics[0], job, first_claim, 64)); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var final_attempt = (try graph.metricBuildPage(&batch, "degree", job.job_id, .reduce_ranks, 0, 0)).?; + final_attempt.attempt = graph_metric_build_max_page_attempts; + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", final_attempt); + try batch.commit(); + } + try std.testing.expect((try graph.claimNextGraphMetricBuildPageAt("degree", job.job_id, .reduce_ranks, 0, "degree-summary-b", 1001)) == null); + + const renewed = try graph.claimNextGraphMetricBuildPageAt("degree", job.job_id, .reduce_ranks, 0, "degree-summary-a", 1001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(graph_metric_build_max_page_attempts, renewed.attempt); + try std.testing.expectEqual(@as(u64, 64), renewed.completed_units); + try std.testing.expectEqual(@as(usize, 1), try graph.executeGraphMetricReduceSummaryBuildPage("degree", metrics[0], job, renewed)); + try std.testing.expectEqual(@as(usize, 65), try graph.graphMetricBuildPlannedScoreCount("degree", metrics[0], job)); +} + +test "graph pagerank planned scan page writes durable out-degree intermediates" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-scan-executor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-scan-executor"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const edge_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &edge_types }, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-a", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-d", "related", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, GraphIndex.graphMetricBuildJobId("pagerank", active_job.target_generation, active_job.started_at_ms)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .prepare_generation, 0)); + + const scan = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + try std.testing.expect(scan.advanced_phase); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, job.phase); + + const doc_a_out_key = try graph.graphMetricBuildPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, "doc-a", 1); + defer alloc.free(doc_a_out_key); + try std.testing.expectEqual(@as(u64, 2), try GraphIndex.readU64OrZero(&txn, doc_a_out_key)); + const doc_b_out_key = try graph.graphMetricBuildPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, "doc-b", 1); + defer alloc.free(doc_b_out_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, doc_b_out_key)); + const doc_c_out_key = try graph.graphMetricBuildPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, "doc-c", 1); + defer alloc.free(doc_c_out_key); + try std.testing.expectEqual(@as(u64, 0), try GraphIndex.readU64OrZero(&txn, doc_c_out_key)); + + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, node, 1); + defer alloc.free(node_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, node_key)); + } + const filtered_node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, "doc-d", 1); + defer alloc.free(filtered_node_key); + try std.testing.expectEqual(@as(u64, 0), try GraphIndex.readU64OrZero(&txn, filtered_node_key)); + } + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{.initialize_ranks}); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, job.phase); + + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 0, node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0 / 3.0), try rank_key.read(&txn), 0.0000001); + } + const filtered_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 0, "doc-d"); + try std.testing.expectEqual(@as(f64, 0.0), try filtered_rank_key.read(&txn)); + + const doc_a_out_total_key = try graph.topologyKey(&txn, "pagerank", active_job.job_id, try graph.graphMetricBuildPageRankOutDegreeKeyAlloc("pagerank", active_job.job_id, "doc-a")); + defer alloc.free(doc_a_out_total_key); + try std.testing.expectEqual(@as(u64, 2), try GraphIndex.readU64OrZero(&txn, doc_a_out_total_key)); + const doc_b_out_total_key = try graph.topologyKey(&txn, "pagerank", active_job.job_id, try graph.graphMetricBuildPageRankOutDegreeKeyAlloc("pagerank", active_job.job_id, "doc-b")); + defer alloc.free(doc_b_out_total_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, doc_b_out_total_key)); + const doc_c_out_total_key = try graph.topologyKey(&txn, "pagerank", active_job.job_id, try graph.graphMetricBuildPageRankOutDegreeKeyAlloc("pagerank", active_job.job_id, "doc-c")); + defer alloc.free(doc_c_out_total_key); + try std.testing.expectEqual(@as(u64, 0), try GraphIndex.readU64OrZero(&txn, doc_c_out_total_key)); + } + + const contributions = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, contributions.phase); + try std.testing.expect(contributions.claimed_page); + try std.testing.expect(contributions.completed_page); + try std.testing.expect(contributions.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + + try std.testing.expectApproxEqAbs(@as(f64, 0.85 * (1.0 / 3.0) / 2.0), try graphMetricOrdinalValueForTest(&graph, &txn, "pagerank", active_job.job_id, .iterate_contributions, 0, "doc-b"), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.85 * (1.0 / 3.0) / 2.0 + 0.85 * (1.0 / 3.0)), try graphMetricOrdinalValueForTest(&graph, &txn, "pagerank", active_job.job_id, .iterate_contributions, 0, "doc-c"), 0.0000001); + try std.testing.expectEqual(@as(f64, 0.0), try graphMetricOrdinalValueForTest(&graph, &txn, "pagerank", active_job.job_id, .iterate_contributions, 0, "doc-d")); + } + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{.reduce_ranks}); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + // Numerical progress counts only the three sealed active nodes; + // the excluded fourth node is not a unit of numerical work. + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + + const doc_a_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-a"); + try std.testing.expectApproxEqAbs(@as(f64, 0.14444444444444446), try doc_a_rank_key.read(&txn), 0.0000001); + const doc_b_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-b"); + try std.testing.expectApproxEqAbs(@as(f64, 0.2861111111111111), try doc_b_rank_key.read(&txn), 0.0000001); + const doc_c_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-c"); + try std.testing.expectApproxEqAbs(@as(f64, 0.5694444444444444), try doc_c_rank_key.read(&txn), 0.0000001); + const doc_d_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-d"); + try std.testing.expectEqual(@as(f64, 0.0), try doc_d_rank_key.read(&txn)); + } + + const convergence = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, convergence.phase); + try std.testing.expect(convergence.claimed_page); + try std.testing.expect(convergence.completed_page); + try std.testing.expect(convergence.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + try std.testing.expectApproxEqAbs(@as(f64, 0.2361111111111111), page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.4722222222222222), page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), page.rank_sum, 0.0000001); + try std.testing.expect(!page.converged); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + try std.testing.expect((try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 1, 3)) == null); + const next_reduce = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, next_reduce.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, next_reduce.range_kind); + const next_check = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 1, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, next_check.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, next_check.range_kind); + + const phase_summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .check_convergence, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, phase_summary.state); + try std.testing.expectApproxEqAbs(@as(f64, 0.2361111111111111), phase_summary.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.4722222222222222), phase_summary.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), phase_summary.rank_sum, 0.0000001); + try std.testing.expect(!phase_summary.converged); + const iteration_summary = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(!iteration_summary.converged); + try std.testing.expect(!iteration_summary.fixed_iteration_limit); + } +} + +test "graph pagerank reclaimed scan page overwrites stale partial output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-scan-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-scan-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-a", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + const prepare = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-setup"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(prepare.advanced_phase); + + const partial_scan_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executePageRankScanBuildPageWithLimit("pagerank", metrics[0], active_job, partial_scan_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "pagerank", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + const attempt_out_degree_key = try graph.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-a"); + defer alloc.free(attempt_out_degree_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_out_degree_key)); + const node_b_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, "doc-b", 1); + defer alloc.free(node_b_key); + try std.testing.expectError(error.NotFound, txn.get(node_b_key)); + const attempt_node_b_key = try graph.graphMetricBuildAttemptPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-b"); + defer alloc.free(attempt_node_b_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_node_b_key)); + } + + const scan_expires_at = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + break :blk page.lease_expires_at_ms; + }; + const reclaimed_scan = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", scan_expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed_scan.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed_scan.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed_scan.completed_units); + try std.testing.expectEqualStrings("", reclaimed_scan.cursor); + { + var stale_out_degrees = std.StringHashMapUnmanaged(u64).empty; + defer { + var key_it = stale_out_degrees.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_out_degrees.deinit(alloc); + } + var stale_nodes = std.StringHashMapUnmanaged(void).empty; + defer { + var key_it = stale_nodes.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_nodes.deinit(alloc); + } + try stale_out_degrees.put(alloc, try alloc.dupe(u8, "doc-a"), 99); + try stale_nodes.put(alloc, try alloc.dupe(u8, "doc-stale"), {}); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writePageRankScanPartialsForAttempt("pagerank", active_job, partial_scan_claim, 0, &stale_out_degrees, &stale_nodes, partial_scan_claim.worker_id, "", 0, partial_scan_claim.total_units)); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const stale_node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, "doc-stale", 1); + defer alloc.free(stale_node_key); + try std.testing.expectError(error.NotFound, txn.get(stale_node_key)); + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "pagerank", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + } + + _ = try graph.executePageRankScanBuildPageWithLimit("pagerank", metrics[0], active_job, reclaimed_scan, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "pagerank", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 2), out_degree); + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("pagerank", active_job.job_id, node, 1); + defer alloc.free(node_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, node_key)); + } + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, job.phase); + } +} + +test "graph pagerank scan adoption maintains one idempotent out-degree total" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-scan-total"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-scan-total"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metric_configs = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metric_configs }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + _ = try graph.runGraphMetricPlannedWorkerStep("pagerank", metric_configs[0], "worker-setup"); + const page = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse + return error.TestExpectedGraphMetricBuildPage; + + const attempt_key = try graph.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, page.page_id, page.attempt, "doc-a"); + defer alloc.free(attempt_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putU64(&batch, attempt_key, 2); + try batch.commit(); + } + _ = try graph.adoptGraphMetricAttemptOutputPage("pagerank", .pagerank, active_job.job_id, page); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(@as(u64, 2), try graph.aggregatePageRankOutDegreeForNode(&txn, "pagerank", active_job.job_id, "doc-a")); + } + + // Re-adopting replacement output for the same page adjusts by the delta; + // it must not add the whole page again after a lease retry. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putU64(&batch, attempt_key, 3); + try batch.commit(); + } + _ = try graph.adoptGraphMetricAttemptOutputPage("pagerank", .pagerank, active_job.job_id, page); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(@as(u64, 3), try graph.aggregatePageRankOutDegreeForNode(&txn, "pagerank", active_job.job_id, "doc-a")); + const ledger_key = try graph.graphMetricBuildPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, "doc-a", page.page_id); + defer alloc.free(ledger_key); + try std.testing.expectEqual(@as(u64, 3), try GraphIndex.readU64OrZero(&txn, ledger_key)); + } +} + +test "graph pagerank reclaimed initialize page overwrites stale rank output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-initialize-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-initialize-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-setup", &.{ .prepare_generation, .scan_edges_and_out_degree }); + + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .initialize_ranks, 0); + const initial_claim = try graph.claimNextGraphMetricBuildPageAt("pagerank", active_job.job_id, .initialize_ranks, 0, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, initial_claim.state); + try std.testing.expectEqual(@as(u64, 1), initial_claim.attempt); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + // Initialization owns vectors, not the immutable scan totals. + const rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 0, node); + try rank_key.write(&batch, 42.0); + } + try batch.commit(); + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt( + "pagerank", + active_job.job_id, + .initialize_ranks, + 0, + initial_claim.page_id, + "worker-b", + initial_claim.lease_expires_at_ms + 1, + ) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + const stale_initialized = [_]GraphIndex.PageRankInitializeNode{.{ + .node = "doc-a", + .out_degree = 123, + }}; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writePageRankInitializeOutputForAttempt("pagerank", active_job, initial_claim, &stale_initialized, 99.0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const out_degree_key = try graph.topologyKey(&txn, "pagerank", active_job.job_id, try graph.graphMetricBuildPageRankOutDegreeKeyAlloc("pagerank", active_job.job_id, "doc-a")); + defer alloc.free(out_degree_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, out_degree_key)); + const rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 0, "doc-a"); + try std.testing.expectApproxEqAbs(@as(f64, 42.0), try rank_key.read(&txn), 0.0000001); + } + + _ = try graph.executePageRankInitializeBuildPage("pagerank", active_job, reclaimed); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .initialize_ranks, 0)); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .initialize_ranks, 0, initial_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const expected_rank = 1.0 / 3.0; + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const out_degree_key = try graph.topologyKey(&txn, "pagerank", active_job.job_id, try graph.graphMetricBuildPageRankOutDegreeKeyAlloc("pagerank", active_job.job_id, node)); + defer alloc.free(out_degree_key); + const out_degree = try GraphIndex.readU64OrZero(&txn, out_degree_key); + if (std.mem.eql(u8, node, "doc-b")) { + try std.testing.expectEqual(@as(u64, 0), out_degree); + } else { + try std.testing.expectEqual(@as(u64, 1), out_degree); + } + const rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 0, node); + try std.testing.expectApproxEqAbs(expected_rank, try rank_key.read(&txn), 0.0000001); + } + + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, job.phase); + } +} + +test "graph pagerank contribution and reduce pages resume from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-contribution-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-contribution-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + + const first_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .iterate_contributions, 0, 3, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, first_claim.state); + _ = try graph.executePageRankContributionBuildPageWithLimit("pagerank", metrics[0], active_job, first_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, job.phase); + try std.testing.expectEqualStrings("", job.cursor); + + const adopted_contribution = try graphMetricOrdinalValueForTest(&graph, &txn, "pagerank", active_job.job_id, .iterate_contributions, 0, "doc-b"); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), adopted_contribution, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.85 * (1.0 / 3.0)), try graph.ordinalAttemptContributionForNodeForTest(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3, first_claim.attempt, "doc-b"), 0.0000001); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + const renewed = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .iterate_contributions, 0, 3, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed.state); + try std.testing.expectEqual(@as(u64, 1), renewed.completed_units); + try std.testing.expect(renewed.cursor.len > 0); + + _ = try graph.executePageRankContributionBuildPageWithLimit("pagerank", metrics[0], active_job, renewed, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .iterate_contributions, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + + try std.testing.expectApproxEqAbs(@as(f64, 2.0 * 0.85 * (1.0 / 3.0)), try graphMetricOrdinalValueForTest(&graph, &txn, "pagerank", active_job.job_id, .iterate_contributions, 0, "doc-b"), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 2.0 * 0.85 / 3.0), try graph.ordinalAttemptContributionForNodeForTest(&txn, "pagerank", active_job.job_id, .iterate_contributions, 0, 3, first_claim.attempt, "doc-b"), 0.0000001); + } + + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 0); + const reduce_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 0, 4, "worker-r", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reduce_claim.state); + _ = try graph.executePageRankReduceBuildPageWithLimit("pagerank", metrics[0], active_job, reduce_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqualStrings("", job.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 0); + const renewed_reduce = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 0, 4, "worker-r", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_reduce.state); + try std.testing.expectEqual(@as(u64, 1), renewed_reduce.completed_units); + try std.testing.expectEqualStrings("", renewed_reduce.cursor); + + _ = try graph.executePageRankReduceBuildPageWithLimit("pagerank", metrics[0], active_job, renewed_reduce, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .reduce_ranks, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + + const doc_a_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-a"); + try std.testing.expectApproxEqAbs(@as(f64, 0.14444444444444446), try doc_a_rank_key.read(&txn), 0.0000001); + const doc_b_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-b"); + try std.testing.expectApproxEqAbs(@as(f64, 0.7111111111111111), try doc_b_rank_key.read(&txn), 0.0000001); + const doc_c_rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 1, "doc-c"); + try std.testing.expectApproxEqAbs(@as(f64, 0.14444444444444446), try doc_c_rank_key.read(&txn), 0.0000001); + } + + const check_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-c", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, check_claim.state); + const partial_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, check_claim, 1); + try std.testing.expect(!partial_check.completed_page); + try std.testing.expectEqual(@as(u64, 1), partial_check.completed_units); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + try std.testing.expect(page.max_delta > 0.0); + try std.testing.expect(page.total_delta > 0.0); + try std.testing.expect(page.rank_sum > 0.0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + try std.testing.expectEqualStrings("", job.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqualStrings("worker-c", page.worker_id); + } + const renewed_check = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-c", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_check.state); + try std.testing.expectEqual(@as(u64, 1), renewed_check.completed_units); + try std.testing.expectEqualStrings("", renewed_check.cursor); + + const completed_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, renewed_check, null); + try std.testing.expect(completed_check.completed_page); + try std.testing.expectApproxEqAbs(@as(f64, 0.37777777777777777), completed_check.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.7555555555555555), completed_check.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), completed_check.rank_sum, 0.0000001); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + try std.testing.expectApproxEqAbs(@as(f64, 0.37777777777777777), page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.7555555555555555), page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), page.rank_sum, 0.0000001); + try std.testing.expect(!page.converged); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + const iteration_summary = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(!iteration_summary.converged); + try std.testing.expect(iteration_summary.fixed_iteration_limit); + } +} + +test "graph pagerank later iteration pages resume from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-later-iteration-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-later-iteration-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, page.state); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .iterate_contributions, 1, 3)) == null); + const job = (try graph.metricBuildJob(&txn, "pagerank")).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + } + + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 1); + const reduce_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 1, 4, "worker-reduce", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reduce_claim.state); + _ = try graph.executePageRankReduceBuildPageWithLimit("pagerank", metrics[0], active_job, reduce_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqualStrings("", job.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 1); + const renewed_reduce = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 1, 4, "worker-reduce", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_reduce.state); + try std.testing.expectEqual(@as(u64, 1), renewed_reduce.completed_units); + try std.testing.expectEqualStrings("", renewed_reduce.cursor); + _ = try graph.executePageRankReduceBuildPageWithLimit("pagerank", metrics[0], active_job, renewed_reduce, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .reduce_ranks, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const rank_key = graphMetricVectorSlotForTest(&graph, "pagerank", active_job.job_id, "rank", 2, "doc-b"); + try std.testing.expect((try rank_key.read(&txn)) > 0.0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + const check_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 1, 5, "worker-check", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, check_claim.state); + const partial_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, check_claim, 1); + try std.testing.expect(!partial_check.completed_page); + try std.testing.expectEqual(@as(u64, 1), partial_check.completed_units); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 1, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + try std.testing.expect(page.rank_sum > 0.0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqualStrings("", job.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + const renewed_check = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 1, 5, "worker-check", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_check.state); + try std.testing.expectEqual(@as(u64, 1), renewed_check.completed_units); + try std.testing.expectEqualStrings("", renewed_check.cursor); + const completed_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, renewed_check, null); + try std.testing.expect(completed_check.completed_page); + try std.testing.expect(completed_check.rank_sum > 0.0); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 1, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 3), page.completed_units); + try std.testing.expect(page.output_fingerprint != 0); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + const iteration_summary = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 1) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(!iteration_summary.converged); + try std.testing.expect(iteration_summary.fixed_iteration_limit); + } +} + +test "graph pagerank dynamic iteration planning updates manifest page count idempotently" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-dynamic-manifest-count"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-dynamic-manifest-count"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + const initial_manifest = try graph.metricBuildManifest(&job_txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + const initial_page_count = initial_manifest.page_count; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(initial_page_count + 5, manifest.page_count); + } + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.planPageRankIterationPagesInBatch(&batch, "pagerank", active_job, 1); + try batch.commit(); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(initial_page_count + 5, manifest.page_count); + } +} + +test "graph pagerank later iteration failed pages retry and advance" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-later-iteration-retry"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-later-iteration-retry"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 1); + const reduce_failed_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), reduce_failed_claim.attempt); + const reduce_failed = try graph.failGraphMetricBuildPage("pagerank", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", "later reduce failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, reduce_failed.state); + try std.testing.expectEqualStrings("later reduce failed", reduce_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .reduce_ranks, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 1); + const reduce_retry = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 1, 4, "worker-retry-reduce", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reduce_retry.state); + try std.testing.expectEqual(@as(u64, 2), reduce_retry.attempt); + try std.testing.expectEqualStrings("", reduce_retry.last_error); + _ = try graph.executePageRankReduceBuildPageWithLimit("pagerank", metrics[0], active_job, reduce_retry, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .reduce_ranks, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 0), summary.failed_pages); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } + + const check_failed_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 1, 5, "worker-fail-check", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), check_failed_claim.attempt); + const check_failed = try graph.failGraphMetricBuildPage("pagerank", active_job.job_id, .check_convergence, 1, 5, "worker-fail-check", "later check failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, check_failed.state); + try std.testing.expectEqualStrings("later check failed", check_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + const check_retry = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 1, 5, "worker-retry-check", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, check_retry.state); + try std.testing.expectEqual(@as(u64, 2), check_retry.attempt); + try std.testing.expectEqualStrings("", check_retry.last_error); + const completed_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, check_retry, null); + try std.testing.expect(completed_check.completed_page); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 0), summary.failed_pages); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + const iteration_summary = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 1) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(iteration_summary.fixed_iteration_limit); + } +} + +test "graph pagerank later iteration exhausted page fails build and preserves prior generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-later-iteration-exhausted-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-later-iteration-exhausted-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runPageRankMetricPlanned("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 3), before_failure_top.len); + + try graph.addEdge("doc-d", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("pagerank", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + _ = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + } + + try drainGraphMetricSummaryForTest(&graph, "pagerank", active_job, .reduce_ranks, 1); + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const page = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 1, 4, worker_id, 2000 + attempt) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(attempt + 1, page.attempt); + _ = try graph.failGraphMetricBuildPage("pagerank", active_job.job_id, .reduce_ranks, 1, 4, worker_id, "retryable later reduction failure"); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, page.state); + try std.testing.expectEqual(@as(u64, graph_metric_build_max_page_attempts), page.attempt); + try std.testing.expectEqualStrings("retryable later reduction failure", page.last_error); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_status = try graph.graphMetricStatus("pagerank"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_status.state); + try std.testing.expectEqual(published_generation, failed_status.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_status.build_job_id); + const exhaustion_reason = "GraphMetricBuildPageAttemptsExhausted: phase=reduce_ranks, iteration=1, page_id=4, attempt=3, cause=retryable later reduction failure"; + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_status.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_status.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_status.recent_failures[0].phase); + try std.testing.expectEqual(@as(u32, 1), failed_status.recent_failures[0].iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.recent_failures[0].last_error); + + const after_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-d")); + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_job.phase); + try std.testing.expectEqual(@as(u32, 1), failed_job.iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("pagerank", rebuilding_generation)); + } +} + +test "graph pagerank convergence page reclaim recomputes without stale partial summary" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-convergence-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-convergence-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks }); + + const check_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-a", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + const partial_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, check_claim, 1); + try std.testing.expect(!partial_check.completed_page); + try std.testing.expect(partial_check.max_delta > 0.0); + try std.testing.expect(partial_check.total_delta > 0.0); + try std.testing.expect(partial_check.rank_sum > 0.0); + + var expires_at: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.max_delta > 0.0); + try std.testing.expect(page.total_delta > 0.0); + try std.testing.expect(page.rank_sum > 0.0); + expires_at = page.lease_expires_at_ms; + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-b", expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.rank_sum, 0.0); + try std.testing.expect(!reclaimed.converged); + + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.updateGraphMetricBuildConvergencePageProgressForAttempt("pagerank", active_job.job_id, 0, 5, check_claim.worker_id, check_claim.attempt, "stale-check", 1, check_claim.total_units, 99.0, 99.0, 99.0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.rank_sum, 0.0); + } + + const completed_check = try graph.executePageRankConvergenceBuildPageWithLimit("pagerank", metrics[0], active_job, reclaimed, null); + try std.testing.expect(completed_check.completed_page); + try std.testing.expectApproxEqAbs(@as(f64, 0.37777777777777777), completed_check.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.7555555555555555), completed_check.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), completed_check.rank_sum, 0.0000001); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectApproxEqAbs(@as(f64, 0.37777777777777777), page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.7555555555555555), page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), page.rank_sum, 0.0000001); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } +} + +fn graphMetricOrdinalValueForTest(graph: *GraphIndex, txn: anytype, metric: []const u8, job_id: u64, phase: GraphIndex.GraphMetricBuildPhase, iteration: u32, node: []const u8) !f64 { + const slot_key = try graph.topologyKey(txn, metric, job_id, try GraphIndex.graphMetricNodeSlotKey(graph.alloc, metric, job_id, node)); + defer graph.alloc.free(slot_key); + if (try GraphIndex.readU64OrZero(txn, slot_key) == 0) return 0; + const values = graph.ordinalContributionsForNodesAlloc(txn, metric, job_id, phase, iteration, &.{node}) catch |err| switch (err) { + error.GraphMetricBuildPhaseNotComplete => return 0, + else => return err, + }; + defer graph.alloc.free(values); + return values[0]; +} + +const GraphMetricVectorSlotForTest = struct { + graph: *GraphIndex, + metric: []const u8, + job_id: u64, + lane: []const u8, + iteration: u32, + node: []const u8, + + fn read(self: @This(), txn: anytype) !f64 { + const key = try self.graph.topologyKey(txn, self.metric, self.job_id, try GraphIndex.graphMetricNodeSlotKey(self.graph.alloc, self.metric, self.job_id, self.node)); + defer self.graph.alloc.free(key); + const slot = try GraphIndex.readU64OrZero(txn, key); + if (slot == 0) return 0; + const values = try self.graph.readGraphMetricVectorSlotsAlloc(txn, self.metric, self.job_id, self.lane, self.iteration, &.{slot}, false); + defer self.graph.alloc.free(values); + return values[0]; + } + + fn exists(self: @This(), txn: anytype) !bool { + const values = self.graph.readGraphMetricVectorAlloc(txn, self.metric, self.job_id, self.lane, self.iteration, &.{self.node}, true) catch |err| switch (err) { + error.InvalidGraphMetricScore => return false, + else => return err, + }; + defer self.graph.alloc.free(values); + return true; + } + + fn write(self: @This(), batch: anytype, value: f64) !void { + try self.graph.writeGraphMetricVector(batch, self.metric, self.job_id, self.lane, self.iteration, &.{.{ .node = self.node, .score = value }}, null, null); + } +}; + +fn graphMetricVectorSlotForTest(graph: *GraphIndex, metric: []const u8, job_id: u64, lane: []const u8, iteration: u32, node: []const u8) GraphMetricVectorSlotForTest { + return .{ .graph = graph, .metric = metric, .job_id = job_id, .lane = lane, .iteration = iteration, .node = node }; +} + +fn graphMetricVectorKeyForNodeForTest(graph: *GraphIndex, metric: []const u8, job_id: u64, lane: []const u8, iteration: u32, node: []const u8) ![]u8 { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const slots = try graph.graphMetricNodeSlotsAlloc(&txn, metric, job_id, &.{node}); + defer graph.alloc.free(slots); + return GraphIndex.graphMetricVectorChunkKey(graph.alloc, metric, job_id, lane, iteration, slots[0] / vector_chunk.entries); +} + +fn drainGraphMetricSummaryForTest(graph: *GraphIndex, metric: []const u8, job: GraphIndex.GraphMetricBuildJob, phase: GraphIndex.GraphMetricBuildPhase, iteration: u32) !void { + const cfg = graph.metricConfig(metric) orelse return error.MetricNotReady; + const count = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = (try graph.metricBuildManifest(&txn, metric, job.job_id)).?; + break :blk graph.graphMetricDegreeReducePageCount(@intCast(manifest.node_count)); + }; + for (0..count + 1) |i| { + const id: u64 = if (i == count) 0 else graph_metric_build_summary_leaf_base + i; + const claim = (try graph.claimGraphMetricBuildPageAt(metric, job.job_id, phase, iteration, id, "summary", 1000)) orelse continue; + var complete = false; + for (0..4096) |_| { + _ = try graph.executeGraphMetricReduceSummaryBuildPage(metric, cfg, job, claim); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + if ((try graph.metricBuildPage(&txn, metric, job.job_id, phase, iteration, id)).?.state == .complete) { + complete = true; + break; + } + } + try std.testing.expect(complete); + } +} + +fn claimGraphMetricDataPageForTest(graph: *GraphIndex, cfg: GraphMetricConfig, job: GraphIndex.GraphMetricBuildJob, phase: GraphIndex.GraphMetricBuildPhase) !GraphIndex.GraphMetricBuildPage { + for (0..1024) |_| { + const page = try graph.claimNextGraphMetricBuildPageAt(cfg.name, job.job_id, phase, 0, "worker-a", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + if (page.range_kind != .summary) return page; + _ = try graph.executeGraphMetricReduceSummaryBuildPage(cfg.name, cfg, job, page); + } + return error.TestGraphMetricBuildDidNotAdvance; +} + +fn expectGraphMetricOrdinalTakeoverForTest(kind: GraphMetricKind) !void { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-ordinal-takeover"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-ordinal-takeover"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const cfg = GraphMetricConfig{ .name = "metric", .kind = kind, .refresh = .manual, .max_iterations = 1, .tolerance = 0.000001 }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &.{cfg} }); + defer graph.close(); + try graph.addEdge("doc-a", "doc-b", "cites", 1, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1, 0, 0, ""); + try graph.acquireGraphMetricBuildLease(cfg.name, try graph.graphMetricCurrentGeneration(cfg.name)); + defer graph.releaseGraphMetricBuildLease(cfg.name) catch {}; + const job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk (try graph.metricBuildJob(&txn, cfg.name)).?; + }; + try drainGraphMetricBuildToPublishForTest(&graph, cfg.name, cfg, "setup", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + const hits = kind == .hits_authority or kind == .hits_hub; + const phases: []const GraphIndex.GraphMetricBuildPhase = if (hits) &.{ .iterate_contributions, .hits_hub_contributions } else &.{.iterate_contributions}; + for (phases) |phase| { + const first = (try graph.claimNextGraphMetricBuildPageAt(cfg.name, job.job_id, phase, 0, "first", 2000)).?; + _ = try graph.executeOrdinalContributionPage(cfg.name, cfg, job, first, 1); + const expires = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const current = (try graph.metricBuildPage(&txn, cfg.name, job.job_id, phase, 0, first.page_id)).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, current.state); + try std.testing.expectEqual(@as(u64, 1), current.completed_units); + // Readers cannot consume a producer before its winning attempt commits. + try std.testing.expectError(error.GraphMetricBuildPhaseNotComplete, graph.ordinalContributionsForNodesAlloc(&txn, cfg.name, job.job_id, phase, 0, &.{ "doc-a", "doc-b", "doc-c" })); + break :blk current.lease_expires_at_ms; + }; + const winner = (try graph.claimGraphMetricBuildPageAt(cfg.name, job.job_id, phase, 0, first.page_id, "winner", expires + 1)).?; + try std.testing.expectEqual(@as(u64, 2), winner.attempt); + try std.testing.expectEqual(@as(u64, 0), winner.completed_units); + try std.testing.expectEqualStrings("", winner.cursor); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executeOrdinalContributionPage(cfg.name, cfg, job, first, null)); + // Replacement changes checkpoint boundaries. The abandoned shard remains + // physically present but must never enter the winning vector. + _ = try graph.executeOrdinalContributionPage(cfg.name, cfg, job, winner, null); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const raw = try graph.ordinalContributionsForNodesAlloc(&txn, cfg.name, job.job_id, phase, 0, &.{ "doc-a", "doc-b", "doc-c" }); + defer alloc.free(raw); + const expected: []const f64 = if (phase == .hits_hub_contributions) &.{ 1, 0, 1 } else if (kind == .pagerank) &.{ 0, 0.5666666666666667, 0 } else &.{ 0, 2.0 / @sqrt(@as(f64, 3)), 0 }; + for (expected, raw) |want, actual| try std.testing.expectApproxEqAbs(want, actual, 0.0000001); + } + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady(cfg.name, job.job_id, phase, 0)); + const reduce_phase: GraphIndex.GraphMetricBuildPhase = if (phase == .hits_hub_contributions) .hits_hub_reduce_ranks else .reduce_ranks; + const reducer = try claimGraphMetricDataPageForTest(&graph, cfg, job, reduce_phase); + const lane: []const u8 = if (phase == .hits_hub_contributions) "hub" else if (hits) "authority" else "rank"; + if (phase == .hits_hub_contributions) { + _ = try graph.executeHitsHubReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, 2); + } else switch (kind) { + .pagerank => { + _ = try graph.executePageRankReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, 2); + }, + .eigenvector => { + _ = try graph.executeEigenvectorReduceBuildPageWithLimit(cfg.name, job, reducer, 2); + }, + .hits_authority, .hits_hub => { + _ = try graph.executeHitsReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, 2); + }, + else => unreachable, + } + // Simulate a stale partial output in the actual vector storage. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.writeGraphMetricVector(&batch, cfg.name, job.job_id, lane, 1, &.{.{ .node = "doc-b", .score = 42 }}, null, null); + try batch.commit(); + } + const reduce_expires = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const current = (try graph.metricBuildPage(&txn, cfg.name, job.job_id, reduce_phase, 0, reducer.page_id)).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, current.state); + try std.testing.expectEqual(@as(u64, 2), current.completed_units); + break :blk current.lease_expires_at_ms; + }; + const replacement = (try graph.claimGraphMetricBuildPageAt(cfg.name, job.job_id, reduce_phase, 0, reducer.page_id, "replacement", reduce_expires + 1)).?; + try std.testing.expectEqual(@as(u64, 2), replacement.attempt); + try std.testing.expectEqual(@as(u64, 0), replacement.completed_units); + if (phase == .hits_hub_contributions) { + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executeHitsHubReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, null)); + _ = try graph.executeHitsHubReduceBuildPageWithLimit(cfg.name, cfg, job, replacement, null); + } else switch (kind) { + .pagerank => { + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executePageRankReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, null)); + _ = try graph.executePageRankReduceBuildPageWithLimit(cfg.name, cfg, job, replacement, null); + }, + .eigenvector => { + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executeEigenvectorReduceBuildPageWithLimit(cfg.name, job, reducer, null)); + _ = try graph.executeEigenvectorReduceBuildPageWithLimit(cfg.name, job, replacement, null); + }, + .hits_authority, .hits_hub => { + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.executeHitsReduceBuildPageWithLimit(cfg.name, cfg, job, reducer, null)); + _ = try graph.executeHitsReduceBuildPageWithLimit(cfg.name, cfg, job, replacement, null); + }, + else => unreachable, + } + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady(cfg.name, job.job_id, reduce_phase, 0)); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const ranks = try graph.readGraphMetricVectorAlloc(&txn, cfg.name, job.job_id, lane, 1, &.{ "doc-a", "doc-b", "doc-c" }, true); + defer alloc.free(ranks); + const expected: []const f64 = if (phase == .hits_hub_contributions) &.{ 1.0 / @sqrt(@as(f64, 2)), 0, 1.0 / @sqrt(@as(f64, 2)) } else if (kind == .pagerank) &.{ 0.14444444444444443, 0.7111111111111111, 0.14444444444444443 } else &.{ 0, 1, 0 }; + for (expected, ranks) |want, actual| try std.testing.expectApproxEqAbs(want, actual, 0.0000001); + const summary = (try graph.metricBuildPhaseSummary(&txn, cfg.name, job.job_id, reduce_phase, 0)).?; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + } +} + +test "graph pagerank reclaimed contribution and reduce pages overwrite partial output" { + try expectGraphMetricOrdinalTakeoverForTest(.pagerank); +} + +test "graph pagerank planned fixed-iteration publish materializes rank scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-publish"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-publish"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, GraphIndex.graphMetricBuildJobId("pagerank", active_job.target_generation, active_job.started_at_ms)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .prepare_generation, 0)); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + + const publish = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + } + + var cleanup_step: usize = 0; + while (cleanup_step < 3) : (cleanup_step += 1) { + const cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(cleanup.claimed_page); + try std.testing.expect(cleanup.completed_page); + try std.testing.expectEqual(cleanup_step == 2, cleanup.published); + } + + var status = try graph.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(@as(u32, 1), status.iterations_completed); + try std.testing.expect(!status.converged); + try std.testing.expectApproxEqAbs(@as(f64, 0.425), status.delta, 0.0000001); + + const top = try graph.graphMetricTopK("pagerank", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("doc-b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 0.7125), top[0].score, 0.0000001); + try std.testing.expectEqualStrings("doc-a", top[1].node); + try std.testing.expectApproxEqAbs(@as(f64, 0.2875), top[1].score, 0.0000001); +} + +test "graph pagerank planned dynamic iteration reaches fixed-limit publish" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-dynamic-publish"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-dynamic-publish"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer graph.releaseGraphMetricBuildLease("pagerank") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, GraphIndex.graphMetricBuildJobId("pagerank", active_job.target_generation, active_job.started_at_ms)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .prepare_generation, 0)); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + const iteration_zero = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(!iteration_zero.converged); + try std.testing.expect(!iteration_zero.fixed_iteration_limit); + } + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + const iteration_one = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 1) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(!iteration_one.converged); + try std.testing.expect(iteration_one.fixed_iteration_limit); + } + + const publish = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + var cleanup_step: usize = 0; + while (cleanup_step < 3) : (cleanup_step += 1) { + const cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(cleanup.claimed_page); + try std.testing.expect(cleanup.completed_page); + try std.testing.expectEqual(cleanup_step == 2, cleanup.published); + } + + var status = try graph.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(@as(u32, 2), status.iterations_completed); + try std.testing.expect(!status.converged); + try std.testing.expectApproxEqAbs(@as(f64, 0.180625), status.delta, 0.0000001); + + const top = try graph.graphMetricTopK("pagerank", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("doc-b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 0.6221875), top[0].score, 0.0000001); + try std.testing.expectEqualStrings("doc-a", top[1].node); + try std.testing.expectApproxEqAbs(@as(f64, 0.3778125), top[1].score, 0.0000001); +} + +test "graph pagerank planned cleanup resumes after non-final cleanup page reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-cleanup-reopen"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-cleanup-reopen"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + const publish = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const visible_before_cleanup = try graph.graphMetricTopK("pagerank", 2); + defer { + for (visible_before_cleanup) |*score| score.deinit(alloc); + alloc.free(visible_before_cleanup); + } + try std.testing.expectEqual(@as(usize, 2), visible_before_cleanup.len); + try std.testing.expectEqualStrings("doc-b", visible_before_cleanup[0].node); + + const abandoned_attempt_key = try graph.graphMetricBuildAttemptPageRankContributionKeyAlloc("pagerank", active_job.job_id, .iterate_contributions, 0, 999, 1, "doc-abandoned"); + defer alloc.free(abandoned_attempt_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putF64(&batch, abandoned_attempt_key, 123.0); + try batch.commit(); + } + + const leased_first_cleanup = try graph.claimGraphMetricBuildPageAt( + "pagerank", + active_job.job_id, + .cleanup_old_generations, + 0, + 0, + "worker-a", + 5000, + ) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, leased_first_cleanup.state); + try std.testing.expect((try graph.claimGraphMetricBuildPageAt( + "pagerank", + active_job.job_id, + .cleanup_old_generations, + 0, + 2, + "worker-final", + 5000, + )) == null); + + const first_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, first_cleanup.phase); + try std.testing.expect(first_cleanup.claimed_page); + try std.testing.expect(first_cleanup.completed_page); + try std.testing.expect(!first_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + const cleanup_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, first_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, cleanup_page.state); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) != null); + _ = try txn.get(abandoned_attempt_key); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + const visible_after_reopen = try graph.graphMetricTopK("pagerank", 2); + defer { + for (visible_after_reopen) |*score| score.deinit(alloc); + alloc.free(visible_after_reopen); + } + try std.testing.expectEqual(@as(usize, 2), visible_after_reopen.len); + try std.testing.expectEqualStrings("doc-b", visible_after_reopen[0].node); + { + var status = try graph.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(active_job.job_id, status.build_job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + } + + const second_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, second_cleanup.phase); + try std.testing.expect(second_cleanup.claimed_page); + try std.testing.expect(second_cleanup.completed_page); + try std.testing.expect(!second_cleanup.published); + const final_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-c"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.claimed_page); + try std.testing.expect(final_cleanup.completed_page); + try std.testing.expect(final_cleanup.published); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "pagerank")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + try std.testing.expectError(error.NotFound, txn.get(abandoned_attempt_key)); + } +} + +test "graph pagerank cleanup page resumes from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-cleanup-cursor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-cleanup-cursor"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + const publish = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..graph_metric_build_cleanup_delete_page_units + 6) |i| { + var node_buf: [64]u8 = undefined; + const node = try std.fmt.bufPrint(&node_buf, "cleanup-extra-{d:0>3}", .{i}); + const key = try graph.graphMetricBuildPageRankOutDegreePartialKeyAlloc("pagerank", active_job.job_id, node, 999); + defer alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + try batch.commit(); + } + + const first_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, first_cleanup.phase); + try std.testing.expect(first_cleanup.claimed_page); + try std.testing.expect(!first_cleanup.completed_page); + try std.testing.expect(!first_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const cleanup_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, first_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, cleanup_page.state); + try std.testing.expect(cleanup_page.cursor.len > 0); + try std.testing.expect(cleanup_page.completed_units >= graph_metric_build_cleanup_delete_page_units); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expectEqualStrings(cleanup_page.cursor, job.cursor); + } + + const visible_after_partial_cleanup = try graph.graphMetricTopK("pagerank", 2); + defer { + for (visible_after_partial_cleanup) |*score| score.deinit(alloc); + alloc.free(visible_after_partial_cleanup); + } + try std.testing.expectEqual(@as(usize, 2), visible_after_partial_cleanup.len); + try std.testing.expectEqualStrings("doc-b", visible_after_partial_cleanup[0].node); + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + const renewed_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, renewed_cleanup.phase); + try std.testing.expect(renewed_cleanup.claimed_page); + try std.testing.expect(renewed_cleanup.completed_page); + try std.testing.expect(!renewed_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const cleanup_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .cleanup_old_generations, 0, renewed_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, cleanup_page.state); + try std.testing.expectEqualStrings("cleanup-job:", cleanup_page.cursor[0.."cleanup-job:".len]); + } + + const second_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-clean-2"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, second_cleanup.phase); + try std.testing.expect(second_cleanup.claimed_page); + try std.testing.expect(second_cleanup.completed_page); + try std.testing.expect(!second_cleanup.published); + const final_cleanup = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-clean-3"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.claimed_page); + try std.testing.expect(final_cleanup.completed_page); + try std.testing.expect(final_cleanup.published); +} + +test "graph pagerank planned build publishes scores matching local runner" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-parity"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-parity"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{ + .{ + .name = "pagerank_local", + .kind = .pagerank, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + .max_iterations = 2, + .tolerance = 0.000001, + }, + .{ + .name = "pagerank_planned", + .kind = .pagerank, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + .max_iterations = 2, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-a", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-a", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + + var local_status = try graph.runGraphMetric("pagerank_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runPageRankMetricPlanned("pagerank_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + try std.testing.expectEqual(@as(u64, 3), planned_status.last_event.?.score_count); + + const local_top = try graph.graphMetricTopK("pagerank_local", 3); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("pagerank_planned", 3); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "graph pagerank warm rebuild normalizes changed node sets across summary pages" { + const alloc = std.testing.allocator; + for ([_]usize{ 2, 66 }) |old_count| { + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-normalized-warm"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-normalized-warm"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual, .damping = 0.999999, .tolerance = 1e-6, .max_iterations = 1 }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + for (0..old_count) |index| { + var from_buf: [32]u8 = undefined; + var to_buf: [32]u8 = undefined; + const from = try std.fmt.bufPrint(&from_buf, "node:{d:0>4}", .{index}); + const to = try std.fmt.bufPrint(&to_buf, "node:{d:0>4}", .{(index + 1) % old_count}); + try graph.addEdge(from, to, "cites", 1, 0, 0, ""); + } + var original = try graph.runPageRankMetricPlanned("rank"); + original.deinit(alloc); + try graph.addEdge("new:a", "new:b", "cites", 1, 0, 0, ""); + try graph.addEdge("new:b", "new:a", "cites", 1, 0, 0, ""); + var rebuilt = try graph.runPageRankMetricPlanned("rank"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expect(rebuilt.delta > 0); // Proves this was a mapped warm seed, not a uniform cold run. + const scores = try graph.graphMetricTopK("rank", old_count + 2); + defer { + for (scores) |*score| score.deinit(alloc); + alloc.free(scores); + } + try std.testing.expectEqual(old_count + 2, scores.len); + var mass = metric_kernels.warm_start.Mass{}; + for (scores) |score| try mass.add(score.score); + try std.testing.expectApproxEqAbs(@as(f64, 1), try mass.total(), 1e-12); + } +} + +test "graph metric execution epoch fences old jobs without hiding published scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-execution-epoch"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-execution-epoch"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "rank", .kind = .pagerank, .refresh = .manual, .max_iterations = 1 }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + try graph.addEdge("a", "b", "cites", 1, 0, 0, ""); + var published = try graph.runPageRankMetricPlanned("rank"); + defer published.deinit(alloc); + try graph.acquireGraphMetricBuildLease("rank", try graph.graphMetricCurrentGeneration("rank")); + try drainGraphMetricBuildToPublishForTest(&graph, "rank", configs[0], "worker", &.{ + .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, + .iterate_contributions, .reduce_ranks, .check_convergence, + }); + _ = try materializeGraphMetricPublishPagesForTest(&graph, "rank", "worker"); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + const job = (try graph.metricBuildJob(&batch, "rank")).?; + var manifest = (try graph.metricBuildManifest(&batch, "rank", job.job_id)).?; + manifest.execution_schema_version = 2; + const key = try graph.graphMetricBuildManifestKeyAlloc("rank", job.job_id); + defer alloc.free(key); + var encoded: [GraphIndex.graph_metric_build_manifest_encoded_len]u8 = undefined; + GraphIndex.encodeGraphMetricBuildManifest(manifest, &encoded); + try std.testing.expectEqual(@as(u64, 2), GraphIndex.decodeGraphMetricBuildManifest(&encoded).?.execution_schema_version); + try batch.put(key, &encoded); + try batch.commit(); + } + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.runGraphMetricPlannedWorkerPageStepAt("rank", configs[0], "worker", 1)); + { + var txn = try graph.beginReadReverseTxn(); + const job_id = (try graph.metricBuildJob(&txn, "rank")).?.job_id; + txn.abort(); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.verifyGraphMetricBuildPublishReady("rank", job_id)); + } + const fenced = try graph.runGraphMetricPlannedCoordinatorStepAt("rank", configs[0], 1); + try std.testing.expect(fenced.failed_build and !fenced.advanced_phase); + var current = try graph.graphMetricStatus("rank"); + defer current.deinit(alloc); + try std.testing.expectEqual(published.published_generation, current.published_generation); + const top = try graph.graphMetricTopK("rank", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); +} + +test "graph pagerank failed planned build preserves prior published generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-failure-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-failure-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runPageRankMetricPlanned("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 2), before_failure_top.len); + + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("pagerank", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + var job_id: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + _ = try graph.metricBuildManifest(&txn, "pagerank", job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + + var failed = try graph.failGraphMetricPlannedBuild("pagerank", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed.build_job_id); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.last_error); + try std.testing.expectEqual(@as(usize, 1), failed.recent_failures.len); + try std.testing.expectEqual(job_id, failed.recent_failures[0].job_id); + + const after_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-c")); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(job_id, failed_job.job_id); + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("pagerank", rebuilding_generation)); + } + + try std.testing.expectError(error.GraphMetricBuildNotActive, graph.failGraphMetricPlannedBuild("pagerank", error.InvalidGraphMetricScore)); +} + +test "graph pagerank coordinator publish failure preserves prior published generation after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-publish-failure-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-publish-failure-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runPageRankMetricPlanned("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 2), before_failure_top.len); + + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("pagerank", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.verifyGraphMetricBuildPublishReady("pagerank", active_job.job_id); + } + const missing_rank_key = try graphMetricVectorKeyForNodeForTest(&graph, "pagerank", active_job.job_id, "rank", active_job.iteration + 1, "doc-a"); + defer alloc.free(missing_rank_key); + const missing_rank_value = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try alloc.dupe(u8, try txn.get(missing_rank_key)); + }; + defer alloc.free(missing_rank_value); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.delete(missing_rank_key); + try batch.commit(); + } + const rejected_materialization = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-missing-final-rank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, rejected_materialization.phase); + try std.testing.expect(rejected_materialization.claimed_page); + try std.testing.expect(!rejected_materialization.completed_page); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .publish_generation, active_job.iteration, rejected_materialization.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, failed_page.state); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_page.last_error); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("pagerank", active_job.score_generation)); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(missing_rank_key, missing_rank_value); + try batch.commit(); + } + _ = try materializeGraphMetricPublishPagesForTest(&graph, "pagerank", "worker-publish-materialize"); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var manifest = try graph.metricBuildManifest(&batch, "pagerank", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + manifest.config_fingerprint += 1; + try graph.putGraphMetricBuildManifestInBatch(&batch, "pagerank", manifest); + try batch.commit(); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.verifyGraphMetricBuildPublishReady("pagerank", active_job.job_id)); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_status = try graph.graphMetricStatus("pagerank"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_status.state); + try std.testing.expectEqual(published_generation, failed_status.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_status.build_job_id); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_status.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_status.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_status.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_status.recent_failures[0].phase); + + const after_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-c")); + } + try std.testing.expect((try graph.countGraphMetricScoreGeneration("pagerank", active_job.score_generation)) > 0); + try drainRetiredGraphMetricScoresForTest(&graph, "pagerank"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_job.phase); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("pagerank", active_job.score_generation)); + } +} + +test "graph pagerank exhausted publish page preserves root cause and prior generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-publish-exhausted-root-cause"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-publish-exhausted-root-cause"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runPageRankMetricPlanned("pagerank"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + var building = try graph.ensureGraphMetricPlannedBuild("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer building.deinit(alloc); + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + }; + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-prepare", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + + const missing_rank_key = try graphMetricVectorKeyForNodeForTest(&graph, "pagerank", active_job.job_id, "rank", active_job.iteration + 1, "doc-a"); + defer alloc.free(missing_rank_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.delete(missing_rank_key); + try batch.commit(); + } + + var exhausted_page_id: u64 = 0; + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const rejected = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", worker_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, rejected.phase); + try std.testing.expect(rejected.claimed_page); + try std.testing.expect(!rejected.completed_page); + if (attempt == 0) { + exhausted_page_id = rejected.page_id; + } else { + try std.testing.expectEqual(exhausted_page_id, rejected.page_id); + } + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .publish_generation, active_job.iteration, exhausted_page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, page.state); + try std.testing.expectEqual(@as(u64, graph_metric_build_max_page_attempts), page.attempt); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", page.last_error); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + const exhaustion_reason = try std.fmt.allocPrint( + alloc, + "GraphMetricBuildPageAttemptsExhausted: phase=publish_generation, iteration={d}, page_id={d}, attempt={d}, cause=InvalidGraphMetricScore", + .{ active_job.iteration, exhausted_page_id, graph_metric_build_max_page_attempts }, + ); + defer alloc.free(exhaustion_reason); + var failed_status = try graph.graphMetricStatus("pagerank"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_status.state); + try std.testing.expectEqual(published_generation, failed_status.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_status.build_job_id); + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_status.recent_failures.len); + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.recent_failures[0].last_error); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_status.recent_failures[0].phase); + + const after_failure_top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-c")); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqualStrings(exhaustion_reason, failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("pagerank", active_job.score_generation)); + } +} + +test "graph pagerank planned build drains partitioned pages across workers" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-multi-worker"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-multi-worker"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "pagerank_local", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "pagerank_planned", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + + var local_status = try graph.runGraphMetric("pagerank_local"); + defer local_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, local_status.state); + + try graph.acquireGraphMetricBuildLease("pagerank_planned", try graph.graphMetricCurrentGeneration("pagerank_planned")); + defer graph.releaseGraphMetricBuildLease("pagerank_planned") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank_planned") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "pagerank_planned", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expect(manifest.page_count > GraphIndex.graph_metric_iterative_build_phases.len); + _ = try graph.metricBuildPage(&txn, "pagerank_planned", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank_planned", active_job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank_planned", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank_planned", active_job.job_id, .reduce_ranks, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + } + + var worker_a_completed: usize = 0; + var worker_b_completed: usize = 0; + var completed = false; + var steps: usize = 0; + while (steps < 200) : (steps += 1) { + const worker_id = if (steps % 2 == 0) "worker-a" else "worker-b"; + const step = try graph.runGraphMetricPlannedWorkerStep("pagerank_planned", metrics[1], worker_id); + if (step.completed_page) { + if (std.mem.eql(u8, worker_id, "worker-a")) { + worker_a_completed += 1; + } else { + worker_b_completed += 1; + } + } + if (step.completed_build and step.phase == .cleanup_old_generations) { + completed = true; + break; + } + // A summary page is deliberately single-owner; the other worker may + // observe an idle tick while its lease holder advances the checkpoint. + } + try std.testing.expect(completed); + try std.testing.expect(worker_a_completed > 0); + try std.testing.expect(worker_b_completed > 0); + + var planned_status = try graph.graphMetricStatus("pagerank_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "pagerank_planned") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, completed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "pagerank_planned")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank_planned", active_job.job_id)) == null); + } + + const top_limit: usize = @intCast(graph.node_count); + const local_top = try graph.graphMetricTopK("pagerank_local", top_limit); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("pagerank_planned", top_limit); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + try std.testing.expectEqualStrings("hub", planned_top[0].node); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "graph pagerank planned worker page step leaves phase and publish to coordinator" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-worker-coordinator-split"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-worker-coordinator-split"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + var started = try graph.ensureGraphMetricPlannedBuild("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + } + + { + const worker_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, worker_step.phase); + try std.testing.expect(worker_step.claimed_page); + try std.testing.expect(worker_step.completed_page); + try std.testing.expect(!worker_step.advanced_phase); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + } + { + const coordinator_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, coordinator_step.phase); + try std.testing.expect(coordinator_step.advanced_phase); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + } + + var guard: usize = 0; + while (guard < 100) : (guard += 1) { + var txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + const phase = job.phase; + txn.abort(); + if (phase == .publish_generation) break; + if (phase == .complete or phase == .cleanup_old_generations) return error.TestExpectedGraphMetricBuildPublishPhase; + + const worker_id = if (guard % 2 == 0) "worker-a" else "worker-b"; + const worker_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", worker_id); + try std.testing.expect(!worker_step.advanced_phase); + if (!worker_step.claimed_page) { + const coordinator_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(phase, coordinator_step.phase); + } else if (worker_step.completed_page) { + _ = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + } + } + try std.testing.expect(guard < 100); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } + + const worker_publish_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish_step.phase); + try std.testing.expect(worker_publish_step.claimed_page); + try std.testing.expect(worker_publish_step.completed_page); + try std.testing.expect(!worker_publish_step.advanced_phase); + try std.testing.expect(!worker_publish_step.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectEqual(@as(u64, 0), try graph.metricPublishedGeneration(&txn, "pagerank")); + } + + const coordinator_publish_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish_step.phase); + try std.testing.expect(coordinator_publish_step.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expect((try graph.metricPublishedGeneration(&txn, "pagerank")) != 0); + } +} + +fn expectPlannedWorkerCoordinatorSplitUntilPublish( + graph: *GraphIndex, + metric_name: []const u8, +) !void { + var started = try graph.ensureGraphMetricPlannedBuild(metric_name, try graph.graphMetricCurrentGeneration(metric_name)); + defer started.deinit(graph.alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + } + { + const worker_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, worker_step.phase); + try std.testing.expect(worker_step.claimed_page); + try std.testing.expect(worker_step.completed_page); + try std.testing.expect(!worker_step.advanced_phase); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + } + { + const coordinator_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, coordinator_step.phase); + try std.testing.expect(coordinator_step.advanced_phase); + } + + var guard: usize = 0; + while (guard < 200) : (guard += 1) { + var txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + const phase = job.phase; + txn.abort(); + if (phase == .publish_generation) break; + if (phase == .complete or phase == .cleanup_old_generations) return error.TestExpectedGraphMetricBuildPublishPhase; + + const worker_id = switch (guard % 3) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const worker_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, worker_id); + try std.testing.expect(!worker_step.advanced_phase); + if (worker_step.completed_page or !worker_step.claimed_page) { + _ = try graph.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + } + } + try std.testing.expect(guard < 200); + + const worker_publish_step = try graph.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish_step.phase); + try std.testing.expect(!worker_publish_step.claimed_page); + try std.testing.expect(!worker_publish_step.advanced_phase); + try std.testing.expect(!worker_publish_step.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectEqual(@as(u64, 0), try graph.metricPublishedGeneration(&txn, metric_name)); + } + + const coordinator_publish_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish_step.phase); + try std.testing.expect(coordinator_publish_step.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expect((try graph.metricPublishedGeneration(&txn, metric_name)) != 0); + } +} + +test "graph planned worker coordinator split applies to degree eigenvector and hits" { + const alloc = std.testing.allocator; + + { + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-worker-coordinator-split"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-worker-coordinator-split"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try expectPlannedWorkerCoordinatorSplitUntilPublish(&graph, "degree"); + } + + { + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-worker-coordinator-split"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-worker-coordinator-split"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try expectPlannedWorkerCoordinatorSplitUntilPublish(&graph, "eigenvector"); + } + + { + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-worker-coordinator-split"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-worker-coordinator-split"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try expectPlannedWorkerCoordinatorSplitUntilPublish(&graph, "hits_authority"); + } +} + +test "graph pagerank planned build resumes after dynamic iteration planning reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-planned-reopen"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-planned-reopen"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "pagerank", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + _ = try graph.metricBuildPage(&txn, "pagerank", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + { + var status = try graph.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(active_job.job_id, status.build_job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, status.phase); + try std.testing.expectEqual(@as(u32, 1), status.build_iteration); + } + + var resumed = try graph.runGraphMetricPlannedActive("pagerank", metrics[0]); + defer resumed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, resumed.state); + try std.testing.expectEqual(@as(u32, 2), resumed.iterations_completed); + try std.testing.expect(!resumed.converged); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, completed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "pagerank")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "pagerank", active_job.job_id)) == null); + } +} + +test "graph metric build degree manifest partitions reverse-edge scan pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-manifest-partitions"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-manifest-partitions"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "degree", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(GraphIndex.graph_metric_degree_build_phases.len, manifest.phase_count); + // Three data-partition pages plus the reduction dependency summary. + try std.testing.expectEqual(GraphIndex.graph_metric_degree_build_phases.len + 4, manifest.page_count); + + const first_scan = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, first_scan.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, first_scan.range_kind); + try std.testing.expect(first_scan.range_lower.len > 0); + try std.testing.expect(first_scan.range_upper.len > 0); + try std.testing.expect(first_scan.output_prefix.len > 0); + + const second_scan = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, second_scan.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, second_scan.range_kind); + try std.testing.expectEqualStrings(first_scan.range_upper, second_scan.range_lower); + try std.testing.expectEqualStrings("", second_scan.range_upper); + try std.testing.expect(first_scan.total_units >= second_scan.total_units); + try std.testing.expect(first_scan.total_units - second_scan.total_units <= 1); + try std.testing.expectEqual(graph.edge_count, first_scan.total_units + second_scan.total_units); + + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 3)) == null); + + const reduce = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .reduce_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, reduce.range_kind); + try std.testing.expect(reduce.range_lower.len > 0); + try std.testing.expect(reduce.range_upper.len > 0); + try std.testing.expect(reduce.output_prefix.len > 0); + + const second_reduce = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .reduce_ranks, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, second_reduce.range_kind); + try std.testing.expectEqualStrings(reduce.range_upper, second_reduce.range_lower); + try std.testing.expectEqualStrings("", second_reduce.range_upper); + try std.testing.expect(reduce.total_units >= second_reduce.total_units); + try std.testing.expect(reduce.total_units - second_reduce.total_units <= 1); + try std.testing.expectEqual(graph.node_count, reduce.total_units + second_reduce.total_units); + + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", active_job.job_id, .reduce_ranks, 0, 4)) == null); + + const partial_cleanup = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .cleanup_old_generations, 0, 0) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, partial_cleanup.range_kind); + try std.testing.expect(partial_cleanup.output_prefix.len > 0); + const final_cleanup = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .cleanup_old_generations, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, final_cleanup.range_kind); + try std.testing.expect(final_cleanup.output_prefix.len > 0); + try std.testing.expect(!std.mem.eql(u8, partial_cleanup.output_prefix, final_cleanup.output_prefix)); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", active_job.job_id, .cleanup_old_generations, 0, 2)) == null); + } + + try graph.ensureGraphMetricBuildManifestForJob("degree", active_job); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try graph.metricBuildManifest(&txn, "degree", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expectEqual(GraphIndex.graph_metric_degree_build_phases.len + 4, manifest.page_count); + const first_scan = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.pending, first_scan.state); + const second_scan = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(graph.edge_count, first_scan.total_units + second_scan.total_units); + } +} + +test "graph degree scan build page honors reverse-edge key range" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-scan-range"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-scan-range"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-d", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + var edge_keys = std.ArrayListUnmanaged([]u8).empty; + defer { + for (edge_keys.items) |key| alloc.free(key); + edge_keys.deinit(alloc); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + var parsed = (try parseReverseEdgeKeyAlloc(alloc, entry.key)) orelse continue; + defer parsed.deinit(alloc); + if (!std.mem.eql(u8, parsed.index_name, "links")) continue; + try edge_keys.append(alloc, try alloc.dupe(u8, entry.key)); + } + } + try std.testing.expectEqual(@as(usize, 2), edge_keys.items.len); + var first_edge = (try parseReverseEdgeKeyAlloc(alloc, edge_keys.items[0])) orelse return error.TestExpectedGraphEdge; + defer first_edge.deinit(alloc); + const expected_source = try alloc.dupe(u8, first_edge.source); + defer alloc.free(expected_source); + const expected_target = try alloc.dupe(u8, first_edge.target); + defer alloc.free(expected_target); + var second_edge = (try parseReverseEdgeKeyAlloc(alloc, edge_keys.items[1])) orelse return error.TestExpectedGraphEdge; + defer second_edge.deinit(alloc); + const excluded_source = try alloc.dupe(u8, second_edge.source); + defer alloc.free(excluded_source); + const excluded_target = try alloc.dupe(u8, second_edge.target); + defer alloc.free(excluded_target); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var page = try graph.metricBuildPage(&batch, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + page.range_kind = .reverse_edges; + page.range_lower = edge_keys.items[0]; + page.range_upper = edge_keys.items[1]; + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", page); + try batch.commit(); + } + + const claimed = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, graph_metric_local_build_worker_id) orelse return error.TestExpectedGraphMetricBuildPage; + const score_count = try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, claimed); + try std.testing.expectEqual(@as(usize, 2), score_count); + + var score_txn = try graph.beginReadReverseTxn(); + defer score_txn.abort(); + const expected_nodes = [_][]const u8{ expected_source, expected_target }; + for (expected_nodes) |node| { + const partial_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", active_job.job_id, node, claimed.page_id); + defer alloc.free(partial_key); + const raw = try score_txn.get(partial_key); + try std.testing.expectEqual(@as(u64, 1), std.mem.readInt(u64, raw[0..8], .little)); + } + const excluded_nodes = [_][]const u8{ excluded_source, excluded_target }; + for (excluded_nodes) |node| { + const partial_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", active_job.job_id, node, claimed.page_id); + defer alloc.free(partial_key); + try std.testing.expectError(error.NotFound, score_txn.get(partial_key)); + } +} + +test "graph degree reduce aggregates partials from multiple scan pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-reduce-multipage"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-reduce-multipage"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + var edge_keys = std.ArrayListUnmanaged([]u8).empty; + defer { + for (edge_keys.items) |key| alloc.free(key); + edge_keys.deinit(alloc); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + var cur = try txn.openCursor(); + defer cur.close(); + var entry_opt = try cur.first(); + while (entry_opt) |entry| : (entry_opt = try cur.next()) { + if (std.mem.startsWith(u8, entry.key, graph_meta_prefix)) continue; + var parsed = (try parseReverseEdgeKeyAlloc(alloc, entry.key)) orelse continue; + defer parsed.deinit(alloc); + if (!std.mem.eql(u8, parsed.index_name, "links")) continue; + try edge_keys.append(alloc, try alloc.dupe(u8, entry.key)); + } + } + try std.testing.expectEqual(@as(usize, 2), edge_keys.items.len); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var first_scan = try graph.metricBuildPage(&batch, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + first_scan.range_kind = .reverse_edges; + first_scan.range_lower = edge_keys.items[0]; + first_scan.range_upper = edge_keys.items[1]; + first_scan.total_units = 1; + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", first_scan); + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", .{ + .job_id = active_job.job_id, + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 99, + .state = .pending, + .range_kind = .reverse_edges, + .range_lower = edge_keys.items[1], + .range_upper = "", + .total_units = 1, + }); + try batch.commit(); + } + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const first_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, first_page)); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0))); + + const second_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 99, "worker-b") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, second_page)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + + const reduce_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-r") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 1), try graph.executeDegreeReduceBuildPageWithLimit("degree", active_job, reduce_page, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const partial = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .reduce_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, partial.state); + try std.testing.expectEqual(@as(u64, 1), partial.completed_units); + try std.testing.expect(partial.cursor.len > 0); + } + const renewed_reduce = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-r") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeReduceBuildPage("degree", active_job, renewed_reduce)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .reduce_ranks, 0)); + + var score_txn = try graph.beginReadReverseTxn(); + defer score_txn.abort(); + const expected_scores = [_]struct { + node: []const u8, + score: f64, + }{ + .{ .node = "doc-a", .score = 1.0 }, + .{ .node = "doc-b", .score = 2.0 }, + .{ .node = "doc-c", .score = 1.0 }, + }; + for (expected_scores) |expected| { + const score_key = try graph.graphMetricScoreKeyAlloc("degree", active_job.score_generation, expected.node); + defer alloc.free(score_key); + const raw = try score_txn.get(score_key); + try std.testing.expectApproxEqAbs(expected.score, GraphIndex.decodeF64(raw) orelse return error.TestExpectedGraphMetricScore, 0.0000001); + } + { + const job = try graph.metricBuildJob(&score_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } +} + +test "graph degree scan page resumes from persisted cursor" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-scan-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-scan-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "hub", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const first_claim = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPageWithLimit("degree", metrics[0], active_job, first_claim, 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expect(page.cursor.len > 0); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0))); + } + + const renewed = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, renewed)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + + const reduce_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-r") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 3), try graph.executeDegreeReduceBuildPage("degree", active_job, reduce_page)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .reduce_ranks, 0)); + + var score_txn = try graph.beginReadReverseTxn(); + defer score_txn.abort(); + const hub_key = try graph.graphMetricScoreKeyAlloc("degree", active_job.score_generation, "hub"); + defer alloc.free(hub_key); + const raw = try score_txn.get(hub_key); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), GraphIndex.decodeF64(raw) orelse return error.TestExpectedGraphMetricScore, 0.0000001); +} + +test "graph planned metric build retires a superseded generation without poisoning newer work" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-generation-coherence"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-generation-coherence"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var building = try graph.ensureGraphMetricPlannedBuild("degree", try graph.graphMetricCurrentGeneration("degree")); + defer building.deinit(alloc); + const stale_job_id = building.build_job_id; + try std.testing.expect(stale_job_id != 0); + + // A write after manifest creation makes every page and publication from + // that manifest stale, even if its individual page leases are valid. + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + try std.testing.expectError( + error.GraphMetricBuildSuperseded, + graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-stale"), + ); + try std.testing.expectError( + error.GraphMetricBuildSuperseded, + graph.verifyGraphMetricBuildPublishReady("degree", stale_job_id), + ); + + const coordinator = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expect(coordinator.failed_build); + try std.testing.expect(!coordinator.published); + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + // The failed attempt remains diagnosable, but its immutable target does + // not poison the newer dirty generation as a terminal failure. + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, status.state); + try std.testing.expectEqualStrings("", status.last_error); + try std.testing.expectEqual(@as(usize, 1), status.recent_failures.len); + try std.testing.expectEqual(building.target_edge_generation, status.recent_failures[0].target_generation); + try std.testing.expectEqualStrings("GraphMetricBuildSuperseded", status.recent_failures[0].last_error); +} + +test "graph degree scan attempt output adopts only on page completion" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-scan-attempt-adopt"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-scan-attempt-adopt"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "hub", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const first_claim = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), first_claim.attempt); + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPageWithLimit("degree", metrics[0], active_job, first_claim, 1)); + + const attempt_hub_key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", active_job.job_id, .scan_edges_and_out_degree, 0, first_claim.page_id, first_claim.attempt, "hub"); + defer alloc.free(attempt_hub_key); + const adopted_hub_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", active_job.job_id, "hub", first_claim.page_id); + defer alloc.free(adopted_hub_key); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_hub_key)); + try std.testing.expectError(error.NotFound, txn.get(adopted_hub_key)); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, first_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + } + + const renewed = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, first_claim.page_id, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), renewed.attempt); + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, renewed)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(attempt_hub_key)); + try std.testing.expectEqual(@as(u64, 2), try GraphIndex.readU64OrZero(&txn, adopted_hub_key)); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, first_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + } + + const reduce_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-r") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 3), try graph.executeDegreeReduceBuildPage("degree", active_job, reduce_page)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .reduce_ranks, 0)); + + const publish = try graph.runGraphMetricPlannedWorkerStep("degree", metrics[0], "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const first_cleanup = try graph.runGraphMetricPlannedWorkerStep("degree", metrics[0], "worker-clean-1"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, first_cleanup.phase); + try std.testing.expect(first_cleanup.completed_page); + try std.testing.expect(!first_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(attempt_hub_key)); + try std.testing.expectError(error.NotFound, txn.get(adopted_hub_key)); + } + + const final_cleanup = try graph.runGraphMetricPlannedWorkerStep("degree", metrics[0], "worker-clean-2"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.completed_page); + try std.testing.expect(final_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(attempt_hub_key)); + try std.testing.expectError(error.NotFound, txn.get(adopted_hub_key)); + const completed_job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + } +} + +test "graph degree scan attempt adoption resumes in bounded pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-scan-bounded-adopt"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-scan-bounded-adopt"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const page = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..graph_metric_build_adoption_page_units + 1) |i| { + const node = try std.fmt.allocPrint(alloc, "node-{d:0>4}", .{i}); + defer alloc.free(node); + const key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page.page_id, page.attempt, node); + defer alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + try batch.commit(); + } + + const first = try graph.adoptGraphMetricAttemptOutputPage("degree", .degree, active_job.job_id, page); + try std.testing.expectEqual(graph_metric_build_adoption_page_units, first.adopted); + try std.testing.expect(!first.reached_end); + + const second = try graph.adoptGraphMetricAttemptOutputPage("degree", .degree, active_job.job_id, page); + try std.testing.expectEqual(@as(usize, 1), second.adopted); + try std.testing.expect(second.reached_end); + + // Durable attempt records are internal typed state. Truncation must fail + // the build page instead of being skipped and silently lowering scores. + const corrupt_attempt_key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page.page_id, page.attempt, "node-corrupt"); + defer alloc.free(corrupt_attempt_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(corrupt_attempt_key, &.{ 1, 2, 3, 4 }); + try batch.commit(); + } + try std.testing.expectError( + error.InvalidGraphMetricBuildManifest, + graph.adoptGraphMetricAttemptOutputPage("degree", .degree, active_job.job_id, page), + ); + + const first_attempt_key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page.page_id, page.attempt, "node-0000"); + defer alloc.free(first_attempt_key); + const last_attempt_key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page.page_id, page.attempt, "node-0512"); + defer alloc.free(last_attempt_key); + const first_adopted_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", active_job.job_id, "node-0000", page.page_id); + defer alloc.free(first_adopted_key); + const last_adopted_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", active_job.job_id, "node-0512", page.page_id); + defer alloc.free(last_adopted_key); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(first_attempt_key)); + try std.testing.expectError(error.NotFound, txn.get(last_attempt_key)); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, first_adopted_key)); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, last_adopted_key)); +} + +test "graph degree scan page reclaim recomputes without double counting partials" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-scan-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-scan-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "hub", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const first_claim = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 2), try graph.executeDegreeScanBuildPageWithLimit("degree", metrics[0], active_job, first_claim, 1)); + + var persisted_cursor: []u8 = ""; + defer if (persisted_cursor.len > 0) alloc.free(persisted_cursor); + var expires_at: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expect(page.cursor.len > 0); + persisted_cursor = try alloc.dupe(u8, page.cursor); + expires_at = page.lease_expires_at_ms; + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqualStrings("worker-b", reclaimed.worker_id); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expect(!std.mem.eql(u8, persisted_cursor, reclaimed.cursor)); + + try std.testing.expectEqual(@as(usize, 3), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, reclaimed)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + + const reduce_page = try graph.claimGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-r") orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 3), try graph.executeDegreeReduceBuildPage("degree", active_job, reduce_page)); + + var score_txn = try graph.beginReadReverseTxn(); + defer score_txn.abort(); + const hub_key = try graph.graphMetricScoreKeyAlloc("degree", active_job.score_generation, "hub"); + defer alloc.free(hub_key); + const raw = try score_txn.get(hub_key); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), GraphIndex.decodeF64(raw) orelse return error.TestExpectedGraphMetricScore, 0.0000001); +} + +test "graph degree reduce score write rejects reclaimed stale attempt" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-reduce-stale-attempt"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-reduce-stale-attempt"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "hub", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const scan_page = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-s", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(usize, 3), try graph.executeDegreeScanBuildPage("degree", metrics[0], active_job, scan_page)); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + + const stale_reduce = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-a", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + const reclaimed_reduce = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-b", stale_reduce.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, stale_reduce.attempt + 1), reclaimed_reduce.attempt); + + const hub_key = try graph.graphMetricScoreKeyAlloc("degree", active_job.score_generation, "hub"); + defer alloc.free(hub_key); + const stale_scores = [_]GraphIndex.GraphMetricScore{.{ + .node = "hub", + .score = 99.0, + }}; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.completeDegreeReduceBuildPageWithScoresForAttempt("degree", active_job, stale_reduce, stale_reduce.worker_id, "stale-reduce", 1, &stale_scores, GraphIndex.graphMetricScoresFingerprint(&stale_scores))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(hub_key)); + } + + const scores = [_]GraphIndex.GraphMetricScore{.{ + .node = "hub", + .score = 2.0, + }}; + const completed = try graph.completeDegreeReduceBuildPageWithScoresForAttempt("degree", active_job, reclaimed_reduce, reclaimed_reduce.worker_id, "degree-reduce:partials=2;scores=1", 1, &scores, GraphIndex.graphMetricScoresFingerprint(&scores)); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, completed.state); + try std.testing.expectEqual(reclaimed_reduce.attempt, completed.attempt); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const raw = try txn.get(hub_key); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), GraphIndex.decodeF64(raw) orelse return error.TestExpectedGraphMetricScore, 0.0000001); + } +} + +test "graph metric build page lease lifecycle supports reclaim and idempotent completion" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-lease"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-lease"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + const start_ms: u64 = 1000; + const first_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", start_ms) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, first_claim.state); + try std.testing.expectEqual(@as(u64, 1), first_claim.attempt); + try std.testing.expectEqualStrings("worker-a", first_claim.worker_id); + try std.testing.expect(first_claim.lease_expires_at_ms > start_ms); + + try std.testing.expect((try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", start_ms + 1)) == null); + const same_worker_claim = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", start_ms + 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), same_worker_claim.attempt); + try std.testing.expectEqualStrings("worker-a", same_worker_claim.worker_id); + + const reclaimed = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", same_worker_claim.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqualStrings("worker-b", reclaimed.worker_id); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 1, 55)); + + const completed = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", 1, 55); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, completed.state); + try std.testing.expectEqual(@as(u64, 0), completed.lease_expires_at_ms); + try std.testing.expectEqual(@as(u64, 1), completed.completed_units); + try std.testing.expectEqual(@as(u64, 55), completed.output_fingerprint); + try std.testing.expect((try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-c", completed.lease_expires_at_ms + 1)) == null); + const completed_again = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-c", 1, 55); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, completed_again.state); + try std.testing.expectError(error.GraphMetricBuildPageOutputMismatch, graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-c", 1, 56)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-a", start_ms); + const failed = try graph.failGraphMetricBuildPage("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-a", "transient"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, failed.state); + try std.testing.expectEqual(@as(u64, 1), failed.attempt); + try std.testing.expectEqualStrings("transient", failed.last_error); + const retried = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", start_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, retried.state); + try std.testing.expectEqual(@as(u64, 2), retried.attempt); + try std.testing.expectEqualStrings("", retried.last_error); + try std.testing.expectEqualStrings("worker-b", retried.worker_id); + + const same_worker_replacement = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", retried.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, same_worker_replacement.state); + try std.testing.expectEqual(@as(u64, 3), same_worker_replacement.attempt); + try std.testing.expectEqualStrings("worker-b", same_worker_replacement.worker_id); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.updateGraphMetricBuildPageProgressForAttempt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", retried.attempt, "stale-cursor", 1, retried.total_units)); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.completeGraphMetricBuildPageForAttempt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", retried.attempt, 1, 66)); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.failGraphMetricBuildPageForAttempt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", retried.attempt, "stale failure")); + const replacement_complete = try graph.completeGraphMetricBuildPageForAttempt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", same_worker_replacement.attempt, 1, 66); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, replacement_complete.state); + try std.testing.expectEqual(@as(u64, 3), replacement_complete.attempt); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.completeGraphMetricBuildPageForAttempt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-b", retried.attempt, 1, 66)); + const legacy_idempotent_complete = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-c", 1, 66); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, legacy_idempotent_complete.state); + try std.testing.expectEqual(@as(u64, 3), legacy_idempotent_complete.attempt); +} + +test "graph metric build page retry policy refuses exhausted attempts" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-retry"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-retry"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const page = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, worker_id, 1000 + attempt) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(attempt + 1, page.attempt); + _ = try graph.failGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, worker_id, "retryable"); + } + + try std.testing.expect((try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-d", 2000)) == null); + try std.testing.expect((try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-d", 2000)) == null); + const summary = try graph.summarizeGraphMetricBuildPhase("degree", active_job.job_id, .scan_edges_and_out_degree, 0); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); +} + +test "graph metric coordinator fails build after page attempts exhaust" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-exhausted-fail"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-exhausted-fail"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + const prepare_worker = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expect(prepare_worker.completed_page); + try std.testing.expect(!prepare_worker.advanced_phase); + const prepare_coordinator = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expect(prepare_coordinator.advanced_phase); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + break :blk job; + }; + + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const page = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, worker_id, 1000 + attempt) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(attempt + 1, page.attempt); + _ = try graph.failGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, worker_id, "retryable"); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.idle, status.phase); + try std.testing.expectEqualStrings( + "GraphMetricBuildPageAttemptsExhausted: phase=scan_edges_and_out_degree, iteration=0, page_id=1, attempt=3, cause=retryable", + status.last_error, + ); + try std.testing.expectEqual(@as(u64, 0), status.published_generation); +} + +test "graph metric coordinator reports expired exhausted page lease" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-expired-exhausted-fail"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-expired-exhausted-fail"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + const prepare_worker = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expect(prepare_worker.completed_page); + const prepare_coordinator = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expect(prepare_coordinator.advanced_phase); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + }; + const first = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + const second = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", first.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + const exhausted = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-c", second.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, graph_metric_build_max_page_attempts), exhausted.attempt); + try std.testing.expectEqualStrings("", exhausted.last_error); + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetricAt("degree", exhausted.lease_expires_at_ms + 1); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + + const exhaustion_reason = "GraphMetricBuildPageAttemptsExhausted: phase=scan_edges_and_out_degree, iteration=0, page_id=1, attempt=3, cause=GraphMetricBuildPageLeaseExpired"; + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqualStrings(exhaustion_reason, status.last_error); + try std.testing.expectEqual(@as(usize, 1), status.recent_failures.len); + try std.testing.expectEqualStrings(exhaustion_reason, status.recent_failures[0].last_error); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqualStrings(exhaustion_reason, failed_job.last_error); + } +} + +test "graph metric build scheduler claims next eligible page" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-scheduler"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-scheduler"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", .{ + .job_id = active_job.job_id, + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 99, + .state = .pending, + .range_kind = .reverse_edges, + .worker_id = "", + .total_units = 2, + }); + try batch.commit(); + } + + const first = try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-a", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), first.page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, first.state); + try std.testing.expectEqualStrings("worker-a", first.worker_id); + + const second = try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-b", 1001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 99), second.page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, second.state); + try std.testing.expectEqualStrings("worker-b", second.worker_id); + + try std.testing.expect((try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-c", 1002)) == null); + + const reclaimed = try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-c", first.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), reclaimed.page_id); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqualStrings("worker-c", reclaimed.worker_id); + + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-c", 1, 10); + const retried = try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-d", second.lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 99), retried.page_id); + try std.testing.expectEqual(@as(u64, 2), retried.attempt); + try std.testing.expectEqualStrings("worker-d", retried.worker_id); +} + +test "graph metric status summarizes multiple active build pages with cap" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-status-cap"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-status-cap"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-prepare", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-prepare", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..graph_metric_status_page_limit) |i| { + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", .{ + .job_id = active_job.job_id, + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 100 + @as(u64, @intCast(i)), + .state = .pending, + .range_kind = .reverse_edges, + .total_units = 4, + }); + } + try batch.commit(); + } + + for (0..graph_metric_status_page_limit + 1) |i| { + var worker_buf: [64]u8 = undefined; + const worker_id = try std.fmt.bufPrint(&worker_buf, "worker-{d}", .{i}); + const page_id: u64 = if (i == 0) 1 else 99 + @as(u64, @intCast(i)); + const claimed = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page_id, worker_id, 2000 + @as(u64, @intCast(i))) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, claimed.state); + try std.testing.expectEqualStrings(worker_id, claimed.worker_id); + _ = try graph.updateGraphMetricBuildPageProgress("degree", active_job.job_id, .scan_edges_and_out_degree, 0, page_id, worker_id, worker_id, if (claimed.total_units > 0) 1 else 0, claimed.total_units); + } + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.phase); + try std.testing.expect(status.build_job_id != 0); + try std.testing.expect(status.build_started_at_ms != 0); + try std.testing.expect(status.build_lease_expires_at_ms > 0); + try std.testing.expect(std.math.isFinite(status.progress)); + try std.testing.expect(status.progress >= 0.0); + try std.testing.expect(status.progress <= 1.0); + try std.testing.expectEqual(@as(u64, graph_metric_status_page_limit + 1), status.build_completed_units); + try std.testing.expectEqual(@as(u64, 1 + graph_metric_status_page_limit * 4), status.build_total_units); + try std.testing.expectEqualStrings("", status.build_cursor); + try std.testing.expect(status.build_pages_truncated); + try std.testing.expectEqual(@as(usize, graph_metric_status_page_limit), status.build_pages.len); + for (status.build_pages) |page_status| { + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, page_status.phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, page_status.range_kind); + try std.testing.expect(page_status.worker_id.len > 0); + try std.testing.expect(page_status.lease_expires_at_ms > 0); + try std.testing.expectEqual(@as(u64, 1), page_status.attempt); + try std.testing.expect(page_status.cursor.len > 0); + try std.testing.expectEqualStrings("", page_status.last_error); + try std.testing.expect(page_status.completed_units > 0); + try std.testing.expect(page_status.total_units > 0); + try std.testing.expect(page_status.completed_units <= page_status.total_units); + } +} + +test "graph metric build page progress cursor survives renew and reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-page-progress"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-page-progress"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + const claim = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + const progress = try graph.updateGraphMetricBuildPageProgress("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", "edge-page:0001", 1, claim.total_units); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, progress.state); + try std.testing.expectEqualStrings("edge-page:0001", progress.cursor); + try std.testing.expectEqual(@as(u64, 1), progress.completed_units); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqualStrings("edge-page:0001", status.build_cursor); + try std.testing.expectEqual(@as(u64, 1), status.build_completed_units); + try std.testing.expectEqual(claim.total_units, status.build_total_units); + try std.testing.expect(!status.build_pages_truncated); + try std.testing.expectEqual(@as(usize, 1), status.build_pages.len); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.build_pages[0].phase); + try std.testing.expectEqual(@as(u32, 0), status.build_pages[0].iteration); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, status.build_pages[0].state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, status.build_pages[0].range_kind); + try std.testing.expectEqualStrings("worker-a", status.build_pages[0].worker_id); + try std.testing.expect(status.build_pages[0].lease_expires_at_ms > 0); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].attempt); + try std.testing.expectEqualStrings("edge-page:0001", status.build_pages[0].cursor); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].completed_units); + try std.testing.expectEqual(claim.total_units, status.build_pages[0].total_units); + try std.testing.expectEqualStrings("", status.build_pages[0].last_error); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const renewed = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqualStrings("edge-page:0001", renewed.cursor); + try std.testing.expectEqual(@as(u64, 1), renewed.completed_units); + } + + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.updateGraphMetricBuildPageProgress("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", "edge-page:bad", 2, claim.total_units)); + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqualStrings("edge-page:0001", page.cursor); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + // Parallel page cursors remain page-local; the aggregate job does not + // publish one worker's checkpoint as a global resume position. + try std.testing.expectEqualStrings("", job.cursor); + try std.testing.expectEqual(@as(u64, 1), job.completed_units); + } + var reopened_status = try graph.graphMetricStatus("degree"); + defer reopened_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, reopened_status.state); + try std.testing.expectEqual(@as(usize, 1), reopened_status.build_pages.len); + try std.testing.expectEqual(@as(u64, 1), reopened_status.build_pages[0].page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reopened_status.build_pages[0].state); + try std.testing.expectEqualStrings("worker-a", reopened_status.build_pages[0].worker_id); + try std.testing.expectEqualStrings("edge-page:0001", reopened_status.build_pages[0].cursor); + + _ = try graph.failGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", "simulated page failure"); + var failed_status = try graph.graphMetricStatus("degree"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, failed_status.state); + try std.testing.expectEqual(@as(usize, 1), failed_status.build_pages.len); + try std.testing.expectEqual(@as(u64, 1), failed_status.build_pages[0].page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, failed_status.build_pages[0].state); + try std.testing.expectEqualStrings("worker-a", failed_status.build_pages[0].worker_id); + try std.testing.expectEqual(@as(u64, 1), failed_status.build_pages[0].attempt); + try std.testing.expectEqualStrings("edge-page:0001", failed_status.build_pages[0].cursor); + try std.testing.expectEqual(@as(u64, 1), failed_status.build_pages[0].completed_units); + try std.testing.expectEqual(claim.total_units, failed_status.build_pages[0].total_units); + try std.testing.expectEqualStrings("simulated page failure", failed_status.build_pages[0].last_error); +} + +test "graph metric build phase barrier persists summary and advances only when complete" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-phase-barrier"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-phase-barrier"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + const pending = try graph.summarizeGraphMetricBuildPhase("degree", active_job.job_id, .prepare_generation, 0); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.pending, pending.state); + try std.testing.expectEqual(@as(u64, 1), pending.expected_pages); + try std.testing.expectEqual(@as(u64, 0), pending.completed_pages); + try std.testing.expectEqual(@as(u64, 0), pending.failed_pages); + try std.testing.expectEqual(@as(u64, 1), pending.total_units); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + const summary = try graph.metricBuildPhaseSummary(&txn, "degree", active_job.job_id, .prepare_generation, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.pending, summary.state); + } + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 100); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "degree", active_job.job_id, .prepare_generation, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.completed_pages); + try std.testing.expectEqual(@as(u64, 1), summary.completed_units); + try std.testing.expectEqual(@as(u64, 100), summary.output_fingerprint); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + const lease = try graph.metricBuildLease(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildLease; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, lease.phase); + } + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000); + _ = try graph.failGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", "scan failed"); + const failed = try graph.summarizeGraphMetricBuildPhase("degree", active_job.job_id, .scan_edges_and_out_degree, 0); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, failed.state); + try std.testing.expectEqual(@as(u64, 1), failed.failed_pages); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + } +} + +test "graph metric build phase barrier summarizes every durable page in phase" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-phase-barrier-pages"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-phase-barrier-pages"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildPageInBatch(&batch, "degree", .{ + .job_id = active_job.job_id, + .phase = .scan_edges_and_out_degree, + .iteration = 0, + .page_id = 99, + .state = .pending, + .range_kind = .reverse_edges, + .worker_id = "", + .total_units = 2, + }); + try batch.commit(); + } + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 10); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 1, 20); + const pending = try graph.summarizeGraphMetricBuildPhase("degree", active_job.job_id, .scan_edges_and_out_degree, 0); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.pending, pending.state); + try std.testing.expectEqual(@as(u64, 2), pending.expected_pages); + try std.testing.expectEqual(@as(u64, 1), pending.completed_pages); + try std.testing.expectEqual(@as(u64, 3), pending.total_units); + try std.testing.expectEqual(@as(u64, 1), pending.completed_units); + // Incomplete phases use the O(1) progress record and defer the full-page + // fingerprint fold until every page is complete. + try std.testing.expectEqual(@as(u64, 0), pending.output_fingerprint); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0))); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 99, "worker-b", 3000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 99, "worker-b", 2, 30); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 2), summary.expected_pages); + try std.testing.expectEqual(@as(u64, 2), summary.completed_pages); + try std.testing.expectEqual(@as(u64, 3), summary.completed_units); + try std.testing.expectEqual(@as(u64, 20 ^ 30), summary.output_fingerprint); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + } +} + +test "graph metric build iteration summary records convergence from check phase" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-iteration-summary"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-iteration-summary"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.00001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try std.testing.expectError(error.GraphMetricBuildIterationNotComplete, graph.recordGraphMetricBuildIterationSummaryFromCheckPhase("pagerank", active_job.job_id, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-a", 1000); + const page = try graph.completeGraphMetricBuildConvergencePage("pagerank", active_job.job_id, 0, 5, "worker-a", 2, 77, 0.25, 0.30, 1.0, false); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectApproxEqAbs(@as(f64, 0.25), page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.30), page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), page.rank_sum, 0.0000001); + try std.testing.expect(!page.converged); + + const iteration_summary = try graph.recordGraphMetricBuildIterationSummaryFromCheckPhase("pagerank", active_job.job_id, 0); + try std.testing.expectEqual(@as(u32, 0), iteration_summary.iteration); + try std.testing.expectEqual(@as(u64, 1), iteration_summary.expected_pages); + try std.testing.expectEqual(@as(u64, 1), iteration_summary.completed_pages); + try std.testing.expectApproxEqAbs(@as(f64, 0.25), iteration_summary.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.30), iteration_summary.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), iteration_summary.rank_sum, 0.0000001); + try std.testing.expect(!iteration_summary.converged); + try std.testing.expect(iteration_summary.fixed_iteration_limit); + try std.testing.expectEqual(@as(u64, 77), iteration_summary.output_fingerprint); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const phase_summary = try graph.metricBuildPhaseSummary(&txn, "pagerank", active_job.job_id, .check_convergence, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, phase_summary.state); + try std.testing.expectApproxEqAbs(@as(f64, 0.25), phase_summary.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.30), phase_summary.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), phase_summary.rank_sum, 0.0000001); + try std.testing.expect(!phase_summary.converged); + const stored_iteration = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expectEqual(@as(u64, 77), stored_iteration.output_fingerprint); + try std.testing.expect(stored_iteration.fixed_iteration_limit); + } +} + +test "graph metric build publish verification requires completed prerequisite phases" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-publish-verify-degree"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-publish-verify-degree"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.verifyGraphMetricBuildPublishReady("degree", active_job.job_id)); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 100); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .prepare_generation, 0)); + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.verifyGraphMetricBuildPublishReady("degree", active_job.job_id)); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 1, 200); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .scan_edges_and_out_degree, 0)); + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.verifyGraphMetricBuildPublishReady("degree", active_job.job_id)); + + _ = try graph.claimGraphMetricBuildPageAt("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-a", 3000); + _ = try graph.completeGraphMetricBuildPage("degree", active_job.job_id, .reduce_ranks, 0, 2, "worker-a", 1, 300); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("degree", active_job.job_id, .reduce_ranks, 0)); + + const verification = try graph.verifyGraphMetricBuildPublishReady("degree", active_job.job_id); + try std.testing.expectEqual(active_job.job_id, verification.job_id); + try std.testing.expectEqual(graph.edge_generation, verification.target_generation); + try std.testing.expectEqual(graph.edge_generation, verification.score_generation); + try std.testing.expectEqual(GraphIndex.graphMetricConfigFingerprint(metrics[0]), verification.config_fingerprint); + try std.testing.expectEqual(@as(u64, 3), verification.expected_phases); + try std.testing.expectEqual(@as(u64, 3), verification.completed_phases); + try std.testing.expectEqual(@as(u64, 3), verification.expected_pages); + try std.testing.expectEqual(@as(u64, 3), verification.completed_pages); + try std.testing.expectEqual(@as(u64, 100 ^ 200 ^ 300), verification.output_fingerprint); + try std.testing.expect(!verification.converged); + try std.testing.expect(!verification.fixed_iteration_limit); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var manifest = try graph.metricBuildManifest(&batch, "degree", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + manifest.config_fingerprint += 1; + try graph.putGraphMetricBuildManifestInBatch(&batch, "degree", manifest); + try batch.commit(); + } + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.verifyGraphMetricBuildPublishReady("degree", active_job.job_id)); +} + +test "graph metric build publish verification records iterative convergence readiness" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-publish-verify-pagerank"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-publish-verify-pagerank"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.00001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("pagerank", try graph.graphMetricCurrentGeneration("pagerank")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .prepare_generation, 0, 0, "worker-a", 1, 11); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .prepare_generation, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 1, 22); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .scan_edges_and_out_degree, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-a", 3000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .initialize_ranks, 0, 2, "worker-a", 2, 33); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .initialize_ranks, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .iterate_contributions, 0, 3, "worker-a", 4000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .iterate_contributions, 0, 3, "worker-a", 1, 44); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .iterate_contributions, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .reduce_ranks, 0, 4, "worker-a", 5000); + _ = try graph.completeGraphMetricBuildPage("pagerank", active_job.job_id, .reduce_ranks, 0, 4, "worker-a", 2, 55); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .reduce_ranks, 0)); + + _ = try graph.claimGraphMetricBuildPageAt("pagerank", active_job.job_id, .check_convergence, 0, 5, "worker-a", 6000); + _ = try graph.completeGraphMetricBuildConvergencePage("pagerank", active_job.job_id, 0, 5, "worker-a", 2, 66, 0.125, 0.25, 1.0, false); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("pagerank", active_job.job_id, .check_convergence, 0)); + + const verification = try graph.verifyGraphMetricBuildPublishReady("pagerank", active_job.job_id); + try std.testing.expectEqual(active_job.job_id, verification.job_id); + try std.testing.expectEqual(@as(u64, 6), verification.expected_phases); + try std.testing.expectEqual(@as(u64, 6), verification.completed_phases); + try std.testing.expectEqual(@as(u64, 6), verification.expected_pages); + try std.testing.expectEqual(@as(u64, 6), verification.completed_pages); + try std.testing.expectEqual(@as(u64, 11 ^ 22 ^ 33 ^ 44 ^ 55 ^ 66), verification.output_fingerprint); + try std.testing.expect(!verification.converged); + try std.testing.expect(verification.fixed_iteration_limit); + try std.testing.expectApproxEqAbs(@as(f64, 0.125), verification.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.25), verification.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), verification.rank_sum, 0.0000001); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const iteration_summary = try graph.metricBuildIterationSummary(&txn, "pagerank", active_job.job_id, 0) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + try std.testing.expect(iteration_summary.fixed_iteration_limit); + try std.testing.expectEqual(@as(u64, 66), iteration_summary.output_fingerprint); + } +} + +test "graph pagerank metric edge filter limits typed score graph" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-filter"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-filter"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .max_iterations = 30, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + var published = try graph.runPageRankMetric("pagerank"); + defer published.deinit(alloc); + + const top = try graph.graphMetricTopK("pagerank", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + for (top) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-x")); + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-y")); + } +} + +test "graph degree metric publishes total incident degree scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-d", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(@as(u32, 1), published.iterations_completed); + try std.testing.expect(published.converged); + + const top = try graph.graphMetricTopK("degree", 4); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 4), top.len); + try std.testing.expectEqualStrings("doc-b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 3.0), top[0].score, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), top[1].score, 0.001); +} + +test "graph degree planned build publishes scores matching local runner" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-planned"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-planned"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "degree_local", + .kind = .degree, + .refresh = .manual, + }, + .{ + .name = "degree_planned", + .kind = .degree, + .refresh = .manual, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-d", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-d", "doc-a", "related", 1.0, 0, 0, ""); + + var local_status = try graph.runGraphMetric("degree_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runDegreeMetricPlanned("degree_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(@as(u64, 2), planned_status.last_event.?.score_count); + try std.testing.expectEqual(graph.edge_generation, planned_status.published_generation); + try std.testing.expectEqual(graph.edge_generation, planned_status.edge_generation); + try std.testing.expectEqual(@as(u32, 1), planned_status.iterations_completed); + try std.testing.expect(planned_status.converged); + + const local_top = try graph.graphMetricTopK("degree_local", 10); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("degree_planned", 10); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree_planned") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); + try std.testing.expectEqual(@as(u64, 0), job.lease_expires_at_ms); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree_planned", job.job_id)) == null); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree_planned", job.job_id, .scan_edges_and_out_degree, 0, 1)) == null); + try std.testing.expect((try graph.metricBuildPhaseSummary(&txn, "degree_planned", job.job_id, .scan_edges_and_out_degree, 0)) == null); + } +} + +test "graph degree planned build executes partitioned scan pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-planned-partitions"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-planned-partitions"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "degree_local", + .kind = .degree, + .refresh = .manual, + }, + .{ + .name = "degree_planned", + .kind = .degree, + .refresh = .manual, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + + var local_status = try graph.runGraphMetric("degree_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runDegreeMetricPlanned("degree_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(graph.edge_generation, planned_status.published_generation); + + const local_top = try graph.graphMetricTopK("degree_local", 10); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("degree_planned", 10); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + try std.testing.expectEqualStrings("hub", planned_top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, @floatFromInt(graph_metric_build_target_scan_page_units + 1)), planned_top[0].score, 0.0000001); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "graph degree planned worker step drives scheduled phases" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-worker-step"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-worker-step"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + const prepare = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(prepare.advanced_phase); + try std.testing.expect(!prepare.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + } + + const scan = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + try std.testing.expect(scan.advanced_phase); + try std.testing.expect(!scan.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + } + + const reduce = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce.phase); + try std.testing.expect(reduce.claimed_page); + try std.testing.expect(reduce.completed_page); + try std.testing.expect(reduce.advanced_phase); + try std.testing.expect(!reduce.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } + + const publish = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(!publish.published); + try std.testing.expect(publish.advanced_phase); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + _ = try graph.metricBuildManifest(&txn, "degree", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + const published_before_cleanup = try graph.graphMetricTopK("degree", 10); + defer { + for (published_before_cleanup) |*score| score.deinit(alloc); + alloc.free(published_before_cleanup); + } + try std.testing.expectEqual(@as(usize, 2), published_before_cleanup.len); + + const partial_cleanup = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, partial_cleanup.phase); + try std.testing.expect(partial_cleanup.claimed_page); + try std.testing.expect(partial_cleanup.completed_page); + try std.testing.expect(!partial_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + const cleanup_page = try graph.metricBuildPage(&txn, "degree", job.job_id, .cleanup_old_generations, 0, partial_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, cleanup_page.state); + _ = try graph.metricBuildManifest(&txn, "degree", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + + const cleanup = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(cleanup.claimed_page); + try std.testing.expect(cleanup.completed_page); + try std.testing.expect(cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", job.job_id)) == null); + } + + var fresh = try graph.graphMetricStatus("degree"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(graph.edge_generation, fresh.published_generation); +} + +test "graph degree planned worker and coordinator steps survive reopened handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-reopened-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-reopened-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var target_buf: [64]u8 = undefined; + const target = try std.fmt.bufPrint(&target_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge("hub", target, "cites", 1.0, 0, 0, ""); + } + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + + const step = try worker.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + const step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, step.phase); + try std.testing.expect(step.advanced_phase); + try std.testing.expect(!step.claimed_page); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker_a = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker_a.close(); + + const step = try worker_a.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-scan-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker_b = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker_b.close(); + + const step = try worker_b.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-scan-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + const step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, step.phase); + try std.testing.expect(step.advanced_phase); + try std.testing.expect(!step.claimed_page); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker_a = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker_a.close(); + + const step = try worker_a.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-reduce-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker_b = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker_b.close(); + + const step = try worker_b.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-reduce-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + + // Planned workers write only their disjoint score pages. The bounded + // global rank tier is derived once by the coordinator immediately + // before the generation pointer becomes visible. + var txn = try worker_b.beginReadReverseTxn(); + defer txn.abort(); + const active_job = try worker_b.metricBuildJob(&txn, "degree") orelse + return error.TestExpectedGraphMetricBuildJob; + const rank_prefix = try worker_b.graphMetricRankPrefixAlloc("degree", active_job.score_generation); + defer alloc.free(rank_prefix); + try std.testing.expectEqual(@as(usize, 0), try GraphIndex.countKeysWithPrefix(&txn, rank_prefix)); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + const reduce_step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_step.phase); + try std.testing.expect(reduce_step.advanced_phase); + + const worker_publish_step = try coordinator.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish_step.phase); + try std.testing.expect(!worker_publish_step.claimed_page); + try std.testing.expect(!worker_publish_step.advanced_phase); + try std.testing.expect(!worker_publish_step.published); + + const publish_step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish_step.phase); + try std.testing.expect(publish_step.advanced_phase); + + var published = try coordinator.graphMetricStatus("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, published.phase); + try std.testing.expect(published.published_generation > 0); + + const visible = try coordinator.graphMetricTopK("degree", 1); + defer { + for (visible) |*score| score.deinit(alloc); + alloc.free(visible); + } + try std.testing.expectEqual(@as(usize, 1), visible.len); + try std.testing.expectEqualStrings("hub", visible[0].node); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + + var completed = false; + for (0..8) |_| { + const partial_cleanup = try worker.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-clean-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, partial_cleanup.phase); + try std.testing.expect(partial_cleanup.claimed_page); + try std.testing.expect(!partial_cleanup.advanced_phase); + try std.testing.expect(!partial_cleanup.published); + if (partial_cleanup.completed_page) { + completed = true; + break; + } + } + try std.testing.expect(completed); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + + var completed = false; + for (0..8) |_| { + const final_cleanup = try worker.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-clean-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.claimed_page); + try std.testing.expect(!final_cleanup.advanced_phase); + if (final_cleanup.published) { + try std.testing.expect(final_cleanup.completed_page); + completed = true; + break; + } + } + try std.testing.expect(completed); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(@as(u64, 0), status.build_job_id); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + + const top = try graph.graphMetricTopK("degree", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("hub", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, @floatFromInt(graph_metric_build_target_scan_page_units + 1)), top[0].score, 0.0000001); + } +} + +test "graph degree planned public build ensure drives reopened workers" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-public-ensure"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-public-ensure"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + + var target_generation: u64 = 0; + var build_job_id: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, started.phase); + try std.testing.expectEqual(target_generation, started.building_generation); + try std.testing.expect(started.build_job_id != 0); + build_job_id = started.build_job_id; + + var repeated = try graph.ensureGraphMetricPlannedBuild("degree", target_generation); + defer repeated.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, repeated.state); + try std.testing.expectEqual(build_job_id, repeated.build_job_id); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + var status = try coordinator.ensureGraphMetricPlannedBuild("degree", target_generation); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(build_job_id, status.build_job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, status.phase); + } + + var finished = false; + for (0..80) |step_i| { + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + + const worker_id = if (step_i % 2 == 0) "worker-public-a" else "worker-public-b"; + const worker_step = try worker.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], worker_id); + try std.testing.expect(!worker_step.advanced_phase); + if (worker_step.completed_build) { + finished = true; + break; + } + } + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + const coordinator_step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expect(!coordinator_step.claimed_page); + if (coordinator_step.completed_build) { + finished = true; + break; + } + } + } + try std.testing.expect(finished); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(u64, 0), status.build_job_id); + + const top = try graph.graphMetricTopK("degree", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("hub", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, @floatFromInt(graph_metric_build_target_scan_page_units + 1)), top[0].score, 0.0000001); + } +} + +test "graph degree planned name-only public steps require coordinator across worker pages" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-public-name-only-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-public-name-only-workers"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + const target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + const prepare_worker = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare_worker.phase); + try std.testing.expect(prepare_worker.claimed_page); + try std.testing.expect(prepare_worker.completed_page); + try std.testing.expect(!prepare_worker.advanced_phase); + try std.testing.expect(!prepare_worker.published); + + const prepare_coordinator = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare_coordinator.phase); + try std.testing.expect(prepare_coordinator.advanced_phase); + + const scan_a = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-scan-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan_a.phase); + try std.testing.expect(scan_a.claimed_page); + try std.testing.expect(scan_a.completed_page); + try std.testing.expect(!scan_a.advanced_phase); + + const scan_not_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan_not_ready.phase); + try std.testing.expect(!scan_not_ready.advanced_phase); + try std.testing.expect(!scan_not_ready.published); + + const scan_b = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-scan-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan_b.phase); + try std.testing.expect(scan_b.claimed_page); + try std.testing.expect(scan_b.completed_page); + try std.testing.expect(scan_b.page_id != scan_a.page_id); + try std.testing.expect(!scan_b.advanced_phase); + + const scan_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan_ready.phase); + try std.testing.expect(scan_ready.advanced_phase); + + const reduce_a = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-reduce-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_a.phase); + try std.testing.expect(reduce_a.claimed_page); + try std.testing.expect(reduce_a.completed_page); + try std.testing.expect(!reduce_a.advanced_phase); + + const reduce_not_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_not_ready.phase); + try std.testing.expect(!reduce_not_ready.advanced_phase); + try std.testing.expect(!reduce_not_ready.published); + + const reduce_b = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-reduce-b"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_b.phase); + try std.testing.expect(reduce_b.claimed_page); + try std.testing.expect(reduce_b.completed_page); + try std.testing.expect(reduce_b.page_id != reduce_a.page_id); + try std.testing.expect(!reduce_b.advanced_phase); + + const reduce_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_ready.phase); + try std.testing.expect(reduce_ready.advanced_phase); + + const worker_publish = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish.phase); + try std.testing.expect(!worker_publish.claimed_page); + try std.testing.expect(!worker_publish.advanced_phase); + try std.testing.expect(!worker_publish.published); + + const coordinator_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish.phase); + try std.testing.expect(coordinator_publish.advanced_phase); + + var published_during_cleanup = try graph.graphMetricStatus("degree"); + defer published_during_cleanup.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, published_during_cleanup.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, published_during_cleanup.phase); + try std.testing.expectEqual(target_generation, published_during_cleanup.published_generation); + + var cleanup_finished = false; + for (0..8) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(cleanup.claimed_page); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + + var fresh = try graph.graphMetricStatus("degree"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, fresh.phase); + try std.testing.expectEqual(target_generation, fresh.published_generation); + + const top = try graph.graphMetricTopK("degree", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("hub", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, @floatFromInt(graph_metric_build_target_scan_page_units + 1)), top[0].score, 0.0000001); +} + +const GraphDegreePublicWorkerThreadCtx = struct { + store_path: [*:0]const u8, + rev_path: [*:0]const u8, + worker_id: []const u8, + err: ?anyerror = null, + phase: GraphIndex.GraphMetricBuildPhase = .idle, + page_id: u64 = 0, + claimed_page: bool = false, + completed_page: bool = false, + advanced_phase: bool = false, + published: bool = false, +}; + +fn runGraphDegreePublicWorkerThread(ctx: *GraphDegreePublicWorkerThreadCtx) void { + const alloc = std.heap.page_allocator; + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var store = docstore.DocStore.open(alloc, ctx.store_path, .{}) catch |err| { + ctx.err = err; + return; + }; + defer store.close(); + var graph = openTestGraphIndex(alloc, &store, ctx.rev_path, "links", .{ .metric_configs = &metrics }) catch |err| { + ctx.err = err; + return; + }; + defer graph.close(); + + runGraphDegreePublicWorkerStepForTest(&graph, ctx) catch |err| { + ctx.err = err; + return; + }; +} + +fn runGraphDegreePublicWorkerStepForTest(graph: *GraphIndex, ctx: *GraphDegreePublicWorkerThreadCtx) !void { + const step = graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", ctx.worker_id) catch |err| { + ctx.err = err; + return; + }; + ctx.phase = step.phase; + ctx.page_id = step.page_id; + ctx.claimed_page = step.claimed_page; + ctx.completed_page = step.completed_page; + ctx.advanced_phase = step.advanced_phase; + ctx.published = step.published; +} + +fn recordGraphDegreePublicWorkerThreadResult( + pages: *[8]u64, + page_count: *usize, + ctx: GraphDegreePublicWorkerThreadCtx, + expected_phase: GraphIndex.GraphMetricBuildPhase, +) !void { + if (ctx.err) |err| return err; + try std.testing.expectEqual(expected_phase, ctx.phase); + try std.testing.expect(!ctx.advanced_phase); + if (!ctx.completed_page) return; + try std.testing.expect(ctx.claimed_page); + for (pages[0..page_count.*]) |page_id| { + if (page_id == ctx.page_id) return; + } + pages[page_count.*] = ctx.page_id; + page_count.* += 1; +} + +const GraphDegreeClaimWorkerThreadCtx = struct { + store_path: [*:0]const u8, + rev_path: [*:0]const u8, + job_id: u64, + worker_id: []const u8, + now_ms: u64, + requested_page_id: u64, + err: ?anyerror = null, + page_id: u64 = 0, + attempt: u64 = 0, + lease_expires_at_ms: u64 = 0, +}; + +fn runGraphDegreeClaimWorkerThread(ctx: *GraphDegreeClaimWorkerThreadCtx) void { + const alloc = std.heap.page_allocator; + var store = docstore.DocStore.open(alloc, ctx.store_path, .{}) catch |err| { + ctx.err = err; + return; + }; + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = openTestGraphIndex(alloc, &store, ctx.rev_path, "links", .{ .metric_configs = &metrics }) catch |err| { + ctx.err = err; + return; + }; + defer graph.close(); + + const page = graph.claimGraphMetricBuildPageAt("degree", ctx.job_id, .scan_edges_and_out_degree, 0, ctx.requested_page_id, ctx.worker_id, ctx.now_ms) catch |err| { + ctx.err = err; + return; + } orelse { + ctx.err = error.TestExpectedGraphMetricBuildPage; + return; + }; + ctx.page_id = page.page_id; + ctx.attempt = page.attempt; + ctx.lease_expires_at_ms = page.lease_expires_at_ms; +} + +const GraphMetricPublicWorkerThreadCtx = struct { + store_path: [*:0]const u8, + rev_path: [*:0]const u8, + metric_name: []const u8, + kind: GraphMetricKind, + worker_id: []const u8, + err: ?anyerror = null, + phase: GraphIndex.GraphMetricBuildPhase = .idle, + page_id: u64 = 0, + claimed_page: bool = false, + completed_page: bool = false, + advanced_phase: bool = false, + published: bool = false, +}; + +fn runGraphMetricPublicWorkerThread(ctx: *GraphMetricPublicWorkerThreadCtx) void { + const alloc = std.heap.page_allocator; + var store = docstore.DocStore.open(alloc, ctx.store_path, .{}) catch |err| { + ctx.err = err; + return; + }; + defer store.close(); + if (ctx.kind == .hits_authority or ctx.kind == .hits_hub) { + const pair_name = if (ctx.kind == .hits_authority) "hits_hub" else "hits_authority"; + const pair_kind: GraphMetricKind = if (ctx.kind == .hits_authority) .hits_hub else .hits_authority; + const metrics = [_]GraphMetricConfig{ + .{ + .name = ctx.metric_name, + .kind = ctx.kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = pair_name, + .kind = pair_kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = openTestGraphIndex(alloc, &store, ctx.rev_path, "links", .{ .metric_configs = &metrics }) catch |err| { + ctx.err = err; + return; + }; + defer graph.close(); + + runGraphMetricPublicWorkerStepForTest(&graph, ctx) catch |err| { + ctx.err = err; + return; + }; + } else { + const metrics = [_]GraphMetricConfig{.{ + .name = ctx.metric_name, + .kind = ctx.kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = openTestGraphIndex(alloc, &store, ctx.rev_path, "links", .{ .metric_configs = &metrics }) catch |err| { + ctx.err = err; + return; + }; + defer graph.close(); + + runGraphMetricPublicWorkerStepForTest(&graph, ctx) catch |err| { + ctx.err = err; + return; + }; + } +} + +fn runGraphMetricPublicWorkerStepForTest(graph: *GraphIndex, ctx: *GraphMetricPublicWorkerThreadCtx) !void { + const step = try graph.runGraphMetricPlannedWorkerPageStepForMetric(ctx.metric_name, ctx.worker_id); + ctx.phase = step.phase; + ctx.page_id = step.page_id; + ctx.claimed_page = step.claimed_page; + ctx.completed_page = step.completed_page; + ctx.advanced_phase = step.advanced_phase; + ctx.published = step.published; +} + +fn runGraphMetricPublicWorkerPairForTest( + alloc: Allocator, + store_path: [*:0]const u8, + rev_path: [*:0]const u8, + worker_ctx_a: *GraphMetricPublicWorkerThreadCtx, + worker_ctx_b: *GraphMetricPublicWorkerThreadCtx, +) !void { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + + if (worker_ctx_a.kind == .hits_authority or worker_ctx_a.kind == .hits_hub) { + const pair_name = if (worker_ctx_a.kind == .hits_authority) "hits_hub" else "hits_authority"; + const pair_kind: GraphMetricKind = if (worker_ctx_a.kind == .hits_authority) .hits_hub else .hits_authority; + const metrics = [_]GraphMetricConfig{ + .{ + .name = worker_ctx_a.metric_name, + .kind = worker_ctx_a.kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = pair_name, + .kind = pair_kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try runGraphMetricPublicWorkerStepForTest(&graph, worker_ctx_a); + try runGraphMetricPublicWorkerStepForTest(&graph, worker_ctx_b); + } else { + const metrics = [_]GraphMetricConfig{.{ + .name = worker_ctx_a.metric_name, + .kind = worker_ctx_a.kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try runGraphMetricPublicWorkerStepForTest(&graph, worker_ctx_a); + try runGraphMetricPublicWorkerStepForTest(&graph, worker_ctx_b); + } +} + +fn recordGraphMetricPublicWorkerThreadResult( + pages: *[8]u64, + page_count: *usize, + ctx: GraphMetricPublicWorkerThreadCtx, + expected_phase: GraphIndex.GraphMetricBuildPhase, +) !void { + if (ctx.err) |err| return err; + try std.testing.expectEqual(expected_phase, ctx.phase); + try std.testing.expect(!ctx.advanced_phase); + try std.testing.expect(!ctx.published); + if (!ctx.completed_page) return; + try std.testing.expect(ctx.claimed_page); + for (pages[0..page_count.*]) |page_id| { + if (page_id == ctx.page_id) return; + } + pages[page_count.*] = ctx.page_id; + page_count.* += 1; +} + +fn expectGraphMetricConcurrentPublicPhase( + alloc: Allocator, + store_path: [*:0]const u8, + rev_path: [*:0]const u8, + metric_name: []const u8, + kind: GraphMetricKind, + expected_phase: GraphIndex.GraphMetricBuildPhase, + expected_pages: usize, +) !void { + const total_expected = expected_pages + if (kind != .degree and + (expected_phase == .initialize_ranks or expected_phase == .reduce_ranks or expected_phase == .hits_hub_reduce_ranks)) + expected_pages + 1 + else + @as(usize, 0); + var pages: [8]u64 = undefined; + var page_count: usize = 0; + for (0..12) |round| { + if (page_count >= total_expected) break; + const worker_a = try std.fmt.allocPrint(alloc, "worker-{s}-a-{d}", .{ @tagName(expected_phase), round }); + defer alloc.free(worker_a); + const worker_b = try std.fmt.allocPrint(alloc, "worker-{s}-b-{d}", .{ @tagName(expected_phase), round }); + defer alloc.free(worker_b); + var worker_ctx_a = GraphMetricPublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .metric_name = metric_name, .kind = kind, .worker_id = worker_a }; + var worker_ctx_b = GraphMetricPublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .metric_name = metric_name, .kind = kind, .worker_id = worker_b }; + try runGraphMetricPublicWorkerPairForTest(alloc, store_path, rev_path, &worker_ctx_a, &worker_ctx_b); + try recordGraphMetricPublicWorkerThreadResult(&pages, &page_count, worker_ctx_a, expected_phase); + try recordGraphMetricPublicWorkerThreadResult(&pages, &page_count, worker_ctx_b, expected_phase); + } + try std.testing.expectEqual(total_expected, page_count); +} + +fn materializeGraphMetricPublishPagesForTest( + graph: *GraphIndex, + metric_name: []const u8, + worker_id: []const u8, +) !usize { + var page_count: usize = 0; + while (page_count < graph_metric_build_max_partition_pages) { + const step = try graph.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, worker_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, step.phase); + if (!step.claimed_page) break; + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.advanced_phase); + try std.testing.expect(!step.published); + page_count += 1; + } + try std.testing.expect(page_count > 0); + return page_count; +} + +test "graph degree planned public workers claim phase pages from concurrent handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-concurrent-public-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-concurrent-public-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + } + + var scan_pages: [8]u64 = undefined; + var scan_page_count: usize = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + for (0..8) |round| { + if (scan_page_count >= 2) break; + const worker_a = try std.fmt.allocPrint(alloc, "worker-scan-a-{d}", .{round}); + defer alloc.free(worker_a); + const worker_b = try std.fmt.allocPrint(alloc, "worker-scan-b-{d}", .{round}); + defer alloc.free(worker_b); + var scan_a = GraphDegreePublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .worker_id = worker_a }; + var scan_b = GraphDegreePublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .worker_id = worker_b }; + try runGraphDegreePublicWorkerStepForTest(&graph, &scan_a); + try runGraphDegreePublicWorkerStepForTest(&graph, &scan_b); + try recordGraphDegreePublicWorkerThreadResult(&scan_pages, &scan_page_count, scan_a, .scan_edges_and_out_degree); + try recordGraphDegreePublicWorkerThreadResult(&scan_pages, &scan_page_count, scan_b, .scan_edges_and_out_degree); + } + } + try std.testing.expectEqual(@as(usize, 2), scan_page_count); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + const scan_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan_ready.phase); + try std.testing.expect(scan_ready.advanced_phase); + } + + var reduce_pages: [8]u64 = undefined; + var reduce_page_count: usize = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + for (0..8) |round| { + if (reduce_page_count >= 2) break; + const worker_a = try std.fmt.allocPrint(alloc, "worker-reduce-a-{d}", .{round}); + defer alloc.free(worker_a); + const worker_b = try std.fmt.allocPrint(alloc, "worker-reduce-b-{d}", .{round}); + defer alloc.free(worker_b); + var reduce_a = GraphDegreePublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .worker_id = worker_a }; + var reduce_b = GraphDegreePublicWorkerThreadCtx{ .store_path = store_path, .rev_path = rev_path, .worker_id = worker_b }; + try runGraphDegreePublicWorkerStepForTest(&graph, &reduce_a); + try runGraphDegreePublicWorkerStepForTest(&graph, &reduce_b); + try recordGraphDegreePublicWorkerThreadResult(&reduce_pages, &reduce_page_count, reduce_a, .reduce_ranks); + try recordGraphDegreePublicWorkerThreadResult(&reduce_pages, &reduce_page_count, reduce_b, .reduce_ranks); + } + } + try std.testing.expectEqual(@as(usize, 2), reduce_page_count); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + const reduce_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, reduce_ready.phase); + try std.testing.expect(reduce_ready.advanced_phase); + + const worker_publish = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish.phase); + try std.testing.expect(!worker_publish.claimed_page); + try std.testing.expect(!worker_publish.advanced_phase); + try std.testing.expect(!worker_publish.published); + + const coordinator_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish.phase); + try std.testing.expect(coordinator_publish.advanced_phase); + + var cleanup_finished = false; + for (0..8) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + + var fresh = try graph.graphMetricStatus("degree"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(target_generation, fresh.published_generation); + try std.testing.expectEqual(@as(usize, 0), fresh.build_pages.len); + } +} + +test "graph degree planned status reports concurrent worker page leases from reopened handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-concurrent-status-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-concurrent-status-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var job_id: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + var started = try graph.ensureGraphMetricPlannedBuild("degree", try graph.graphMetricCurrentGeneration("degree")); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + job_id = started.build_job_id; + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + } + + var worker_a = GraphDegreeClaimWorkerThreadCtx{ + .store_path = store_path, + .rev_path = rev_path, + .job_id = job_id, + .worker_id = "worker-status-a", + .now_ms = 10_000, + .requested_page_id = 1, + }; + var worker_b = GraphDegreeClaimWorkerThreadCtx{ + .store_path = store_path, + .rev_path = rev_path, + .job_id = job_id, + .worker_id = "worker-status-b", + .now_ms = 10_001, + .requested_page_id = 2, + }; + runGraphDegreeClaimWorkerThread(&worker_a); + if (worker_a.err) |err| return err; + runGraphDegreeClaimWorkerThread(&worker_b); + if (worker_b.err) |err| return err; + try std.testing.expect(worker_a.page_id != worker_b.page_id); + try std.testing.expectEqual(@as(u64, 1), worker_a.attempt); + try std.testing.expectEqual(@as(u64, 1), worker_b.attempt); + try std.testing.expect(worker_a.lease_expires_at_ms > worker_a.now_ms); + try std.testing.expect(worker_b.lease_expires_at_ms > worker_b.now_ms); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.phase); + try std.testing.expectEqual(job_id, status.build_job_id); + try std.testing.expect(!status.build_pages_truncated); + try std.testing.expectEqual(@as(usize, 2), status.build_pages.len); + + var saw_worker_a = false; + var saw_worker_b = false; + for (status.build_pages) |page| { + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, page.phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.reverse_edges, page.range_kind); + try std.testing.expectEqual(@as(u32, 0), page.iteration); + try std.testing.expectEqual(@as(u64, 1), page.attempt); + try std.testing.expect(page.lease_expires_at_ms > 0); + try std.testing.expectEqualStrings("", page.last_error); + if (std.mem.eql(u8, page.worker_id, "worker-status-a")) { + saw_worker_a = true; + try std.testing.expectEqual(worker_a.page_id, page.page_id); + try std.testing.expectEqual(worker_a.lease_expires_at_ms, page.lease_expires_at_ms); + } else if (std.mem.eql(u8, page.worker_id, "worker-status-b")) { + saw_worker_b = true; + try std.testing.expectEqual(worker_b.page_id, page.page_id); + try std.testing.expectEqual(worker_b.lease_expires_at_ms, page.lease_expires_at_ms); + } else { + return error.TestUnexpectedGraphMetricBuildWorker; + } + } + try std.testing.expect(saw_worker_a); + try std.testing.expect(saw_worker_b); + + const blocked = try graph.runGraphMetricPlannedCoordinatorStepForMetric("degree"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, blocked.phase); + try std.testing.expect(!blocked.advanced_phase); + try std.testing.expect(!blocked.published); +} + +test "graph pagerank planned public workers claim partitioned phases from concurrent handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-concurrent-public-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-concurrent-public-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + _ = try graph.metricBuildPage(&txn, "pagerank", job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank", job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank", job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "pagerank", job.job_id, .initialize_ranks, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + } + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + } + + const partitioned_phases = [_]GraphIndex.GraphMetricBuildPhase{ + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + }; + for (partitioned_phases) |phase| { + try expectGraphMetricConcurrentPublicPhase(alloc, store_path, rev_path, "pagerank", .pagerank, phase, 2); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + const phase_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(phase, phase_ready.phase); + try std.testing.expect(phase_ready.advanced_phase); + } + + try expectGraphMetricConcurrentPublicPhase( + alloc, + store_path, + rev_path, + "pagerank", + .pagerank, + .publish_generation, + 2, + ); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + const worker_publish = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish.phase); + try std.testing.expect(!worker_publish.claimed_page); + try std.testing.expect(!worker_publish.advanced_phase); + try std.testing.expect(!worker_publish.published); + + const coordinator_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish.phase); + try std.testing.expect(coordinator_publish.advanced_phase); + + const duplicate_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate_publish.phase); + try std.testing.expect(!duplicate_publish.advanced_phase); + try std.testing.expect(!duplicate_publish.published); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + + var fresh = try graph.graphMetricStatus("pagerank"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(target_generation, fresh.published_generation); + try std.testing.expectEqual(@as(u32, 1), fresh.iterations_completed); + try std.testing.expectEqual(@as(usize, 0), fresh.build_pages.len); + try std.testing.expectEqual(@as(usize, 1), fresh.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, fresh.recent_events[0].kind); + + const after_complete = try graph.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + const top = try graph.graphMetricTopK("pagerank", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("hub", top[0].node); + } +} + +test "graph pagerank reopened coordinators do not duplicate publish" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-pagerank-reopened-coordinator-publish-race"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-pagerank-reopened-coordinator-publish-race"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + var setup_steps: usize = 0; + while (setup_steps < 64) : (setup_steps += 1) { + const step = try graph.runGraphMetricPlannedWorkerStep("pagerank", metrics[0], "worker-a"); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.published); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + if (job.phase == .publish_generation) break; + } + try std.testing.expect(setup_steps < 64); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "pagerank") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + _ = try materializeGraphMetricPublishPagesForTest(&coordinator, "pagerank", "worker-publish-materialize"); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + var status = try coordinator.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var duplicate_coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + var status = try duplicate_coordinator.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try duplicate_coordinator.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var reader = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer reader.close(); + + var fresh = try reader.graphMetricStatus("pagerank"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(target_generation, fresh.published_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, fresh.phase); + try std.testing.expectEqual(@as(usize, 1), fresh.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, fresh.recent_events[0].kind); + + const after_complete = try reader.runGraphMetricPlannedCoordinatorStepForMetric("pagerank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + var after_duplicate = try reader.graphMetricStatus("pagerank"); + defer after_duplicate.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), after_duplicate.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, after_duplicate.recent_events[0].kind); + } +} + +test "graph eigenvector reopened coordinators do not duplicate publish" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-reopened-coordinator-publish-race"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-reopened-coordinator-publish-race"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "hub", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "hub", "cites", 1.0, 0, 0, ""); + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + var setup_steps: usize = 0; + while (setup_steps < 64) : (setup_steps += 1) { + const step = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-a"); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.published); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + if (job.phase == .publish_generation) break; + } + try std.testing.expect(setup_steps < 64); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + _ = try materializeGraphMetricPublishPagesForTest(&coordinator, "eigenvector", "worker-publish-materialize"); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + var status = try coordinator.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var duplicate_coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + var status = try duplicate_coordinator.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try duplicate_coordinator.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var reader = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer reader.close(); + + var fresh = try reader.graphMetricStatus("eigenvector"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(target_generation, fresh.published_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, fresh.phase); + try std.testing.expectEqual(@as(usize, 1), fresh.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, fresh.recent_events[0].kind); + + const after_complete = try reader.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + var after_duplicate = try reader.graphMetricStatus("eigenvector"); + defer after_duplicate.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), after_duplicate.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, after_duplicate.recent_events[0].kind); + } +} + +test "graph hits reopened coordinators do not duplicate paired publish" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-reopened-coordinator-publish-race"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-reopened-coordinator-publish-race"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + var setup_steps: usize = 0; + while (setup_steps < 64) : (setup_steps += 1) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.published); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + if (job.phase == .publish_generation) break; + } + try std.testing.expect(setup_steps < 64); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + + _ = try materializeGraphMetricPublishPagesForTest(&coordinator, "hits_authority", "worker-publish-materialize"); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + var authority = try coordinator.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try coordinator.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var duplicate_coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + var authority = try duplicate_coordinator.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try duplicate_coordinator.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try duplicate_coordinator.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var reader = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer reader.close(); + + var authority = try reader.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try reader.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, hub.state); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, authority.phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, hub.phase); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + + const after_complete = try reader.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + var after_authority = try reader.graphMetricStatus("hits_authority"); + defer after_authority.deinit(alloc); + var after_hub = try reader.graphMetricStatus("hits_hub"); + defer after_hub.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), after_authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), after_hub.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, after_authority.recent_events[0].kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, after_hub.recent_events[0].kind); + } +} + +test "graph eigenvector planned public workers claim partitioned phases from concurrent handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-concurrent-public-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-concurrent-public-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + _ = try graph.metricBuildPage(&txn, "eigenvector", job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "eigenvector", job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "eigenvector", job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "eigenvector", job.job_id, .initialize_ranks, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + } + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + } + + const partitioned_phases = [_]GraphIndex.GraphMetricBuildPhase{ + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + }; + for (partitioned_phases) |phase| { + try expectGraphMetricConcurrentPublicPhase(alloc, store_path, rev_path, "eigenvector", .eigenvector, phase, 2); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + const phase_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(phase, phase_ready.phase); + try std.testing.expect(phase_ready.advanced_phase); + } + + try expectGraphMetricConcurrentPublicPhase( + alloc, + store_path, + rev_path, + "eigenvector", + .eigenvector, + .publish_generation, + 2, + ); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + const worker_publish = try graph.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish.phase); + try std.testing.expect(!worker_publish.claimed_page); + try std.testing.expect(!worker_publish.advanced_phase); + try std.testing.expect(!worker_publish.published); + + const coordinator_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish.phase); + try std.testing.expect(coordinator_publish.advanced_phase); + + const duplicate_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate_publish.phase); + try std.testing.expect(!duplicate_publish.advanced_phase); + try std.testing.expect(!duplicate_publish.published); + + var cleanup_finished = false; + for (0..8) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + + var fresh = try graph.graphMetricStatus("eigenvector"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(target_generation, fresh.published_generation); + try std.testing.expectEqual(@as(u32, 1), fresh.iterations_completed); + try std.testing.expectEqual(@as(usize, 0), fresh.build_pages.len); + try std.testing.expectEqual(@as(usize, 1), fresh.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, fresh.recent_events[0].kind); + + const after_complete = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + const top = try graph.graphMetricTopK("eigenvector", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + } +} + +test "graph hits planned public workers claim partitioned paired phases from concurrent handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-concurrent-public-workers"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-concurrent-public-workers"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var target_generation: u64 = 0; + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + } + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + target_generation = graph.edge_generation; + + var started = try graph.ensureGraphMetricPlannedBuild("hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + _ = try graph.metricBuildPage(&txn, "hits_authority", job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "hits_authority", job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "hits_authority", job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPage(&txn, "hits_authority", job.job_id, .initialize_ranks, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + } + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + } + + const hits_partitioned_phases = [_]GraphIndex.GraphMetricBuildPhase{ + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + }; + for (hits_partitioned_phases) |phase| { + try expectGraphMetricConcurrentPublicPhase(alloc, store_path, rev_path, "hits_authority", .hits_authority, phase, 2); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + const phase_ready = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(phase, phase_ready.phase); + try std.testing.expect(phase_ready.advanced_phase); + } + + try expectGraphMetricConcurrentPublicPhase( + alloc, + store_path, + rev_path, + "hits_authority", + .hits_authority, + .publish_generation, + 2, + ); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ + .metric_configs = &metrics, + }); + defer graph.close(); + + const worker_publish = try graph.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-publish"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, worker_publish.phase); + try std.testing.expect(!worker_publish.claimed_page); + try std.testing.expect(!worker_publish.advanced_phase); + try std.testing.expect(!worker_publish.published); + + const coordinator_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, coordinator_publish.phase); + try std.testing.expect(coordinator_publish.advanced_phase); + + const duplicate_publish = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate_publish.phase); + try std.testing.expect(!duplicate_publish.advanced_phase); + try std.testing.expect(!duplicate_publish.published); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + + var authority = try graph.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, hub.state); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(u32, 1), authority.iterations_completed); + try std.testing.expectEqual(authority.iterations_completed, hub.iterations_completed); + try std.testing.expectEqual(@as(usize, 0), authority.build_pages.len); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + + const after_complete = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + const authorities = try graph.graphMetricTopK("hits_authority", 2); + defer { + for (authorities) |*score| score.deinit(alloc); + alloc.free(authorities); + } + const hubs = try graph.graphMetricTopK("hits_hub", 2); + defer { + for (hubs) |*score| score.deinit(alloc); + alloc.free(hubs); + } + try std.testing.expectEqual(@as(usize, 2), authorities.len); + try std.testing.expectEqual(@as(usize, 2), hubs.len); + } +} + +test "graph planned drain completes single metric families through metric-name boundary" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-planned-drain-single-vector"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-planned-drain-single-vector"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{ + .{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }, + .{ + .name = "pagerank", + .kind = .pagerank, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + + const target_generation = graph.edge_generation; + const worker_ids = [_][]const u8{ "worker-a", "worker-b", "worker-c" }; + const metric_names = [_][]const u8{ "degree", "pagerank", "eigenvector" }; + for (metric_names) |metric_name| { + var status = try graph.runGraphMetricPlannedDrain(metric_name, target_generation, .{ + .worker_ids = &worker_ids, + .max_steps = 256, + }); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + + const top = try graph.graphMetricTopK(metric_name, 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, metric_name)) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, metric_name, completed_job.job_id)) == null); + } + } +} + +test "graph degree planned public failure preserves prior published generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-public-fail-build"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-public-fail-build"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try std.testing.expectError(error.GraphMetricBuildJobNotFound, graph.failGraphMetricPlannedBuild("degree", error.InvalidGraphMetricScore)); + + try graph.addEdge("doc-a", "hub", "cites", 1.0, 0, 0, ""); + var published = try graph.runDegreeMetricPlanned("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const prior_generation = published.published_generation; + + const before_failure = try graph.graphMetricTopK("degree", 10); + defer { + for (before_failure) |*score| score.deinit(alloc); + alloc.free(before_failure); + } + try std.testing.expectEqual(@as(usize, 2), before_failure.len); + + try graph.addEdge("doc-b", "hub", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > prior_generation); + + var building = try graph.ensureGraphMetricPlannedBuild("degree", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("degree", "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + var failed = try graph.failGraphMetricPlannedBuild("degree", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(prior_generation, failed.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed.build_job_id); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.last_error); + try std.testing.expectEqual(@as(usize, 1), failed.recent_failures.len); + try std.testing.expectEqual(building.build_job_id, failed.recent_failures[0].job_id); + + const after_failure = try graph.graphMetricTopK("degree", 10); + defer { + for (after_failure) |*score| score.deinit(alloc); + alloc.free(after_failure); + } + try std.testing.expectEqual(before_failure.len, after_failure.len); + for (before_failure, after_failure) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-b")); + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildLease(&txn, "degree")) == null); + const failed_job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, failed_job.job_id); + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", failed_job.job_id)) == null); + } + + try std.testing.expectError(error.GraphMetricBuildNotActive, graph.failGraphMetricPlannedBuild("degree", error.InvalidGraphMetricScore)); +} + +test "graph degree planned expired worker page is reclaimed across reopened handles" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-reopened-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-reopened-reclaim"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + + var job_id: u64 = 0; + var target_generation: u64 = 0; + var score_generation: u64 = 0; + var dead_worker_lease_expires_at_ms: u64 = 0; + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + target_generation = graph.edge_generation; + try graph.acquireGraphMetricBuildLease("degree", target_generation); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + const advanced = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advanced.phase); + try std.testing.expect(advanced.advanced_phase); + + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + job_id = job.job_id; + score_generation = job.score_generation; + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + + const claim = try worker.claimGraphMetricBuildPageAt("degree", job_id, .scan_edges_and_out_degree, 0, 1, "worker-dead", 1000) orelse return error.TestExpectedGraphMetricBuildPage; + dead_worker_lease_expires_at_ms = claim.lease_expires_at_ms; + try std.testing.expectEqual(@as(u64, 1), claim.attempt); + _ = try worker.updateGraphMetricBuildPageProgress("degree", job_id, .scan_edges_and_out_degree, 0, 1, "worker-dead", "dead-cursor", 1, claim.total_units); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var observer = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer observer.close(); + + var status = try observer.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.phase); + try std.testing.expectEqual(job_id, status.build_job_id); + try std.testing.expectEqual(target_generation, status.building_generation); + try std.testing.expectEqual(@as(usize, 1), status.build_pages.len); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].page_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, status.build_pages[0].state); + try std.testing.expectEqualStrings("worker-dead", status.build_pages[0].worker_id); + try std.testing.expectEqual(dead_worker_lease_expires_at_ms, status.build_pages[0].lease_expires_at_ms); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].attempt); + try std.testing.expectEqualStrings("dead-cursor", status.build_pages[0].cursor); + try std.testing.expectEqual(@as(u64, 1), status.build_pages[0].completed_units); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var early_worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer early_worker.close(); + + try std.testing.expect((try early_worker.claimGraphMetricBuildPageAt("degree", job_id, .scan_edges_and_out_degree, 0, 1, "worker-early", dead_worker_lease_expires_at_ms - 1)) == null); + var txn = try early_worker.beginReadReverseTxn(); + defer txn.abort(); + const still_leased = try early_worker.metricBuildPage(&txn, "degree", job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, still_leased.state); + try std.testing.expectEqualStrings("worker-dead", still_leased.worker_id); + try std.testing.expectEqual(@as(u64, 1), still_leased.attempt); + try std.testing.expectEqualStrings("dead-cursor", still_leased.cursor); + } + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var reclaim_worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer reclaim_worker.close(); + + const reclaimed = try reclaim_worker.claimGraphMetricBuildPageAt("degree", job_id, .scan_edges_and_out_degree, 0, 1, "worker-reclaim", dead_worker_lease_expires_at_ms + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqualStrings("worker-reclaim", reclaimed.worker_id); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + + var txn = try reclaim_worker.beginReadReverseTxn(); + const job = try reclaim_worker.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + txn.abort(); + try std.testing.expect((try reclaim_worker.executeDegreeScanBuildPage("degree", metrics[0], job, reclaimed)) > 0); + } + + var finished = false; + for (0..80) |step_i| { + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var worker = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer worker.close(); + const worker_id = if (step_i % 2 == 0) "worker-a" else "worker-b"; + const worker_step = try worker.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], worker_id); + if (worker_step.completed_build) { + finished = true; + break; + } + } + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var coordinator = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer coordinator.close(); + const coordinator_step = try coordinator.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + if (coordinator_step.completed_build) { + finished = true; + break; + } + } + } + try std.testing.expect(finished); + + { + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var status = try graph.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + + const top = try graph.graphMetricTopK("degree", 2); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + try std.testing.expectEqualStrings("hub", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, @floatFromInt(graph_metric_build_target_scan_page_units + 1)), top[0].score, 0.0000001); + } +} + +test "graph degree planned worker step uses injected time for lease reclaim" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-worker-step-clock"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-worker-step-clock"); + defer cleanupTmp(rev_path); + + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetricAt("degree", "worker-prepare", 1000); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advanced = try graph.runGraphMetricPlannedCoordinatorStepForMetricAt("degree", 1001); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advanced.phase); + try std.testing.expect(advanced.advanced_phase); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try graph.cloneGraphMetricBuildJobAlloc(try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob); + }; + defer graph.deinitClonedGraphMetricBuildJob(active_job); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, active_job.phase); + + const dead_claim = try graph.claimNextGraphMetricBuildPageAt("degree", active_job.job_id, .scan_edges_and_out_degree, 0, "worker-dead", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), dead_claim.attempt); + try std.testing.expect(dead_claim.lease_expires_at_ms > 2000); + _ = try graph.updateGraphMetricBuildPageProgress("degree", active_job.job_id, .scan_edges_and_out_degree, 0, dead_claim.page_id, "worker-dead", "dead-cursor", 1, dead_claim.total_units); + + for (0..16) |_| { + const live = try graph.runGraphMetricPlannedWorkerPageStepForMetricAt("degree", "worker-live", 1500); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, live.phase); + if (!live.claimed_page) break; + try std.testing.expect(live.completed_page); + try std.testing.expect(live.page_id != dead_claim.page_id); + } + + const early = try graph.runGraphMetricPlannedWorkerPageStepForMetricAt("degree", "worker-early", dead_claim.lease_expires_at_ms - 1); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, early.phase); + try std.testing.expect(!early.claimed_page); + try std.testing.expect(!early.completed_page); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, dead_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqualStrings("worker-dead", page.worker_id); + try std.testing.expectEqual(@as(u64, 1), page.attempt); + try std.testing.expectEqualStrings("dead-cursor", page.cursor); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + } + + const reclaimed = try graph.runGraphMetricPlannedWorkerPageStepForMetricAt("degree", "worker-reclaim", dead_claim.lease_expires_at_ms + 1); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, reclaimed.phase); + try std.testing.expect(reclaimed.claimed_page); + try std.testing.expect(reclaimed.completed_page); + try std.testing.expectEqual(dead_claim.page_id, reclaimed.page_id); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "degree", active_job.job_id, .scan_edges_and_out_degree, 0, dead_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqualStrings("worker-reclaim", page.worker_id); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + try std.testing.expectEqual(@as(u64, 0), page.lease_expires_at_ms); + try std.testing.expectEqualStrings("", page.cursor); + } +} + +test "graph degree planned coordinator ticks are idempotent across barriers and publish" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-coordinator-idempotent"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-coordinator-idempotent"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-{d:0>3}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + } + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + const prepare = try graph.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-prepare"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + const duplicate_prepare = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, duplicate_prepare.phase); + try std.testing.expect(!duplicate_prepare.advanced_phase); + try std.testing.expect(!duplicate_prepare.published); + + while (true) { + const step = try graph.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-scan"); + if (!step.claimed_page) break; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, step.phase); + try std.testing.expect(step.completed_page); + } + const advance_scan = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, advance_scan.phase); + try std.testing.expect(advance_scan.advanced_phase); + const duplicate_scan = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, duplicate_scan.phase); + try std.testing.expect(!duplicate_scan.advanced_phase); + try std.testing.expect(!duplicate_scan.published); + + while (true) { + const step = try graph.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-reduce"); + if (!step.claimed_page) break; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, step.phase); + try std.testing.expect(step.completed_page); + } + const advance_reduce = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, advance_reduce.phase); + try std.testing.expect(advance_reduce.advanced_phase); + + var stale_publish_job: GraphIndex.GraphMetricBuildJob = undefined; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + stale_publish_job = try graph.cloneGraphMetricBuildJobAlloc(try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob); + } + defer graph.deinitClonedGraphMetricBuildJob(stale_publish_job); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, stale_publish_job.phase); + const publish_step = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish_step.phase); + try std.testing.expect(publish_step.advanced_phase); + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.publishGraphMetricBuildFromCoordinator("degree", metrics[0], stale_publish_job)); + + var published = try graph.graphMetricStatus("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, published.phase); + try std.testing.expectEqual(@as(usize, 1), published.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, published.recent_events[0].kind); + const published_generation = published.published_generation; + + const cleanup_coordinator = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup_coordinator.phase); + try std.testing.expect(!cleanup_coordinator.advanced_phase); + try std.testing.expect(!cleanup_coordinator.published); + + var cleanup_done = false; + for (0..8) |_| { + const cleanup = try graph.runGraphMetricPlannedWorkerPageStep("degree", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + try std.testing.expect(cleanup.completed_page); + cleanup_done = true; + break; + } + } + try std.testing.expect(cleanup_done); + + const after_complete = try graph.runGraphMetricPlannedCoordinatorStep("degree", metrics[0]); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, after_complete.phase); + try std.testing.expect(after_complete.published); + try std.testing.expect(!after_complete.advanced_phase); + + var fresh = try graph.graphMetricStatus("degree"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(published_generation, fresh.published_generation); + try std.testing.expectEqual(@as(usize, 1), fresh.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, fresh.recent_events[0].kind); +} + +test "graph degree planned build serves prior generation while active" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-planned-active"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-planned-active"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runDegreeMetricPlanned("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const first_generation = published.published_generation; + + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + const second_generation = graph.edge_generation; + try std.testing.expect(second_generation > first_generation); + try graph.acquireGraphMetricBuildLease("degree", second_generation); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var building = try graph.graphMetricStatus("degree"); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(first_generation, building.published_generation); + try std.testing.expectEqual(second_generation, building.building_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, building.phase); + + const stale_top = try graph.graphMetricTopK("degree", 10); + defer { + for (stale_top) |*score| score.deinit(alloc); + alloc.free(stale_top); + } + try std.testing.expectEqual(@as(usize, 2), stale_top.len); + for (stale_top) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-c")); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.001); + } + try std.testing.expectError(error.GraphMetricBuildPublishNotReady, graph.verifyGraphMetricBuildPublishReady("degree", building.build_job_id)); + + try graph.releaseGraphMetricBuildLease("degree"); + var rebuilt = try graph.runDegreeMetricPlanned("degree"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expectEqual(second_generation, rebuilt.published_edge_generation); + try std.testing.expect(rebuilt.published_generation != first_generation); + + const fresh_top = try graph.graphMetricTopK("degree", 10); + defer { + for (fresh_top) |*score| score.deinit(alloc); + alloc.free(fresh_top); + } + try std.testing.expectEqual(@as(usize, 3), fresh_top.len); + try std.testing.expectEqualStrings("doc-b", fresh_top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), fresh_top[0].score, 0.001); +} + +test "graph metric verified publish cleans completed job namespace only" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-job-cleanup"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-job-cleanup"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + var status = try graph.runDegreeMetricPlanned("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + + var job_id: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", job_id)) == null); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", job_id, .scan_edges_and_out_degree, 0, 1)) == null); + try std.testing.expect((try graph.metricBuildPhaseSummary(&txn, "degree", job_id, .scan_edges_and_out_degree, 0)) == null); + } + + const removed = try graph.cleanupGraphMetricBuildJobKeys("degree", job_id); + try std.testing.expectEqual(@as(usize, 0), removed); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", job_id)) == null); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", job_id, .scan_edges_and_out_degree, 0, 1)) == null); + try std.testing.expect((try graph.metricBuildPhaseSummary(&txn, "degree", job_id, .scan_edges_and_out_degree, 0)) == null); + } + + var fresh = try graph.graphMetricStatus("degree"); + defer fresh.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, fresh.state); + try std.testing.expectEqual(status.published_generation, fresh.published_generation); +} + +fn drainRetiredGraphMetricScoresForTest(graph: *GraphIndex, metric_name: []const u8) !void { + var steps: usize = 0; + while (try graph.cleanupRetiredGraphMetricScoreGenerationPage(metric_name)) { + steps += 1; + if (steps > 16) return error.TestGraphMetricCleanupDidNotConverge; + } +} + +fn drainFailedGraphMetricBuildJobsForTest(graph: *GraphIndex, metric_name: []const u8) !void { + var steps: usize = 0; + while (try graph.cleanupFailedGraphMetricBuildJobPage(metric_name)) { + steps += 1; + if (steps > 128) return error.TestGraphMetricCleanupDidNotConverge; + } +} + +test "graph metric failed planned build records failure before bounded namespace cleanup" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-failed-job-cleanup"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-failed-job-cleanup"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .reduce_ranks }) |expected_phase| { + const step = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + } + + var job_id: u64 = 0; + var score_generation: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + score_generation = job.score_generation; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectEqual(@as(usize, 2), try graph.countGraphMetricScoreGeneration("degree", score_generation)); + _ = try graph.metricBuildManifest(&txn, "degree", job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + _ = try graph.metricBuildPage(&txn, "degree", job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.metricBuildPhaseSummary(&txn, "degree", job_id, .scan_edges_and_out_degree, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + } + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + for (0..graph_metric_build_cleanup_delete_page_units * 2 + 6) |i| { + var node_buf: [64]u8 = undefined; + const node = try std.fmt.bufPrint(&node_buf, "failed-cleanup-extra-{d:0>3}", .{i}); + const key = try graph.graphMetricBuildAttemptDegreePartialKeyAlloc("degree", job_id, .scan_edges_and_out_degree, 0, 999, 1, node); + defer alloc.free(key); + try GraphIndex.putU64(&batch, key, 1); + } + try batch.commit(); + } + + try graph.recordGraphMetricFailure("degree", error.InvalidGraphMetricScore); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + // Failure visibility is committed without an unbounded namespace + // delete; maintenance performs that work on subsequent bounded ticks. + _ = try graph.metricBuildManifest(&txn, "degree", job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + try drainRetiredGraphMetricScoresForTest(&graph, "degree"); + try std.testing.expect(try graph.cleanupFailedGraphMetricBuildJobPage("degree")); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const prefix = try graph.graphMetricBuildJobNamespacePrefixAlloc("degree", job_id); + defer alloc.free(prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const entry = (try cur.seekAtOrAfter(prefix)) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expect(std.mem.startsWith(u8, entry.key, prefix)); + } + try drainFailedGraphMetricBuildJobsForTest(&graph, "degree"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_job.phase); + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_job.last_error); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("degree", score_generation)); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", job_id)) == null); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", job_id, .scan_edges_and_out_degree, 0, 1)) == null); + try std.testing.expect((try graph.metricBuildPhaseSummary(&txn, "degree", job_id, .scan_edges_and_out_degree, 0)) == null); + } + + var failed = try graph.graphMetricStatus("degree"); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(@as(u64, 0), failed.published_generation); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.last_error); + try std.testing.expectEqual(@as(usize, 1), failed.recent_failures.len); + try std.testing.expectEqual(job_id, failed.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed.recent_failures[0].phase); + try std.testing.expectEqual(@as(u64, 1), failed.recent_failures[0].retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.recent_failures[0].last_error); +} + +test "graph metric failed planned build retains bounded diagnostics" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-failed-job-retention"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-failed-job-retention"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + const prepare = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + + var job_id: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, job.phase); + } + + for (0..graph_metric_recent_event_limit + 2) |_| { + try graph.recordGraphMetricFailure("degree", error.InvalidGraphMetricScore); + } + + var failed = try graph.graphMetricStatus("degree"); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(@as(u64, graph_metric_recent_event_limit + 2), failed.retry_count); + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), failed.recent_failures.len); + try std.testing.expectEqual(@as(u64, graph_metric_recent_event_limit + 2), failed.recent_failures[0].sequence); + try std.testing.expectEqual(@as(u64, graph_metric_recent_event_limit + 2), failed.recent_failures[0].retry_count); + try std.testing.expectEqual(job_id, failed.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, failed.recent_failures[0].phase); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.recent_failures[0].last_error); + try std.testing.expectEqual(@as(u64, 3), failed.recent_failures[failed.recent_failures.len - 1].sequence); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, failed.last_event.?.kind); + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), failed.recent_events.len); + try std.testing.expectEqual(@as(u64, graph_metric_recent_event_limit + 2), failed.recent_events[0].sequence); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, failed.recent_events[0].kind); + try std.testing.expectEqual(@as(u64, 3), failed.recent_events[failed.recent_events.len - 1].sequence); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const pruned_key = try graph.graphMetricFailureRecordKeyAlloc("degree", 1); + defer alloc.free(pruned_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_key)); + const retained_key = try graph.graphMetricFailureRecordKeyAlloc("degree", 3); + defer alloc.free(retained_key); + _ = try txn.get(retained_key); + const pruned_event_key = try graph.graphMetricEventKeyAlloc("degree", 1); + defer alloc.free(pruned_event_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_event_key)); + const retained_event_key = try graph.graphMetricEventKeyAlloc("degree", 3); + defer alloc.free(retained_event_key); + _ = try txn.get(retained_event_key); + } +} + +test "graph metric repeated failed planned builds bound diagnostics and cleanup abandoned namespaces" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-repeated-failed-job-cleanup"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-repeated-failed-job-cleanup"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const failed_build_count = graph_metric_recent_event_limit + 2; + var failed_job_ids: [failed_build_count]u64 = undefined; + var failed_score_generations: [failed_build_count]u64 = undefined; + for (0..failed_build_count) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-failed-{d}", .{i}); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + const target_generation = graph.edge_generation; + + var building = try graph.ensureGraphMetricPlannedBuild("degree", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .reduce_ranks }) |expected_phase| { + const step = try graph.runDegreeMetricPlannedWorkerStep("degree", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + failed_job_ids[i] = job.job_id; + failed_score_generations[i] = job.score_generation; + try std.testing.expectEqual(target_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expect((try graph.countGraphMetricScoreGeneration("degree", job.score_generation)) > 0); + _ = try graph.metricBuildManifest(&txn, "degree", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + + var failed = try graph.failGraphMetricPlannedBuild("degree", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(@as(u64, i + 1), failed.retry_count); + try std.testing.expectEqual(@as(usize, @min(i + 1, graph_metric_recent_event_limit)), failed.recent_failures.len); + try drainRetiredGraphMetricScoresForTest(&graph, "degree"); + try drainFailedGraphMetricBuildJobsForTest(&graph, "degree"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildLease(&txn, "degree")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "degree", failed_job_ids[i])) == null); + try std.testing.expect((try graph.metricBuildPage(&txn, "degree", failed_job_ids[i], .scan_edges_and_out_degree, 0, 1)) == null); + try std.testing.expect((try graph.metricBuildPhaseSummary(&txn, "degree", failed_job_ids[i], .scan_edges_and_out_degree, 0)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("degree", failed_score_generations[i])); + const job_namespace_prefix = try graph.graphMetricBuildJobNamespacePrefixAlloc("degree", failed_job_ids[i]); + defer alloc.free(job_namespace_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const maybe_job_namespace = try cur.seekAtOrAfter(job_namespace_prefix); + if (maybe_job_namespace) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, job_namespace_prefix)); + } + } + + var retained = try graph.graphMetricStatus("degree"); + defer retained.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, retained.state); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.retry_count); + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), retained.recent_failures.len); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.recent_failures[0].sequence); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.recent_events[0].sequence); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, retained.last_event.?.kind); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const pruned_failure_key = try graph.graphMetricFailureRecordKeyAlloc("degree", 1); + defer alloc.free(pruned_failure_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_failure_key)); + const retained_failure_key = try graph.graphMetricFailureRecordKeyAlloc("degree", failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_failure_key); + _ = try txn.get(retained_failure_key); + const pruned_event_key = try graph.graphMetricEventKeyAlloc("degree", 1); + defer alloc.free(pruned_event_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_event_key)); + const retained_event_key = try graph.graphMetricEventKeyAlloc("degree", failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_event_key); + _ = try txn.get(retained_event_key); + } +} + +fn drainGraphMetricBuildToPublishForTest( + graph: *GraphIndex, + metric_name: []const u8, + cfg: GraphMetricConfig, + worker_id: []const u8, + phases: []const GraphIndex.GraphMetricBuildPhase, +) !void { + for (phases) |expected_phase| { + var page_steps: usize = 0; + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep(metric_name, cfg, worker_id); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(!step.failed_build); + page_steps += 1; + if (step.advanced_phase) break; + if (page_steps > (2 * graph_metric_build_max_partition_pages + 1) * 64) + return error.TestGraphMetricBuildDidNotAdvance; + } + } +} + +fn expectGraphMetricBuildNamespaceRemovedForTest( + graph: *GraphIndex, + metric_name: []const u8, + job_id: u64, +) !void { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + + const job_namespace_prefix = try graph.graphMetricBuildJobNamespacePrefixAlloc(metric_name, job_id); + defer graph.alloc.free(job_namespace_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const maybe_job_namespace = try cur.seekAtOrAfter(job_namespace_prefix); + if (maybe_job_namespace) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, job_namespace_prefix)); +} + +fn seedUnpublishedGraphMetricScoreForTest( + graph: *GraphIndex, + metric_name: []const u8, + score_generation: u64, + node: []const u8, +) !void { + const scores = [_]GraphIndex.GraphMetricScore{.{ + .node = node, + .score = 1.0, + }}; + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricScoresInBatch(&batch, metric_name, score_generation, scores[0..]); + try batch.commit(); +} + +fn verifyRepeatedFailedIterativeMetricBuildCleanup( + alloc: Allocator, + metric_name: []const u8, + kind: GraphMetricKind, +) !void { + var store_label_buf: [128]u8 = undefined; + const store_label = try std.fmt.bufPrint(&store_label_buf, "store-{s}-repeated-failed-job-cleanup", .{metric_name}); + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, store_label); + defer cleanupTmp(store_path); + var rev_label_buf: [128]u8 = undefined; + const rev_label = try std.fmt.bufPrint(&rev_label_buf, "rev-{s}-repeated-failed-job-cleanup", .{metric_name}); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, rev_label); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = metric_name, + .kind = kind, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const phases = [_]GraphIndex.GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .check_convergence, + }; + const failed_build_count = graph_metric_recent_event_limit + 2; + var failed_job_ids: [failed_build_count]u64 = undefined; + var failed_score_generations: [failed_build_count]u64 = undefined; + for (0..failed_build_count) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "{s}-failed-{d}", .{ metric_name, i }); + try graph.addEdge(source, "hub", "cites", 1.0, 0, 0, ""); + const target_generation = graph.edge_generation; + + var building = try graph.ensureGraphMetricPlannedBuild(metric_name, target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + + try drainGraphMetricBuildToPublishForTest(&graph, metric_name, metrics[0], "worker-a", phases[0..]); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, metric_name) orelse return error.TestExpectedGraphMetricBuildJob; + failed_job_ids[i] = job.job_id; + failed_score_generations[i] = job.score_generation; + try std.testing.expectEqual(target_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.metricBuildManifest(&txn, metric_name, job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + try seedUnpublishedGraphMetricScoreForTest(&graph, metric_name, failed_score_generations[i], "abandoned-score"); + try std.testing.expect((try graph.countGraphMetricScoreGeneration(metric_name, failed_score_generations[i])) > 0); + + var failed = try graph.failGraphMetricPlannedBuild(metric_name, error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(@as(u64, i + 1), failed.retry_count); + try std.testing.expectEqual(@as(usize, @min(i + 1, graph_metric_recent_event_limit)), failed.recent_failures.len); + try drainRetiredGraphMetricScoresForTest(&graph, metric_name); + try drainFailedGraphMetricBuildJobsForTest(&graph, metric_name); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildLease(&txn, metric_name)) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, metric_name, failed_job_ids[i])) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration(metric_name, failed_score_generations[i])); + } + try expectGraphMetricBuildNamespaceRemovedForTest(&graph, metric_name, failed_job_ids[i]); + } + + var retained = try graph.graphMetricStatus(metric_name); + defer retained.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, retained.state); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.retry_count); + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), retained.recent_failures.len); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.recent_failures[0].sequence); + try std.testing.expectEqual(@as(u64, failed_build_count), retained.recent_events[0].sequence); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, retained.last_event.?.kind); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const pruned_failure_key = try graph.graphMetricFailureRecordKeyAlloc(metric_name, 1); + defer alloc.free(pruned_failure_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_failure_key)); + const retained_failure_key = try graph.graphMetricFailureRecordKeyAlloc(metric_name, failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_failure_key); + _ = try txn.get(retained_failure_key); + const pruned_event_key = try graph.graphMetricEventKeyAlloc(metric_name, 1); + defer alloc.free(pruned_event_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_event_key)); + const retained_event_key = try graph.graphMetricEventKeyAlloc(metric_name, failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_event_key); + _ = try txn.get(retained_event_key); + } +} + +test "graph metric repeated failed iterative builds bound diagnostics and cleanup abandoned namespaces" { + const alloc = std.testing.allocator; + try verifyRepeatedFailedIterativeMetricBuildCleanup(alloc, "pagerank", .pagerank); + try verifyRepeatedFailedIterativeMetricBuildCleanup(alloc, "eigenvector", .eigenvector); +} + +test "graph metric repeated failed hits builds bound diagnostics and cleanup abandoned namespaces" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-repeated-failed-job-cleanup"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-repeated-failed-job-cleanup"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const phases = [_]GraphIndex.GraphMetricBuildPhase{ + .prepare_generation, + .scan_edges_and_out_degree, + .initialize_ranks, + .iterate_contributions, + .reduce_ranks, + .hits_hub_contributions, + .hits_hub_reduce_ranks, + .check_convergence, + }; + const failed_build_count = graph_metric_recent_event_limit + 2; + var failed_job_ids: [failed_build_count]u64 = undefined; + var failed_score_generations: [failed_build_count]u64 = undefined; + for (0..failed_build_count) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "hits-failed-{d}", .{i}); + try graph.addEdge(source, "authority", "cites", 1.0, 0, 0, ""); + const target_generation = graph.edge_generation; + + var building = try graph.ensureGraphMetricPlannedBuild("hits_authority", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + + try drainGraphMetricBuildToPublishForTest(&graph, "hits_authority", metrics[0], "worker-a", phases[0..]); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + failed_job_ids[i] = job.job_id; + failed_score_generations[i] = job.score_generation; + try std.testing.expectEqual(target_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.metricBuildManifest(&txn, "hits_authority", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + try seedUnpublishedGraphMetricScoreForTest(&graph, "hits_authority", failed_score_generations[i], "abandoned-authority"); + try seedUnpublishedGraphMetricScoreForTest(&graph, "hits_hub", failed_score_generations[i], "abandoned-hub"); + try std.testing.expect((try graph.countGraphMetricScoreGeneration("hits_authority", failed_score_generations[i])) > 0); + try std.testing.expect((try graph.countGraphMetricScoreGeneration("hits_hub", failed_score_generations[i])) > 0); + + var failed_authority = try graph.failGraphMetricPlannedBuild("hits_authority", error.InvalidGraphMetricScore); + defer failed_authority.deinit(alloc); + var failed_hub = try graph.graphMetricStatus("hits_hub"); + defer failed_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_hub.state); + try std.testing.expectEqual(@as(u64, i + 1), failed_authority.retry_count); + try std.testing.expectEqual(@as(u64, i + 1), failed_hub.retry_count); + try std.testing.expectEqual(@as(usize, @min(i + 1, graph_metric_recent_event_limit)), failed_authority.recent_failures.len); + try std.testing.expectEqual(failed_authority.recent_failures.len, failed_hub.recent_failures.len); + try drainRetiredGraphMetricScoresForTest(&graph, "hits_authority"); + try drainRetiredGraphMetricScoresForTest(&graph, "hits_hub"); + try drainFailedGraphMetricBuildJobsForTest(&graph, "hits_authority"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildLease(&txn, "hits_authority")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", failed_job_ids[i])) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_authority", failed_score_generations[i])); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_hub", failed_score_generations[i])); + } + try expectGraphMetricBuildNamespaceRemovedForTest(&graph, "hits_authority", failed_job_ids[i]); + } + + var retained_authority = try graph.graphMetricStatus("hits_authority"); + defer retained_authority.deinit(alloc); + var retained_hub = try graph.graphMetricStatus("hits_hub"); + defer retained_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, retained_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, retained_hub.state); + try std.testing.expectEqual(@as(u64, failed_build_count), retained_authority.retry_count); + try std.testing.expectEqual(retained_authority.retry_count, retained_hub.retry_count); + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), retained_authority.recent_failures.len); + try std.testing.expectEqual(retained_authority.recent_failures.len, retained_hub.recent_failures.len); + try std.testing.expectEqual(@as(u64, failed_build_count), retained_authority.recent_failures[0].sequence); + try std.testing.expectEqual(retained_authority.recent_failures[0].sequence, retained_hub.recent_failures[0].sequence); + try std.testing.expectEqual(@as(u64, failed_build_count), retained_authority.recent_events[0].sequence); + try std.testing.expectEqual(retained_authority.recent_events[0].sequence, retained_hub.recent_events[0].sequence); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, retained_authority.last_event.?.kind); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.failed, retained_hub.last_event.?.kind); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + inline for (.{ "hits_authority", "hits_hub" }) |metric_name| { + const pruned_failure_key = try graph.graphMetricFailureRecordKeyAlloc(metric_name, 1); + defer alloc.free(pruned_failure_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_failure_key)); + const retained_failure_key = try graph.graphMetricFailureRecordKeyAlloc(metric_name, failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_failure_key); + _ = try txn.get(retained_failure_key); + const pruned_event_key = try graph.graphMetricEventKeyAlloc(metric_name, 1); + defer alloc.free(pruned_event_key); + try std.testing.expectError(error.NotFound, txn.get(pruned_event_key)); + const retained_event_key = try graph.graphMetricEventKeyAlloc(metric_name, failed_build_count - graph_metric_recent_event_limit + 1); + defer alloc.free(retained_event_key); + _ = try txn.get(retained_event_key); + } + } +} + +test "graph metric build job cleanup refuses active job namespace" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-job-cleanup-active"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-job-cleanup-active"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + + var txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + txn.abort(); + + try std.testing.expectError(error.GraphMetricBuildJobActive, graph.cleanupGraphMetricBuildJobKeys("degree", job.job_id)); + { + var check_txn = try graph.beginReadReverseTxn(); + defer check_txn.abort(); + _ = try graph.metricBuildManifest(&check_txn, "degree", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + _ = try graph.metricBuildPage(&check_txn, "degree", job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + } +} + +test "graph degree planned build honors edge filter during scan page execution" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-planned-filter"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-planned-filter"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{ + .{ + .name = "degree_local", + .kind = .degree, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + }, + .{ + .name = "degree_planned", + .kind = .degree, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + + var local_status = try graph.runGraphMetric("degree_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runDegreeMetricPlanned("degree_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(@as(u64, 2), planned_status.last_event.?.score_count); + + const local_top = try graph.graphMetricTopK("degree_local", 10); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("degree_planned", 10); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + try std.testing.expectEqual(@as(usize, 2), planned_top.len); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expect(!std.mem.eql(u8, planned.node, "doc-x")); + try std.testing.expect(!std.mem.eql(u8, planned.node, "doc-y")); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "graph metric filtered scan checkpoints backfill survives reopen and concurrent mutations" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-typed-backfill"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-typed-backfill"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &.{"cites"} }, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try graph.addEdge("source", "b", "cites", 1, 0, 0, ""); + try graph.addEdge("source", "c", "cites", 1, 0, 0, ""); + while (!try graph.prepareGraphMetricPartitionForConfigStep(metrics[0], 1)) {} + // Simulate pre-postings state (or repair) while retaining a cached plan. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.delete(typed_edges.ready_key); + for ([_][]const u8{ "b", "c" }) |target| { + const raw = try reverseEdgeKeyAlloc(alloc, target, "links", "cites", "source"); + defer alloc.free(raw); + try typed_edges.update(alloc, &batch, "cites", raw, "source", target, false); + } + try batch.commit(); + } + try std.testing.expect(!try graph.prepareGraphMetricPartitionForConfigStep(metrics[0], 1)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try txn.get(typed_edges.cursor_key)).len > 0); + try std.testing.expectError(error.NotFound, txn.get(typed_edges.ready_key)); + } + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + // Insert behind the checkpoint and remove an already-backfilled edge. + try graph.batchApply(&.{.{ .source = "source", .target = "a", .edge_type = "cites" }}, &.{.{ .source = "source", .target = "b", .edge_type = "cites" }}); + var steps: usize = 0; + while (!try graph.prepareGraphMetricPartitionForConfigStep(metrics[0], 1)) { + steps += 1; + try std.testing.expect(steps < 64); + } + const reference = try graph.benchmarkMetricEdgeScan(metrics[0].edge_filter, true); + const selected = try graph.benchmarkMetricEdgeScan(metrics[0].edge_filter, false); + try std.testing.expectEqual(@as(u64, 2), selected.matched); + try std.testing.expectEqual(reference.checksum, selected.checksum); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, typed_edges.Cursor.init(alloc, &txn, metrics[0].edge_filter, "", "", "foreign-checkpoint")); + } + const plan_key = try graph.graphMetricPartitionPlanKeyAlloc(metrics[0].edge_filter); + defer alloc.free(plan_key); + graph.metric_configs = &.{}; + try std.testing.expect(try graph.cleanupGraphMetricTopologyPage()); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectError(error.NotFound, txn.get(plan_key)); +} + +test "graph metric filtered scan checkpoints skip excluded edges and resume selected types" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-filter-cursor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-filter-cursor"); + defer cleanupTmp(rev_path); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{ "references", "cites" }; + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + // Excluded edges consume no scan budget. Checkpoint across two selected + // type ranges, even when their configured order is not lexical. + try graph.addEdge("doc-filtered-source", "doc-a", "related", 1.0, 0, 0, ""); + try graph.addEdge("doc-source", "doc-z", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-source", "doc-b", "references", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("degree", try graph.graphMetricCurrentGeneration("degree")); + defer graph.releaseGraphMetricBuildLease("degree") catch {}; + var job_txn = try graph.beginReadReverseTxn(); + const job = try graph.metricBuildJob(&job_txn, "degree") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + const prepare = try graph.runGraphMetricPlannedWorkerStep("degree", metrics[0], "worker"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + + const claimed = try graph.claimNextGraphMetricBuildPageAt("degree", job.job_id, .scan_edges_and_out_degree, 0, "worker", 1000) orelse + return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeDegreeScanBuildPageWithLimit("degree", metrics[0], job, claimed, 1); + { + var checkpoint_txn = try graph.beginReadReverseTxn(); + defer checkpoint_txn.abort(); + const checkpoint = try graph.metricBuildPage(&checkpoint_txn, "degree", job.job_id, .scan_edges_and_out_degree, 0, claimed.page_id) orelse + return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, checkpoint.state); + try std.testing.expectEqual(@as(u64, 1), checkpoint.completed_units); + try std.testing.expect(checkpoint.cursor.len > 0); + } + + const renewed = try graph.claimGraphMetricBuildPageAt("degree", job.job_id, .scan_edges_and_out_degree, 0, claimed.page_id, "worker", 1001) orelse + return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeDegreeScanBuildPageWithLimit("degree", metrics[0], job, renewed, null); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const complete = try graph.metricBuildPage(&txn, "degree", job.job_id, .scan_edges_and_out_degree, 0, claimed.page_id) orelse + return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, complete.state); + // Completion seals the whole scheduling range, not just selected edges. + try std.testing.expectEqual(complete.total_units, complete.completed_units); + const kept_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", job.job_id, "doc-source", claimed.page_id); + defer alloc.free(kept_key); + try std.testing.expectEqual(@as(u64, 2), try GraphIndex.readU64OrZero(&txn, kept_key)); + const filtered_key = try graph.graphMetricBuildDegreePartialKeyAlloc("degree", job.job_id, "doc-filtered-source", claimed.page_id); + defer alloc.free(filtered_key); + try std.testing.expectEqual(@as(u64, 0), try GraphIndex.readU64OrZero(&txn, filtered_key)); +} + +test "graph degree metric edge filter limits typed score graph" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-degree-filter"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-degree-filter"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + var published = try graph.runGraphMetric("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphMetricEdgeFilterMode.types, published.edge_filter.mode); + try std.testing.expectEqual(@as(usize, 1), published.edge_filter.types.len); + try std.testing.expectEqualStrings("cites", published.edge_filter.types[0]); + try std.testing.expectEqual(@as(u32, GraphIndex.graph_metric_meta_schema_version), published.metadata_version); + + @constCast(graph.metric_configs)[0].edge_filter = .{}; + var published_scope_status = try graph.graphMetricStatus("degree"); + defer published_scope_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.stale, published_scope_status.state); + try std.testing.expect(published_scope_status.build_queued); + try std.testing.expectEqual(published_scope_status.edge_generation, published_scope_status.queued_generation); + try std.testing.expectEqual(GraphMetricEdgeFilterMode.types, published_scope_status.edge_filter.mode); + try std.testing.expectEqual(@as(usize, 1), published_scope_status.edge_filter.types.len); + try std.testing.expectEqualStrings("cites", published_scope_status.edge_filter.types[0]); + + const top = try graph.graphMetricTopK("degree", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + for (top) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-x")); + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-y")); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.001); + } +} + +test "graph metric materialization deletion clears scores and allows rebuild" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-delete"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-delete"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + .refresh = .manual, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-c", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("degree"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const top = try graph.graphMetricTopK("degree", 3); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 3), top.len); + try std.testing.expectEqualStrings("doc-b", top[0].node); + try std.testing.expectEqualStrings("doc-a", top[1].node); + try std.testing.expectEqualStrings("doc-c", top[2].node); + { + const rank_prefix = try graph.graphMetricRankPrefixAlloc("degree", published_generation); + defer alloc.free(rank_prefix); + var rank_txn = try graph.beginReadReverseTxn(); + defer rank_txn.abort(); + try std.testing.expectEqual(top.len, try GraphIndex.countKeysWithPrefix(&rank_txn, rank_prefix)); + } + + // Published generations created before the ranked keyspace was introduced + // remain readable during rolling upgrades. New publications always restore + // the efficient ordered representation. + { + const rank_prefix = try graph.graphMetricRankPrefixAlloc("degree", published_generation); + defer alloc.free(rank_prefix); + var rank_batch = try graph.beginWriteReverseBatch(); + errdefer rank_batch.abort(); + try std.testing.expectEqual(top.len, try graph.deleteKeysWithPrefixInBatch(&rank_batch, rank_prefix)); + try rank_batch.commit(); + } + const legacy_top = try graph.graphMetricTopK("degree", 1); + defer { + for (legacy_top) |*score| score.deinit(alloc); + alloc.free(legacy_top); + } + try std.testing.expectEqual(@as(usize, 1), legacy_top.len); + try std.testing.expectEqualStrings("doc-b", legacy_top[0].node); + + // Deletion is an operator control-plane action and must remain available + // even when asynchronous generation retirement has fallen behind. + { + var backlog_batch = try graph.beginWriteReverseBatch(); + errdefer backlog_batch.abort(); + const retired_key = try graph.graphMetricRetiredScoreGenerationKeyAlloc("degree"); + defer alloc.free(retired_key); + const next_retired_key = try graph.graphMetricNextRetiredScoreGenerationKeyAlloc("degree"); + defer alloc.free(next_retired_key); + const retirement_phase_key = try graph.graphMetricRetiredScoreCleanupPhaseKeyAlloc("degree"); + defer alloc.free(retirement_phase_key); + const retirement_cursor_key = try graph.graphMetricRetiredScoreCleanupCursorKeyAlloc("degree"); + defer alloc.free(retirement_cursor_key); + try GraphIndex.putU64(&backlog_batch, retired_key, 101); + try GraphIndex.putU64(&backlog_batch, next_retired_key, 102); + try GraphIndex.putU64(&backlog_batch, retirement_phase_key, 1); + try backlog_batch.put(retirement_cursor_key, "resume-here"); + try backlog_batch.commit(); + } + + try graph.deleteGraphMetricMaterialization("degree"); + var deleted_status = try graph.graphMetricStatus("degree"); + defer deleted_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.disabled, deleted_status.state); + try std.testing.expect(!deleted_status.build_queued); + try std.testing.expectEqual(@as(u64, 0), deleted_status.published_generation); + try std.testing.expectError(error.MetricNotReady, graph.graphMetricTopK("degree", 1)); + try std.testing.expectError(error.GraphMetricDisabled, graph.runGraphMetric("degree")); + try std.testing.expect(!(try graph.cleanupRetiredGraphMetricScoreGenerationPage("degree"))); + const stale_scores = [_]GraphIndex.GraphMetricScore{.{ .node = "doc-a", .score = 99.0 }}; + try std.testing.expectError(error.GraphMetricDisabled, graph.publishGraphMetricScores( + "degree", + published_generation, + &stale_scores, + .{ + .converged = true, + .iterations_completed = 1, + .computed_at_ms = 1, + .config_fingerprint = GraphIndex.graphMetricConfigFingerprint(metrics[0]), + .edge_filter = metrics[0].edge_filter, + }, + )); + try std.testing.expectError(error.MetricNotReady, graph.graphMetricTopK("degree", 1)); + { + var job_txn = try graph.beginReadReverseTxn(); + defer job_txn.abort(); + try std.testing.expect((try graph.metricBuildJob(&job_txn, "degree")) == null); + } + + var paused_after_delete = try graph.pauseGraphMetricMaintenance("degree"); + defer paused_after_delete.deinit(alloc); + try std.testing.expect(paused_after_delete.maintenance_paused); + try graph.deleteGraphMetricMaterialization("degree"); + var deleted_paused_status = try graph.graphMetricStatus("degree"); + defer deleted_paused_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.disabled, deleted_paused_status.state); + try std.testing.expect(!deleted_paused_status.build_queued); + try std.testing.expect(!deleted_paused_status.maintenance_paused); + + var resumed_after_delete = try graph.resumeGraphMetricMaintenance("degree"); + defer resumed_after_delete.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.not_ready, resumed_after_delete.state); + + var rebuilt = try graph.runGraphMetric("degree"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expect(rebuilt.published_generation > published_generation); + try std.testing.expect(rebuilt.recent_events.len >= 1); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.publish, rebuilt.recent_events[0].kind); + const rebuilt_top = try graph.graphMetricTopK("degree", 3); + defer { + for (rebuilt_top) |*score| score.deinit(alloc); + alloc.free(rebuilt_top); + } + try std.testing.expectEqual(@as(usize, 3), rebuilt_top.len); + try std.testing.expectEqualStrings("doc-b", rebuilt_top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), rebuilt_top[0].score, 0.001); + + var last_status: ?GraphIndex.GraphMetricStatus = null; + defer if (last_status) |*status| status.deinit(alloc); + for (0..10) |_| { + if (last_status) |*status| { + status.deinit(alloc); + last_status = null; + } + var paused = try graph.pauseGraphMetricMaintenance("degree"); + paused.deinit(alloc); + last_status = try graph.resumeGraphMetricMaintenance("degree"); + } + const retained = last_status.?; + try std.testing.expectEqual(@as(usize, graph_metric_recent_event_limit), retained.recent_events.len); + try std.testing.expectEqual(GraphIndex.GraphMetricEventKind.@"resume", retained.recent_events[0].kind); + for (retained.recent_events[0 .. retained.recent_events.len - 1], retained.recent_events[1..]) |newer, older| { + try std.testing.expect(newer.sequence > older.sequence); + } +} + +test "graph metric native rank index retains only the supported top-k prefix" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-metric-bounded-rank-prefix"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-metric-bounded-rank-prefix"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const configs = [_]GraphMetricConfig{.{ .name = "degree", .kind = .degree, .refresh = .manual }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &configs }); + defer graph.close(); + + const score_count = graph_metric_rank_entry_limit + 2; + const scores = try alloc.alloc(GraphIndex.GraphMetricScore, score_count); + defer alloc.free(scores); + var initialized: usize = 0; + defer for (scores[0..initialized]) |score| alloc.free(score.node); + for (scores, 0..) |*score, i| { + score.* = .{ + .node = try std.fmt.allocPrint(alloc, "node-{d}", .{i}), + .score = if (i + 1 == score_count) -1 else @floatFromInt(i), + }; + initialized += 1; + } + + const generation = 1; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricScoresInBatch(&batch, "degree", generation, scores[0..graph_metric_rank_entry_limit]); + try batch.commit(); + } + // A later better score must evict the durable worst boundary. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricScoresInBatch(&batch, "degree", generation, scores[graph_metric_rank_entry_limit .. graph_metric_rank_entry_limit + 1]); + try batch.commit(); + } + // A later worse score still enters the point vector but leaves the full + // rank prefix untouched through the boundary fast path. + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricScoresInBatch(&batch, "degree", generation, scores[graph_metric_rank_entry_limit + 1 ..]); + try batch.commit(); + } + + const rank_prefix = try graph.graphMetricRankPrefixAlloc("degree", generation); + defer alloc.free(rank_prefix); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(graph_metric_rank_entry_limit, try GraphIndex.countKeysWithPrefix(&txn, rank_prefix)); + const admitted_key = try graph.graphMetricRankKeyAlloc("degree", generation, scores[graph_metric_rank_entry_limit].score, scores[graph_metric_rank_entry_limit].node); + defer alloc.free(admitted_key); + _ = try txn.get(admitted_key); + const evicted_key = try graph.graphMetricRankKeyAlloc("degree", generation, scores[0].score, scores[0].node); + defer alloc.free(evicted_key); + try std.testing.expectError(error.NotFound, txn.get(evicted_key)); + const rejected_key = try graph.graphMetricRankKeyAlloc("degree", generation, scores[graph_metric_rank_entry_limit + 1].score, scores[graph_metric_rank_entry_limit + 1].node); + defer alloc.free(rejected_key); + try std.testing.expectError(error.NotFound, txn.get(rejected_key)); + } + try std.testing.expectEqual(score_count, try graph.countGraphMetricScoreGeneration("degree", generation)); +} + +test "graph eigenvector metric publishes normalized centrality scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 100, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-b", "cites", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("eigenvector"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expect(published.iterations_completed > 0); + try std.testing.expect(published.converged or published.iterations_completed == 100); + + const top = try graph.graphMetricTopK("eigenvector", 3); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 3), top.len); + try std.testing.expectEqualStrings("doc-b", top[0].node); + try std.testing.expect(top[0].score > top[1].score); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), top[0].score, 0.001); +} + +test "graph eigenvector metric edge filter limits typed score graph" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-filter"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-filter"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + .max_iterations = 20, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("eigenvector"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphMetricEdgeFilterMode.types, published.edge_filter.mode); + try std.testing.expectEqual(@as(usize, 1), published.edge_filter.types.len); + try std.testing.expectEqualStrings("cites", published.edge_filter.types[0]); + + const top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + for (top) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-x")); + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-y")); + } +} + +test "graph eigenvector planned build publishes scores matching local runner" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-planned-parity"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-planned-parity"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "eigenvector_local", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 6, + .tolerance = 0.000001, + }, + .{ + .name = "eigenvector_planned", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 6, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-d", "cites", 1.0, 0, 0, ""); + + var local_status = try graph.runGraphMetric("eigenvector_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runEigenvectorMetricPlanned("eigenvector_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + + const local_top = try graph.graphMetricTopK("eigenvector_local", 10); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("eigenvector_planned", 10); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector_planned") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); +} + +test "graph eigenvector planned build matches local runner on disconnected reducible graph" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-disconnected-reducible-parity"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-disconnected-reducible-parity"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "eigenvector_local", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 8, + .tolerance = 0.000001, + }, + .{ + .name = "eigenvector_planned", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 8, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-b", "doc-a", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-d", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-d", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-e", "doc-e", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-f", "doc-g", "cites", 1.0, 0, 0, ""); + + var local_status = try graph.runGraphMetric("eigenvector_local"); + defer local_status.deinit(alloc); + var planned_status = try graph.runEigenvectorMetricPlanned("eigenvector_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + + const local_top = try graph.graphMetricTopK("eigenvector_local", 10); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph.graphMetricTopK("eigenvector_planned", 10); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(@as(usize, 7), local_top.len); + try std.testing.expectEqual(local_top.len, planned_top.len); + var norm_sq: f64 = 0.0; + var saw_reducible_zero = false; + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expect(std.math.isFinite(local.score)); + try std.testing.expect(std.math.isFinite(planned.score)); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + norm_sq += local.score * local.score; + if ((std.mem.eql(u8, local.node, "doc-f") or std.mem.eql(u8, local.node, "doc-g")) and local.score == 0.0) { + saw_reducible_zero = true; + } + } + try std.testing.expectApproxEqAbs(@as(f64, 1.0), @sqrt(norm_sq), 0.0000001); + try std.testing.expect(saw_reducible_zero); + + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector_planned") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); +} + +test "graph eigenvector reclaimed scan page overwrites stale partial output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-scan-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-scan-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-a", "doc-c", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + defer graph.releaseGraphMetricBuildLease("eigenvector") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + const prepare = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-setup"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(prepare.advanced_phase); + + const partial_scan_claim = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executePageRankScanBuildPageWithLimit("eigenvector", metrics[0], active_job, partial_scan_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "eigenvector", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + const attempt_out_degree_key = try graph.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc("eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-a"); + defer alloc.free(attempt_out_degree_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_out_degree_key)); + const node_b_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("eigenvector", active_job.job_id, "doc-b", 1); + defer alloc.free(node_b_key); + try std.testing.expectError(error.NotFound, txn.get(node_b_key)); + const attempt_node_b_key = try graph.graphMetricBuildAttemptPageRankNodePartialKeyAlloc("eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-b"); + defer alloc.free(attempt_node_b_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_node_b_key)); + } + + const scan_expires_at = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + break :blk page.lease_expires_at_ms; + }; + const reclaimed_scan = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", scan_expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed_scan.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed_scan.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed_scan.completed_units); + try std.testing.expectEqualStrings("", reclaimed_scan.cursor); + { + var stale_out_degrees = std.StringHashMapUnmanaged(u64).empty; + defer { + var key_it = stale_out_degrees.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_out_degrees.deinit(alloc); + } + var stale_nodes = std.StringHashMapUnmanaged(void).empty; + defer { + var key_it = stale_nodes.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_nodes.deinit(alloc); + } + try stale_out_degrees.put(alloc, try alloc.dupe(u8, "doc-a"), 99); + try stale_nodes.put(alloc, try alloc.dupe(u8, "doc-stale"), {}); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writePageRankScanPartialsForAttempt("eigenvector", active_job, partial_scan_claim, 0, &stale_out_degrees, &stale_nodes, partial_scan_claim.worker_id, "", 0, partial_scan_claim.total_units)); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const stale_node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("eigenvector", active_job.job_id, "doc-stale", 1); + defer alloc.free(stale_node_key); + try std.testing.expectError(error.NotFound, txn.get(stale_node_key)); + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "eigenvector", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + } + + _ = try graph.executePageRankScanBuildPageWithLimit("eigenvector", metrics[0], active_job, reclaimed_scan, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "eigenvector", active_job.job_id, "doc-a"); + try std.testing.expectEqual(@as(u64, 2), out_degree); + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("eigenvector", active_job.job_id, node, 1); + defer alloc.free(node_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, node_key)); + } + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, job.phase); + } +} + +test "graph eigenvector reclaimed initialize page overwrites stale rank output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-initialize-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-initialize-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + defer graph.releaseGraphMetricBuildLease("eigenvector") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-setup", &.{ .prepare_generation, .scan_edges_and_out_degree }); + + try drainGraphMetricSummaryForTest(&graph, "eigenvector", active_job, .initialize_ranks, 0); + const initial_claim = try graph.claimNextGraphMetricBuildPageAt("eigenvector", active_job.job_id, .initialize_ranks, 0, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, initial_claim.state); + try std.testing.expectEqual(@as(u64, 1), initial_claim.attempt); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const rank_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 0, node); + try rank_key.write(&batch, 42.0); + } + try batch.commit(); + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt( + "eigenvector", + active_job.job_id, + .initialize_ranks, + 0, + initial_claim.page_id, + "worker-b", + initial_claim.lease_expires_at_ms + 1, + ) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + + const stale_initialized = [_]GraphIndex.PageRankInitializeNode{.{ .node = "doc-a" }}; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writeEigenvectorInitializeOutputForAttempt("eigenvector", active_job, initial_claim, &stale_initialized, 99.0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const rank_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 0, "doc-a"); + try std.testing.expectApproxEqAbs(@as(f64, 42.0), try rank_key.read(&txn), 0.0000001); + } + + _ = try graph.executeEigenvectorInitializeBuildPage("eigenvector", active_job, reclaimed); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .initialize_ranks, 0)); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .initialize_ranks, 0, initial_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const expected_rank = 1.0 / @sqrt(@as(f64, 3.0)); + inline for (.{ "doc-a", "doc-b", "doc-c" }) |node| { + const rank_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 0, node); + try std.testing.expectApproxEqAbs(expected_rank, try rank_key.read(&txn), 0.0000001); + } + + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, job.phase); + } +} + +test "graph eigenvector contribution and reduce pages resume from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + + const contribution_claim = try graph.claimNextGraphMetricBuildPageAt("eigenvector", active_job.job_id, .iterate_contributions, 0, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeEigenvectorContributionBuildPageWithLimit("eigenvector", metrics[0], active_job, contribution_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + const partial = try graphMetricOrdinalValueForTest(&graph, &txn, "eigenvector", active_job.job_id, .iterate_contributions, 0, "doc-b"); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), partial, 0.0); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try graph.ordinalAttemptContributionForNodeForTest(&txn, "eigenvector", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id, contribution_claim.attempt, "doc-b"), 0.0000001); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + const renewed_contribution = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_contribution.state); + try std.testing.expectEqual(@as(u64, 1), renewed_contribution.completed_units); + try std.testing.expect(renewed_contribution.cursor.len > 0); + _ = try graph.executeEigenvectorContributionBuildPageWithLimit("eigenvector", metrics[0], active_job, renewed_contribution, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .iterate_contributions, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const contribution = try graphMetricOrdinalValueForTest(&graph, &txn, "eigenvector", active_job.job_id, .iterate_contributions, 0, "doc-b"); + try std.testing.expectApproxEqAbs(2.0 / @sqrt(@as(f64, 3.0)), contribution, 0.0000001); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + } + + try drainGraphMetricSummaryForTest(&graph, "eigenvector", active_job, .reduce_ranks, 0); + const reduce_claim = try graph.claimNextGraphMetricBuildPageAt("eigenvector", active_job.job_id, .reduce_ranks, 0, "worker-r", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeEigenvectorReduceBuildPageWithLimit("eigenvector", active_job, reduce_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 0, reduce_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try drainGraphMetricSummaryForTest(&graph, "eigenvector", active_job, .reduce_ranks, 0); + const renewed_reduce = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .reduce_ranks, 0, reduce_claim.page_id, "worker-r", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_reduce.state); + try std.testing.expectEqual(@as(u64, 1), renewed_reduce.completed_units); + try std.testing.expectEqualStrings("", renewed_reduce.cursor); + _ = try graph.executeEigenvectorReduceBuildPageWithLimit("eigenvector", active_job, renewed_reduce, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .reduce_ranks, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const rank_a_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 1, "doc-a"); + const rank_b_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 1, "doc-b"); + const rank_c_key = graphMetricVectorSlotForTest(&graph, "eigenvector", active_job.job_id, "rank", 1, "doc-c"); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), try rank_a_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), try rank_b_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), try rank_c_key.read(&txn), 0.0000001); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } +} + +test "graph eigenvector reclaimed contribution and reduce pages overwrite stale output" { + try expectGraphMetricOrdinalTakeoverForTest(.eigenvector); +} + +test "graph eigenvector convergence page reclaim recomputes without stale partial summary" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-convergence-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-convergence-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + defer graph.releaseGraphMetricBuildLease("eigenvector") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks }); + + const check_claim = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .check_convergence, 0, 5, "worker-a", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + const partial_check = try graph.executePageRankConvergenceBuildPageWithLimit("eigenvector", metrics[0], active_job, check_claim, 1); + try std.testing.expect(!partial_check.completed_page); + try std.testing.expect(partial_check.max_delta > 0.0); + try std.testing.expect(partial_check.total_delta > 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), partial_check.rank_sum, 0.0); + + var expires_at: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.max_delta > 0.0); + try std.testing.expect(page.total_delta > 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.rank_sum, 0.0); + expires_at = page.lease_expires_at_ms; + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .check_convergence, 0, 5, "worker-b", expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.rank_sum, 0.0); + try std.testing.expect(!reclaimed.converged); + + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.updateGraphMetricBuildConvergencePageProgressForAttempt("eigenvector", active_job.job_id, 0, 5, check_claim.worker_id, check_claim.attempt, "stale-check", 1, check_claim.total_units, 99.0, 99.0, 99.0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), page.rank_sum, 0.0); + } + + const completed_check = try graph.executePageRankConvergenceBuildPageWithLimit("eigenvector", metrics[0], active_job, reclaimed, null); + try std.testing.expect(completed_check.completed_page); + try std.testing.expectApproxEqAbs(@as(f64, 0.5773502691896258), completed_check.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.5773502691896257), completed_check.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), completed_check.rank_sum, 0.0000001); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .check_convergence, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .check_convergence, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectApproxEqAbs(@as(f64, 0.5773502691896258), page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.5773502691896257), page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), page.rank_sum, 0.0000001); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } +} + +test "graph eigenvector later iteration failed pages retry and advance" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-later-iteration-retry"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-later-iteration-retry"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + defer graph.releaseGraphMetricBuildLease("eigenvector") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + try drainGraphMetricSummaryForTest(&graph, "eigenvector", active_job, .reduce_ranks, 1); + const reduce_failed_claim = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), reduce_failed_claim.attempt); + const reduce_failed = try graph.failGraphMetricBuildPage("eigenvector", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", "later eigenvector reduce failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, reduce_failed.state); + try std.testing.expectEqualStrings("later eigenvector reduce failed", reduce_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .reduce_ranks, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + { + const step = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-retry-reduce"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, step.phase); + try std.testing.expectEqual(@as(u64, 4), step.page_id); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + const summary = try graph.metricBuildPhaseSummary(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } + + const check_failed_claim = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .check_convergence, 1, 5, "worker-fail-check", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), check_failed_claim.attempt); + const check_failed = try graph.failGraphMetricBuildPage("eigenvector", active_job.job_id, .check_convergence, 1, 5, "worker-fail-check", "later eigenvector check failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, check_failed.state); + try std.testing.expectEqualStrings("later eigenvector check failed", check_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("eigenvector", active_job.job_id, .check_convergence, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "eigenvector", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + { + const step = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-retry-check"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, step.phase); + try std.testing.expectEqual(@as(u64, 5), step.page_id); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .check_convergence, 1, 5) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + const summary = try graph.metricBuildPhaseSummary(&txn, "eigenvector", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.metricBuildIterationSummary(&txn, "eigenvector", active_job.job_id, 1) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + } +} + +test "graph eigenvector later iteration exhausted page fails build and preserves prior generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-later-iteration-exhausted-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-later-iteration-exhausted-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runEigenvectorMetricPlanned("eigenvector"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 3), before_failure_top.len); + + try graph.addEdge("doc-new", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("eigenvector", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + _ = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + } + + try drainGraphMetricSummaryForTest(&graph, "eigenvector", active_job, .reduce_ranks, 1); + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const page = try graph.claimGraphMetricBuildPageAt("eigenvector", active_job.job_id, .reduce_ranks, 1, 4, worker_id, 2000 + attempt) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(attempt + 1, page.attempt); + _ = try graph.failGraphMetricBuildPage("eigenvector", active_job.job_id, .reduce_ranks, 1, 4, worker_id, "retryable later eigenvector reduction failure"); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, page.state); + try std.testing.expectEqual(@as(u64, graph_metric_build_max_page_attempts), page.attempt); + try std.testing.expectEqualStrings("retryable later eigenvector reduction failure", page.last_error); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_status = try graph.graphMetricStatus("eigenvector"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_status.state); + try std.testing.expectEqual(published_generation, failed_status.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_status.build_job_id); + const exhaustion_reason = "GraphMetricBuildPageAttemptsExhausted: phase=reduce_ranks, iteration=1, page_id=4, attempt=3, cause=retryable later eigenvector reduction failure"; + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_status.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_status.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_status.recent_failures[0].phase); + try std.testing.expectEqual(@as(u32, 1), failed_status.recent_failures[0].iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_status.recent_failures[0].last_error); + + const after_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new")); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, failed_job.phase); + try std.testing.expectEqual(@as(u32, 1), failed_job.iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "eigenvector", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("eigenvector", active_job.score_generation)); + } +} + +test "graph eigenvector cleanup page resumes after reopen with published scores visible" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-cleanup-cursor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-cleanup-cursor"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("eigenvector", try graph.graphMetricCurrentGeneration("eigenvector")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + const publish = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const visible_before_cleanup = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (visible_before_cleanup) |*score| score.deinit(alloc); + alloc.free(visible_before_cleanup); + } + try std.testing.expectEqual(@as(usize, 3), visible_before_cleanup.len); + + const abandoned_attempt_key = try graph.graphMetricBuildAttemptPageRankContributionKeyAlloc("eigenvector", active_job.job_id, .iterate_contributions, 0, 999, 1, "doc-abandoned"); + defer alloc.free(abandoned_attempt_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putF64(&batch, abandoned_attempt_key, 123.0); + for (0..graph_metric_build_cleanup_delete_page_units + 6) |i| { + var node_buf: [64]u8 = undefined; + const node = try std.fmt.bufPrint(&node_buf, "cleanup-extra-{d:0>3}", .{i}); + const key = try graph.graphMetricBuildPageRankKeyAlloc("eigenvector", active_job.job_id, 99, node); + defer alloc.free(key); + try GraphIndex.putF64(&batch, key, 1.0); + } + try batch.commit(); + } + + const first_cleanup = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, first_cleanup.phase); + try std.testing.expect(first_cleanup.claimed_page); + try std.testing.expect(!first_cleanup.completed_page); + try std.testing.expect(!first_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const cleanup_page = try graph.metricBuildPage(&txn, "eigenvector", active_job.job_id, .cleanup_old_generations, 0, first_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, cleanup_page.state); + try std.testing.expect(cleanup_page.cursor.len > 0); + try std.testing.expect(cleanup_page.completed_units >= graph_metric_build_cleanup_delete_page_units); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expectEqualStrings(cleanup_page.cursor, job.cursor); + } + + const visible_after_partial_cleanup = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (visible_after_partial_cleanup) |*score| score.deinit(alloc); + alloc.free(visible_after_partial_cleanup); + } + try std.testing.expectEqual(visible_before_cleanup.len, visible_after_partial_cleanup.len); + for (visible_before_cleanup, visible_after_partial_cleanup) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const visible_after_reopen = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (visible_after_reopen) |*score| score.deinit(alloc); + alloc.free(visible_after_reopen); + } + try std.testing.expectEqual(visible_before_cleanup.len, visible_after_reopen.len); + for (visible_before_cleanup, visible_after_reopen) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + } + { + var status = try graph.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(active_job.job_id, status.build_job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + } + + const final_cleanup = try graph.runGraphMetricPlannedWorkerStep("eigenvector", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.claimed_page); + try std.testing.expect(final_cleanup.completed_page); + try std.testing.expect(final_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "eigenvector")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "eigenvector", active_job.job_id)) == null); + try std.testing.expectError(error.NotFound, txn.get(abandoned_attempt_key)); + } + + var final_status = try graph.graphMetricStatus("eigenvector"); + defer final_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, final_status.state); +} + +test "graph eigenvector failed planned build preserves prior published generation" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-planned-failure-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-planned-failure-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runEigenvectorMetricPlanned("eigenvector"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 3), before_failure_top.len); + + try graph.addEdge("doc-new", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("eigenvector", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + var job_id: u64 = 0; + var score_generation: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + score_generation = job.score_generation; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + _ = try graph.metricBuildManifest(&txn, "eigenvector", job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } + + var failed = try graph.failGraphMetricPlannedBuild("eigenvector", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed.build_job_id); + try std.testing.expectEqual(@as(u64, 1), failed.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed.last_error); + try std.testing.expectEqual(@as(usize, 1), failed.recent_failures.len); + try std.testing.expectEqual(job_id, failed.recent_failures[0].job_id); + + const after_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new")); + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(job_id, failed_job.job_id); + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "eigenvector", job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("eigenvector", score_generation)); + } + + try std.testing.expectError(error.GraphMetricBuildNotActive, graph.failGraphMetricPlannedBuild("eigenvector", error.InvalidGraphMetricScore)); +} + +test "graph eigenvector coordinator publish failure preserves prior published generation after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-eigenvector-publish-failure-preserves-published"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-eigenvector-publish-failure-preserves-published"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{.{ + .name = "eigenvector", + .kind = .eigenvector, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-a", "doc-b", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-c", "doc-b", "cites", 1.0, 0, 0, ""); + var published = try graph.runEigenvectorMetricPlanned("eigenvector"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + const published_generation = published.published_generation; + + const before_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (before_failure_top) |*score| score.deinit(alloc); + alloc.free(before_failure_top); + } + try std.testing.expectEqual(@as(usize, 3), before_failure_top.len); + + try graph.addEdge("doc-new", "doc-b", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("eigenvector", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + try drainGraphMetricBuildToPublishForTest(&graph, "eigenvector", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .check_convergence }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.verifyGraphMetricBuildPublishReady("eigenvector", active_job.job_id); + } + _ = try materializeGraphMetricPublishPagesForTest(&graph, "eigenvector", "worker-publish-materialize"); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var manifest = try graph.metricBuildManifest(&batch, "eigenvector", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + manifest.config_fingerprint += 1; + try graph.putGraphMetricBuildManifestInBatch(&batch, "eigenvector", manifest); + try batch.commit(); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.verifyGraphMetricBuildPublishReady("eigenvector", active_job.job_id)); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("eigenvector"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_status = try graph.graphMetricStatus("eigenvector"); + defer failed_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_status.state); + try std.testing.expectEqual(published_generation, failed_status.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_status.build_job_id); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_status.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_status.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_status.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_status.recent_failures[0].phase); + + const after_failure_top = try graph.graphMetricTopK("eigenvector", 10); + defer { + for (after_failure_top) |*score| score.deinit(alloc); + alloc.free(after_failure_top); + } + try std.testing.expectEqual(before_failure_top.len, after_failure_top.len); + for (before_failure_top, after_failure_top) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new")); + } + try std.testing.expect((try graph.countGraphMetricScoreGeneration("eigenvector", active_job.score_generation)) > 0); + try drainRetiredGraphMetricScoresForTest(&graph, "eigenvector"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "eigenvector") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_job.phase); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "eigenvector", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("eigenvector", active_job.score_generation)); + } +} + +test "graph hits metrics publish authority and hub scores" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 50, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 50, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + + var authority_status = try graph.runGraphMetric("hits_authority"); + defer authority_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, authority_status.state); + try std.testing.expect(authority_status.iterations_completed > 0); + + var hub_status = try graph.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expect(hub_status.iterations_completed > 0); + try std.testing.expectEqual(authority_status.published_generation, hub_status.published_generation); + + const authorities = try graph.graphMetricTopK("hits_authority", 3); + defer { + for (authorities) |*score| score.deinit(alloc); + alloc.free(authorities); + } + try std.testing.expectEqual(@as(usize, 3), authorities.len); + try std.testing.expectEqualStrings("doc-authority", authorities[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), authorities[0].score, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), authorities[1].score, 0.001); + + const hubs = try graph.graphMetricTopK("hits_hub", 3); + defer { + for (hubs) |*score| score.deinit(alloc); + alloc.free(hubs); + } + try std.testing.expectEqual(@as(usize, 3), hubs.len); + try std.testing.expect(hubs[0].score >= hubs[1].score); + try std.testing.expect(hubs[1].score > hubs[2].score); + try std.testing.expectApproxEqAbs(@as(f64, 0.707106), hubs[0].score, 0.001); +} + +test "graph hits metric edge filter limits typed score graph" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-filter"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-filter"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const filter_types = [_][]const u8{"cites"}; + const metrics = [_]GraphMetricConfig{.{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .edge_filter = .{ .mode = .types, .types = &filter_types }, + .max_iterations = 20, + }}; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-x", "doc-y", "related", 1.0, 0, 0, ""); + + var published = try graph.runGraphMetric("hits_authority"); + defer published.deinit(alloc); + try std.testing.expectEqual(GraphMetricEdgeFilterMode.types, published.edge_filter.mode); + try std.testing.expectEqual(@as(usize, 1), published.edge_filter.types.len); + try std.testing.expectEqualStrings("cites", published.edge_filter.types[0]); + + const top = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + for (top) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-x")); + try std.testing.expect(!std.mem.eql(u8, score.node, "doc-y")); + } +} + +test "graph hits planned build publishes paired scores matching local runner" { + const alloc = std.testing.allocator; + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 8, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 8, + .tolerance = 0.000001, + }, + }; + + var local_store_buf: [256]u8 = undefined; + const local_store_path = tmpPath(&local_store_buf, "store-hits-local-parity"); + defer cleanupTmp(local_store_path); + var local_rev_buf: [256]u8 = undefined; + const local_rev_path = tmpPath(&local_rev_buf, "rev-hits-local-parity"); + defer cleanupTmp(local_rev_path); + + var local_store = try docstore.DocStore.open(alloc, local_store_path, .{}); + defer local_store.close(); + var local_graph = try openTestGraphIndex(alloc, &local_store, local_rev_path, "links", .{ .metric_configs = &metrics }); + defer local_graph.close(); + + try local_graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try local_graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try local_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var local_status = try local_graph.runGraphMetric("hits_authority"); + defer local_status.deinit(alloc); + var local_hub_status = try local_graph.graphMetricStatus("hits_hub"); + defer local_hub_status.deinit(alloc); + + var planned_store_buf: [256]u8 = undefined; + const planned_store_path = tmpPath(&planned_store_buf, "store-hits-planned-parity"); + defer cleanupTmp(planned_store_path); + var planned_rev_buf: [256]u8 = undefined; + const planned_rev_path = tmpPath(&planned_rev_buf, "rev-hits-planned-parity"); + defer cleanupTmp(planned_rev_path); + + var planned_store = try docstore.DocStore.open(alloc, planned_store_path, .{}); + defer planned_store.close(); + var planned_graph = try openTestGraphIndex(alloc, &planned_store, planned_rev_path, "links", .{ .metric_configs = &metrics }); + defer planned_graph.close(); + + try planned_graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try planned_graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try planned_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var planned_status = try planned_graph.runHitsMetricPlanned("hits_authority"); + defer planned_status.deinit(alloc); + var planned_hub_status = try planned_graph.graphMetricStatus("hits_hub"); + defer planned_hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_hub_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_hub_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(planned_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + try std.testing.expectEqual(local_hub_status.iterations_completed, planned_hub_status.iterations_completed); + try std.testing.expectEqual(local_hub_status.converged, planned_hub_status.converged); + try std.testing.expectApproxEqAbs(local_hub_status.delta, planned_hub_status.delta, 0.0000001); + + const local_authorities = try local_graph.graphMetricTopK("hits_authority", 10); + defer { + for (local_authorities) |*score| score.deinit(alloc); + alloc.free(local_authorities); + } + const planned_authorities = try planned_graph.graphMetricTopK("hits_authority", 10); + defer { + for (planned_authorities) |*score| score.deinit(alloc); + alloc.free(planned_authorities); + } + try std.testing.expectEqual(local_authorities.len, planned_authorities.len); + for (local_authorities, planned_authorities) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + const local_hubs = try local_graph.graphMetricTopK("hits_hub", 10); + defer { + for (local_hubs) |*score| score.deinit(alloc); + alloc.free(local_hubs); + } + const planned_hubs = try planned_graph.graphMetricTopK("hits_hub", 10); + defer { + for (planned_hubs) |*score| score.deinit(alloc); + alloc.free(planned_hubs); + } + try std.testing.expectEqual(local_hubs.len, planned_hubs.len); + for (local_hubs, planned_hubs) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try planned_graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, job.phase); +} + +test "graph hits active planned rebuild keeps prior published pair visible" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-active-rebuild-visible-pair"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-active-rebuild-visible-pair"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var published = try graph.runHitsMetricPlanned("hits_authority"); + defer published.deinit(alloc); + var published_hub = try graph.graphMetricStatus("hits_hub"); + defer published_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published_hub.state); + try std.testing.expectEqual(published.published_generation, published_hub.published_generation); + const published_generation = published.published_generation; + + const before_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (before_authorities) |*score| score.deinit(alloc); + alloc.free(before_authorities); + } + const before_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (before_hubs) |*score| score.deinit(alloc); + alloc.free(before_hubs); + } + try std.testing.expectEqual(@as(usize, 3), before_authorities.len); + try std.testing.expectEqual(@as(usize, 3), before_hubs.len); + + try graph.addEdge("doc-new-hub", "doc-new-authority", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + + var started = try graph.ensureGraphMetricPlannedBuild("hits_authority", rebuilding_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(rebuilding_generation, started.building_generation); + + var job_id: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + try std.testing.expectEqual(started.build_job_id, job.job_id); + } + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks }) |expected_phase| { + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-rebuild"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(!step.published); + if (step.advanced_phase) break; + } + } + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + const new_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", job_id, "authority", 1, "doc-new-authority"); + try std.testing.expect(try new_authority_key.exists(&txn)); + const new_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", job_id, "hub", 1, "doc-new-hub"); + try std.testing.expect(try new_hub_key.exists(&txn)); + } + + var rebuilding_authority = try graph.graphMetricStatus("hits_authority"); + defer rebuilding_authority.deinit(alloc); + var rebuilding_hub = try graph.graphMetricStatus("hits_hub"); + defer rebuilding_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, rebuilding_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, rebuilding_authority.phase); + try std.testing.expectEqual(published_generation, rebuilding_authority.published_generation); + try std.testing.expectEqual(published_generation, rebuilding_hub.published_generation); + + const visible_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (visible_authorities) |*score| score.deinit(alloc); + alloc.free(visible_authorities); + } + const visible_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (visible_hubs) |*score| score.deinit(alloc); + alloc.free(visible_hubs); + } + try std.testing.expectEqual(before_authorities.len, visible_authorities.len); + for (before_authorities, visible_authorities) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-authority")); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + try std.testing.expectEqual(before_hubs.len, visible_hubs.len); + for (before_hubs, visible_hubs) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-authority")); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } +} + +test "graph hits reclaimed scan page overwrites stale partial output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-scan-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-scan-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub", "doc-authority-a", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub", "doc-authority-b", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + defer graph.releaseGraphMetricBuildLease("hits_authority") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + const prepare = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-setup"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(prepare.advanced_phase); + + const partial_scan_claim = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executePageRankScanBuildPageWithLimit("hits_authority", metrics[0], active_job, partial_scan_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "hits_authority", active_job.job_id, "doc-hub"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + const attempt_out_degree_key = try graph.graphMetricBuildAttemptPageRankOutDegreePartialKeyAlloc("hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-hub"); + defer alloc.free(attempt_out_degree_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_out_degree_key)); + const node_authority_a_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("hits_authority", active_job.job_id, "doc-authority-a", 1); + defer alloc.free(node_authority_a_key); + try std.testing.expectError(error.NotFound, txn.get(node_authority_a_key)); + const attempt_node_authority_a_key = try graph.graphMetricBuildAttemptPageRankNodePartialKeyAlloc("hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, partial_scan_claim.page_id, partial_scan_claim.attempt, "doc-authority-a"); + defer alloc.free(attempt_node_authority_a_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, attempt_node_authority_a_key)); + } + + const scan_expires_at = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + break :blk page.lease_expires_at_ms; + }; + const reclaimed_scan = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1, "worker-b", scan_expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed_scan.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed_scan.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed_scan.completed_units); + try std.testing.expectEqualStrings("", reclaimed_scan.cursor); + { + var stale_out_degrees = std.StringHashMapUnmanaged(u64).empty; + defer { + var key_it = stale_out_degrees.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_out_degrees.deinit(alloc); + } + var stale_nodes = std.StringHashMapUnmanaged(void).empty; + defer { + var key_it = stale_nodes.keyIterator(); + while (key_it.next()) |key_ptr| alloc.free(key_ptr.*); + stale_nodes.deinit(alloc); + } + try stale_out_degrees.put(alloc, try alloc.dupe(u8, "doc-hub"), 99); + try stale_nodes.put(alloc, try alloc.dupe(u8, "doc-stale"), {}); + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writePageRankScanPartialsForAttempt("hits_authority", active_job, partial_scan_claim, 0, &stale_out_degrees, &stale_nodes, partial_scan_claim.worker_id, "", 0, partial_scan_claim.total_units)); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const stale_node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("hits_authority", active_job.job_id, "doc-stale", 1); + defer alloc.free(stale_node_key); + try std.testing.expectError(error.NotFound, txn.get(stale_node_key)); + const partial_out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "hits_authority", active_job.job_id, "doc-hub"); + try std.testing.expectEqual(@as(u64, 0), partial_out_degree); + } + + _ = try graph.executePageRankScanBuildPageWithLimit("hits_authority", metrics[0], active_job, reclaimed_scan, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const out_degree = try graph.aggregatePageRankOutDegreeForNode(&txn, "hits_authority", active_job.job_id, "doc-hub"); + try std.testing.expectEqual(@as(u64, 2), out_degree); + inline for (.{ "doc-hub", "doc-authority-a", "doc-authority-b" }) |node| { + const node_key = try graph.graphMetricBuildPageRankNodePartialKeyAlloc("hits_authority", active_job.job_id, node, 1); + defer alloc.free(node_key); + try std.testing.expectEqual(@as(u64, 1), try GraphIndex.readU64OrZero(&txn, node_key)); + } + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.initialize_ranks, job.phase); + } +} + +test "graph hits reclaimed initialize page overwrites stale rank output" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-initialize-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-initialize-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + defer graph.releaseGraphMetricBuildLease("hits_authority") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "hits_authority", metrics[0], "worker-setup", &.{ .prepare_generation, .scan_edges_and_out_degree }); + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .initialize_ranks, 0); + const initial_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .initialize_ranks, 0, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, initial_claim.state); + try std.testing.expectEqual(@as(u64, 1), initial_claim.attempt); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + inline for (.{ "doc-authority", "doc-hub-a", "doc-hub-b" }) |node| { + const authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 0, node); + try authority_key.write(&batch, 42.0); + const hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 0, node); + try hub_key.write(&batch, 43.0); + } + try batch.commit(); + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt( + "hits_authority", + active_job.job_id, + .initialize_ranks, + 0, + initial_claim.page_id, + "worker-b", + initial_claim.lease_expires_at_ms + 1, + ) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + + const stale_initialized = [_]GraphIndex.PageRankInitializeNode{.{ .node = "doc-authority" }}; + try std.testing.expectError(error.GraphMetricBuildPageNotLeased, graph.writeHitsInitializeOutputForAttempt("hits_authority", active_job, initial_claim, &stale_initialized, 99.0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 0, "doc-authority"); + const hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 0, "doc-authority"); + try std.testing.expectApproxEqAbs(@as(f64, 42.0), try authority_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 43.0), try hub_key.read(&txn), 0.0000001); + } + + _ = try graph.executeHitsInitializeBuildPage("hits_authority", active_job, reclaimed); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .initialize_ranks, 0)); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .initialize_ranks, 0, initial_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(page.total_units, page.completed_units); + + const expected_rank = 1.0 / @sqrt(@as(f64, 3.0)); + inline for (.{ "doc-authority", "doc-hub-a", "doc-hub-b" }) |node| { + const authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 0, node); + const hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 0, node); + try std.testing.expectApproxEqAbs(expected_rank, try authority_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(expected_rank, try hub_key.read(&txn), 0.0000001); + } + + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.iterate_contributions, job.phase); + } +} + +test "graph hits contribution and reduce pages resume from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "hits_authority", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks }); + + const contribution_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .iterate_contributions, 0, "worker-a", 2000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeHitsContributionBuildPageWithLimit("hits_authority", metrics[0], active_job, contribution_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + const partial = try graphMetricOrdinalValueForTest(&graph, &txn, "hits_authority", active_job.job_id, .iterate_contributions, 0, "doc-authority"); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), partial, 0.0); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try graph.ordinalAttemptContributionForNodeForTest(&txn, "hits_authority", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id, contribution_claim.attempt, "doc-authority"), 0.0000001); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + const renewed_contribution = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id, "worker-a", 2001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_contribution.state); + try std.testing.expectEqual(@as(u64, 1), renewed_contribution.completed_units); + try std.testing.expect(renewed_contribution.cursor.len > 0); + _ = try graph.executeHitsContributionBuildPageWithLimit("hits_authority", metrics[0], active_job, renewed_contribution, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .iterate_contributions, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const contribution = try graphMetricOrdinalValueForTest(&graph, &txn, "hits_authority", active_job.job_id, .iterate_contributions, 0, "doc-authority"); + try std.testing.expectApproxEqAbs(@sqrt(@as(f64, 3.0)), contribution, 0.0000001); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .iterate_contributions, 0, contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .reduce_ranks, 0); + const reduce_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .reduce_ranks, 0, "worker-r", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeHitsReduceBuildPageWithLimit("hits_authority", metrics[0], active_job, reduce_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 0, reduce_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .reduce_ranks, 0); + const renewed_reduce = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .reduce_ranks, 0, reduce_claim.page_id, "worker-r", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_reduce.state); + try std.testing.expectEqual(@as(u64, 1), renewed_reduce.completed_units); + try std.testing.expectEqualStrings("", renewed_reduce.cursor); + _ = try graph.executeHitsReduceBuildPageWithLimit("hits_authority", metrics[0], active_job, renewed_reduce, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .reduce_ranks, 0)); + + inline for (.{ .hits_hub_contributions, .hits_hub_reduce_ranks }) |expected_phase| { + var phase_steps: usize = 0; + while (true) { + phase_steps += 1; + try std.testing.expect(phase_steps <= 4); + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-hub"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const authority_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, "doc-authority"); + const hub_a_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, "doc-hub-a"); + const hub_a_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, "doc-hub-a"); + const authority_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, "doc-authority"); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), try authority_authority_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), try hub_a_authority_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try hub_a_hub_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try authority_hub_key.read(&txn), 0.0000001); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } +} + +test "graph hits hub contribution and hub reduce pages resume from durable cursor after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-hub-resume"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-hub-resume"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + try drainGraphMetricBuildToPublishForTest(&graph, "hits_authority", metrics[0], "worker-a", &.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_contributions, job.phase); + } + + const hub_contribution_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_contributions, 0, "worker-hc", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeHitsHubContributionBuildPageWithLimit("hits_authority", metrics[0], active_job, hub_contribution_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_contributions, 0, hub_contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expect(page.cursor.len > 0); + try std.testing.expect((try graph.hitsHubRawSummary(&txn, "hits_authority", active_job.job_id, 0)) == null); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + const renewed_hub_contribution = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_contributions, 0, hub_contribution_claim.page_id, "worker-hc", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_hub_contribution.state); + try std.testing.expectEqual(@as(u64, 1), renewed_hub_contribution.attempt); + try std.testing.expectEqual(@as(u64, 1), renewed_hub_contribution.completed_units); + try std.testing.expect(renewed_hub_contribution.cursor.len > 0); + _ = try graph.executeHitsHubContributionBuildPageWithLimit("hits_authority", metrics[0], active_job, renewed_hub_contribution, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .hits_hub_contributions, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_contributions, 0, hub_contribution_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, job.phase); + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 0); + const hub_reduce_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, "worker-hr", 5001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, hub_reduce_claim.range_kind); + _ = try graph.executeHitsHubReduceBuildPageWithLimit("hits_authority", metrics[0], active_job, hub_reduce_claim, 1); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, hub_reduce_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectEqualStrings("", page.cursor); + _ = try graph.graphMetricReduceSummaryValue(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 0); + const renewed_hub_reduce = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, hub_reduce_claim.page_id, "worker-hr", 5001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, renewed_hub_reduce.state); + try std.testing.expectEqual(@as(u64, 1), renewed_hub_reduce.attempt); + try std.testing.expectEqual(@as(u64, 1), renewed_hub_reduce.completed_units); + try std.testing.expectEqualStrings("", renewed_hub_reduce.cursor); + _ = try graph.executeHitsHubReduceBuildPageWithLimit("hits_authority", metrics[0], active_job, renewed_hub_reduce, null); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, hub_reduce_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + const hub_a_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, "doc-hub-a"); + const authority_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, "doc-authority"); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try hub_a_hub_key.read(&txn), 0.0000001); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), try authority_hub_key.read(&txn), 0.0000001); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } +} + +test "graph hits reduce pages only write their planned node range" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-reduce-page-range"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-reduce-page-range"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + var source_buf: [64]u8 = undefined; + var source_count: usize = 0; + while (source_count < graph_metric_build_target_reduce_page_units + 1) : (source_count += 1) { + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{source_count}); + try graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + } + try graph.addEdge("doc-hub-000", "doc-target-only", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions }) |expected_phase| { + var phase_steps: usize = 0; + while (true) { + phase_steps += 1; + try std.testing.expect(phase_steps <= 64); + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + if (step.advanced_phase) break; + } + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .reduce_ranks, 0); + const first_reduce = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .reduce_ranks, 0, "worker-r1", 3002) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 4), first_reduce.page_id); + var first_reduce_range_lower: []u8 = ""; + defer if (first_reduce_range_lower.len > 0) alloc.free(first_reduce_range_lower); + var first_reduce_range_upper: []u8 = ""; + defer if (first_reduce_range_upper.len > 0) alloc.free(first_reduce_range_upper); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const persisted_first_reduce = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 0, first_reduce.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, persisted_first_reduce.range_kind); + try std.testing.expect(persisted_first_reduce.range_upper.len > 0); + if (persisted_first_reduce.range_lower.len > 0) first_reduce_range_lower = try alloc.dupe(u8, persisted_first_reduce.range_lower); + if (persisted_first_reduce.range_upper.len > 0) first_reduce_range_upper = try alloc.dupe(u8, persisted_first_reduce.range_upper); + } + + var nodes = std.ArrayListUnmanaged([]u8).empty; + defer { + for (nodes.items) |node| alloc.free(node); + nodes.deinit(alloc); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try graph.collectPageRankScannedNodes(&txn, "hits_authority", active_job.job_id, &nodes); + } + var first_page_node: []const u8 = ""; + var second_page_node: []const u8 = ""; + for (nodes.items) |node| { + if (first_page_node.len == 0 and + (first_reduce_range_lower.len == 0 or std.mem.order(u8, node, first_reduce_range_lower) != .lt) and + (first_reduce_range_upper.len == 0 or std.mem.order(u8, node, first_reduce_range_upper) == .lt)) + { + first_page_node = node; + } + if (second_page_node.len == 0 and first_reduce_range_upper.len > 0 and std.mem.order(u8, node, first_reduce_range_upper) != .lt) { + second_page_node = node; + } + } + try std.testing.expect(first_page_node.len > 0); + try std.testing.expect(second_page_node.len > 0); + + _ = try graph.executeHitsReduceBuildPage("hits_authority", metrics[0], active_job, first_reduce); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .reduce_ranks, 0))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const first_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, first_page_node); + const first_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, first_page_node); + const second_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, second_page_node); + const second_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, second_page_node); + try std.testing.expect(try first_authority_key.exists(&txn)); + try std.testing.expect(!try first_hub_key.exists(&txn)); + try std.testing.expect(!try second_authority_key.exists(&txn)); + try std.testing.expect(!try second_hub_key.exists(&txn)); + + var first_reduce_count: usize = 0; + var first_reduce_contribution_sum: f64 = 0.0; + var first_reduce_rank_sum: f64 = 0.0; + for (nodes.items) |node| { + if (first_reduce_range_lower.len > 0 and std.mem.order(u8, node, first_reduce_range_lower) == .lt) continue; + if (first_reduce_range_upper.len > 0 and std.mem.order(u8, node, first_reduce_range_upper) != .lt) continue; + const authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, node); + const authority = try authority_key.read(&txn); + first_reduce_contribution_sum += authority; + first_reduce_rank_sum += authority; + first_reduce_count += 1; + } + try std.testing.expect(first_reduce_count > 0); + const persisted_first_reduce = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 0, first_reduce.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + const generic_reduce_fingerprint = GraphIndex.graphMetricPageRankReduceFingerprint(first_reduce, first_reduce_count, first_reduce_contribution_sum, first_reduce_rank_sum); + try std.testing.expectEqual(generic_reduce_fingerprint, persisted_first_reduce.output_fingerprint); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .reduce_ranks, 0); + const second_reduce = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .reduce_ranks, 0, "worker-r2", 3001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 5), second_reduce.page_id); + _ = try graph.executeHitsReduceBuildPage("hits_authority", metrics[0], active_job, second_reduce); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .reduce_ranks, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const second_authority_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "authority", 1, second_page_node); + const second_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, second_page_node); + try std.testing.expect(try second_authority_key.exists(&txn)); + try std.testing.expect(!try second_hub_key.exists(&txn)); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_contributions, job.phase); + } + + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-hub-c"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_contributions, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graphMetricOrdinalValueForTest(&graph, &txn, "hits_authority", active_job.job_id, .hits_hub_contributions, 0, first_page_node)) > 0.0); + try std.testing.expect((try graphMetricOrdinalValueForTest(&graph, &txn, "hits_authority", active_job.job_id, .hits_hub_contributions, 0, second_page_node)) > 0.0); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, job.phase); + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 0); + const first_hub_reduce = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, "worker-hr1", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.nodes, first_hub_reduce.range_kind); + var first_hub_reduce_count: usize = 0; + var first_hub_reduce_raw_sum: f64 = 0.0; + var first_hub_reduce_rank_sum: f64 = 0.0; + _ = try graph.executeHitsHubReduceBuildPage("hits_authority", metrics[0], active_job, first_hub_reduce); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0))); + var original_hub_summary: GraphIndex.HitsHubRawSummary = undefined; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const first_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, first_page_node); + const second_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, second_page_node); + try std.testing.expect(try first_hub_key.exists(&txn)); + try std.testing.expect(!try second_hub_key.exists(&txn)); + const reduce_summary_page = try graph.graphMetricReduceSummaryValue(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + const summary: GraphIndex.HitsHubRawSummary = .{ + .count = std.math.cast(usize, graph.node_count) orelse std.math.maxInt(usize), + .norm = @sqrt(reduce_summary_page.rank_sum), + .raw_fingerprint = reduce_summary_page.output_fingerprint, + .fingerprint = reduce_summary_page.output_fingerprint, + }; + try std.testing.expect(summary.count > 0); + try std.testing.expect(summary.norm > 0.0); + try std.testing.expect(summary.raw_fingerprint != 0); + try std.testing.expect(summary.fingerprint != 0); + original_hub_summary = summary; + for (nodes.items) |node| { + if (first_reduce_range_lower.len > 0 and std.mem.order(u8, node, first_reduce_range_lower) == .lt) continue; + if (first_reduce_range_upper.len > 0 and std.mem.order(u8, node, first_reduce_range_upper) != .lt) continue; + const raw_hub = try graphMetricOrdinalValueForTest(&graph, &txn, "hits_authority", active_job.job_id, .hits_hub_contributions, 0, node); + const hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, node); + const hub = try hub_key.read(&txn); + first_hub_reduce_raw_sum += raw_hub; + first_hub_reduce_rank_sum += hub; + first_hub_reduce_count += 1; + } + const persisted_first_hub_reduce = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, first_hub_reduce.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + const hits_reduce_fingerprint = GraphIndex.graphMetricHitsReduceFingerprint(first_hub_reduce, first_hub_reduce_count, first_hub_reduce_raw_sum, first_hub_reduce_rank_sum, summary); + try std.testing.expectEqual(hits_reduce_fingerprint, persisted_first_hub_reduce.output_fingerprint); + } + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 0); + const second_hub_reduce = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0, "worker-hr2", 4001) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try graph.executeHitsHubReduceBuildPage("hits_authority", metrics[0], active_job, second_hub_reduce); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const second_hub_key = graphMetricVectorSlotForTest(&graph, "hits_authority", active_job.job_id, "hub", 1, second_page_node); + try std.testing.expect(try second_hub_key.exists(&txn)); + const reduce_summary_page = try graph.graphMetricReduceSummaryValue(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 0) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectApproxEqAbs(original_hub_summary.norm, @sqrt(reduce_summary_page.rank_sum), 0.0000001); + try std.testing.expectEqual(original_hub_summary.raw_fingerprint, reduce_summary_page.output_fingerprint); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } +} + +test "graph hits planned build drains partitioned paired pages across workers" { + const alloc = std.testing.allocator; + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + + var local_store_buf: [256]u8 = undefined; + const local_store_path = tmpPath(&local_store_buf, "store-hits-local-multi-worker"); + defer cleanupTmp(local_store_path); + var local_rev_buf: [256]u8 = undefined; + const local_rev_path = tmpPath(&local_rev_buf, "rev-hits-local-multi-worker"); + defer cleanupTmp(local_rev_path); + var local_store = try docstore.DocStore.open(alloc, local_store_path, .{}); + defer local_store.close(); + var local_graph = try openTestGraphIndex(alloc, &local_store, local_rev_path, "links", .{ .metric_configs = &metrics }); + defer local_graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try local_graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + var target_buf: [64]u8 = undefined; + const target = try std.fmt.bufPrint(&target_buf, "doc-target-{d:0>3}", .{i}); + try local_graph.addEdge("doc-shared-hub", target, "cites", 1.0, 0, 0, ""); + } + try local_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var local_status = try local_graph.runGraphMetric("hits_authority"); + defer local_status.deinit(alloc); + var local_hub_status = try local_graph.graphMetricStatus("hits_hub"); + defer local_hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, local_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, local_hub_status.state); + try std.testing.expectEqual(local_status.published_generation, local_hub_status.published_generation); + + var planned_store_buf: [256]u8 = undefined; + const planned_store_path = tmpPath(&planned_store_buf, "store-hits-planned-multi-worker"); + defer cleanupTmp(planned_store_path); + var planned_rev_buf: [256]u8 = undefined; + const planned_rev_path = tmpPath(&planned_rev_buf, "rev-hits-planned-multi-worker"); + defer cleanupTmp(planned_rev_path); + var planned_store = try docstore.DocStore.open(alloc, planned_store_path, .{}); + defer planned_store.close(); + var planned_graph = try openTestGraphIndex(alloc, &planned_store, planned_rev_path, "links", .{ .metric_configs = &metrics }); + defer planned_graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try planned_graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + var target_buf: [64]u8 = undefined; + const target = try std.fmt.bufPrint(&target_buf, "doc-target-{d:0>3}", .{i}); + try planned_graph.addEdge("doc-shared-hub", target, "cites", 1.0, 0, 0, ""); + } + try planned_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + try planned_graph.acquireGraphMetricBuildLease("hits_authority", planned_graph.edge_generation); + defer planned_graph.releaseGraphMetricBuildLease("hits_authority") catch {}; + + var job_txn = try planned_graph.beginReadReverseTxn(); + const active_job = try planned_graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + { + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const manifest = try planned_graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + try std.testing.expect(manifest.page_count > GraphIndex.graph_metric_iterative_build_phases.len); + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 0, 5) orelse return error.TestExpectedGraphMetricBuildPage; + } + + var worker_a_completed: usize = 0; + var worker_b_completed: usize = 0; + var completed = false; + var steps: usize = 0; + while (steps < 240) : (steps += 1) { + const worker_id = if (steps % 2 == 0) "worker-a" else "worker-b"; + const step = try planned_graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], worker_id); + if (step.completed_page) { + if (std.mem.eql(u8, worker_id, "worker-a")) { + worker_a_completed += 1; + } else { + worker_b_completed += 1; + } + } + if (step.completed_build and step.phase == .cleanup_old_generations) { + completed = true; + break; + } + // A paired HITS phase may temporarily have no claimable page while the + // sibling metric's coordinator lease is still live. That is expected + // contention, not lack of build progress; the bounded loop below still + // catches a permanently stalled build. + } + try std.testing.expect(completed); + try std.testing.expect(worker_a_completed > 0); + try std.testing.expect(worker_b_completed > 0); + + var planned_status = try planned_graph.graphMetricStatus("hits_authority"); + defer planned_status.deinit(alloc); + var planned_hub_status = try planned_graph.graphMetricStatus("hits_hub"); + defer planned_hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_hub_status.state); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_hub_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(planned_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_hub_status.iterations_completed, planned_hub_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectEqual(local_hub_status.converged, planned_hub_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + try std.testing.expectApproxEqAbs(local_hub_status.delta, planned_hub_status.delta, 0.0000001); + + { + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try planned_graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, completed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try planned_graph.metricBuildLease(&txn, "hits_authority")) == null); + try std.testing.expect((try planned_graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) == null); + } + + const top_limit: usize = @intCast(planned_graph.node_count); + const local_authorities = try local_graph.graphMetricTopK("hits_authority", top_limit); + defer { + for (local_authorities) |*score| score.deinit(alloc); + alloc.free(local_authorities); + } + const planned_authorities = try planned_graph.graphMetricTopK("hits_authority", top_limit); + defer { + for (planned_authorities) |*score| score.deinit(alloc); + alloc.free(planned_authorities); + } + try std.testing.expectEqual(local_authorities.len, planned_authorities.len); + for (local_authorities, planned_authorities) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + const local_hubs = try local_graph.graphMetricTopK("hits_hub", top_limit); + defer { + for (local_hubs) |*score| score.deinit(alloc); + alloc.free(local_hubs); + } + const planned_hubs = try planned_graph.graphMetricTopK("hits_hub", top_limit); + defer { + for (planned_hubs) |*score| score.deinit(alloc); + alloc.free(planned_hubs); + } + try std.testing.expectEqual(local_hubs.len, planned_hubs.len); + for (local_hubs, planned_hubs) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +fn expectGraphMetricBuildPhasePageCountAtLeastForTest( + graph: *GraphIndex, + txn: anytype, + metric_name: []const u8, + job_id: u64, + phase: GraphIndex.GraphMetricBuildPhase, + iteration: u32, + min_pages: usize, +) !void { + const prefix = try graph.graphMetricBuildPagePrefixAlloc(metric_name, job_id, phase, iteration); + defer graph.alloc.free(prefix); + var cursor = try txn.openCursor(); + defer cursor.close(); + var count: usize = 0; + var entry_opt = try cursor.first(); + while (entry_opt) |entry| : (entry_opt = try cursor.next()) { + if (std.mem.startsWith(u8, entry.key, prefix)) count += 1; + } + try std.testing.expect(count >= min_pages); +} + +test "graph hits larger manifest resumes across reopen boundaries" { + const alloc = std.testing.allocator; + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + }; + + var local_store_buf: [256]u8 = undefined; + const local_store_path = tmpPath(&local_store_buf, "store-hits-local-large-manifest-reopen"); + defer cleanupTmp(local_store_path); + var local_rev_buf: [256]u8 = undefined; + const local_rev_path = tmpPath(&local_rev_buf, "rev-hits-local-large-manifest-reopen"); + defer cleanupTmp(local_rev_path); + var local_store = try docstore.DocStore.open(alloc, local_store_path, .{}); + defer local_store.close(); + var local_graph = try openTestGraphIndex(alloc, &local_store, local_rev_path, "links", .{ .metric_configs = &metrics }); + defer local_graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 9) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try local_graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + } + try local_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var local_status = try local_graph.runGraphMetric("hits_authority"); + defer local_status.deinit(alloc); + var local_hub_status = try local_graph.graphMetricStatus("hits_hub"); + defer local_hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, local_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, local_hub_status.state); + try std.testing.expectEqual(local_status.published_generation, local_hub_status.published_generation); + try std.testing.expectEqual(@as(u32, 2), local_status.iterations_completed); + try std.testing.expectEqual(local_status.iterations_completed, local_hub_status.iterations_completed); + + var planned_store_buf: [256]u8 = undefined; + const planned_store_path = tmpPath(&planned_store_buf, "store-hits-planned-large-manifest-reopen"); + defer cleanupTmp(planned_store_path); + var planned_rev_buf: [256]u8 = undefined; + const planned_rev_path = tmpPath(&planned_rev_buf, "rev-hits-planned-large-manifest-reopen"); + defer cleanupTmp(planned_rev_path); + var planned_store = try docstore.DocStore.open(alloc, planned_store_path, .{}); + defer planned_store.close(); + var planned_graph = try openTestGraphIndex(alloc, &planned_store, planned_rev_path, "links", .{ .metric_configs = &metrics }); + defer planned_graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 9) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try planned_graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + } + try planned_graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + const target_generation = planned_graph.edge_generation; + + try planned_graph.acquireGraphMetricBuildLease("hits_authority", target_generation); + var initial_manifest_page_count: usize = 0; + var active_job = blk: { + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try planned_graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + const manifest = try planned_graph.metricBuildManifest(&txn, "hits_authority", job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + initial_manifest_page_count = manifest.page_count; + try std.testing.expect(manifest.page_count >= GraphIndex.graph_metric_hits_build_phases.len + 10); + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", job.job_id, .scan_edges_and_out_degree, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", job.job_id, .scan_edges_and_out_degree, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", job.job_id, .initialize_ranks, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + _ = try planned_graph.metricBuildPage(&txn, "hits_authority", job.job_id, .reduce_ranks, 0, 4) orelse return error.TestExpectedGraphMetricBuildPage; + break :blk job; + }; + + var worker_a_completed: usize = 0; + var worker_b_completed: usize = 0; + var worker_c_completed: usize = 0; + var reopen_count: usize = 0; + var observed_second_iteration = false; + var observed_dynamic_manifest_growth = false; + var observed_hub_contribution_pages = false; + var observed_hub_reduce_pages = false; + var completed = false; + var steps: usize = 0; + while (steps < 512) : (steps += 1) { + const worker_id = switch (steps % 3) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + const step = try planned_graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], worker_id); + if (step.completed_page) { + if (std.mem.eql(u8, worker_id, "worker-a")) { + worker_a_completed += 1; + } else if (std.mem.eql(u8, worker_id, "worker-b")) { + worker_b_completed += 1; + } else { + worker_c_completed += 1; + } + } + if (step.completed_build and step.phase == .cleanup_old_generations) { + completed = true; + break; + } + // A paired HITS phase may temporarily have no claimable page while the + // sibling metric's coordinator lease is still live. The overall step + // bound remains the liveness assertion. + { + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + if (try planned_graph.metricBuildJob(&txn, "hits_authority")) |job| { + if (job.job_id == active_job.job_id) { + observed_second_iteration = observed_second_iteration or job.iteration == 1; + switch (job.phase) { + .hits_hub_contributions => { + try expectGraphMetricBuildPhasePageCountAtLeastForTest(&planned_graph, &txn, "hits_authority", active_job.job_id, .hits_hub_contributions, job.iteration, 2); + observed_hub_contribution_pages = true; + }, + .hits_hub_reduce_ranks => { + try expectGraphMetricBuildPhasePageCountAtLeastForTest(&planned_graph, &txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, job.iteration, 2); + observed_hub_reduce_pages = true; + }, + else => {}, + } + } + } + } + + if (steps % 5 == 4) { + planned_graph.close(); + planned_graph = try openTestGraphIndex(alloc, &planned_store, planned_rev_path, "links", .{ .metric_configs = &metrics }); + reopen_count += 1; + var status = try planned_graph.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + var hub_status = try planned_graph.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(status.published_generation, hub_status.published_generation); + if (status.state == .building) { + try std.testing.expectEqual(active_job.job_id, status.build_job_id); + try std.testing.expect(status.build_pages.len <= graph_metric_status_page_limit); + } + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const maybe_job = try planned_graph.metricBuildJob(&txn, "hits_authority"); + if (maybe_job) |job| { + if (job.job_id == active_job.job_id) active_job = job; + if (try planned_graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) |manifest| { + observed_dynamic_manifest_growth = observed_dynamic_manifest_growth or manifest.page_count > initial_manifest_page_count; + } + } + } + } + try std.testing.expect(completed); + try std.testing.expect(reopen_count >= 3); + try std.testing.expect(observed_second_iteration); + try std.testing.expect(observed_dynamic_manifest_growth); + try std.testing.expect(observed_hub_contribution_pages); + try std.testing.expect(observed_hub_reduce_pages); + try std.testing.expect(worker_a_completed > 0); + try std.testing.expect(worker_b_completed > 0); + try std.testing.expect(worker_c_completed > 0); + + var planned_status = try planned_graph.graphMetricStatus("hits_authority"); + defer planned_status.deinit(alloc); + var planned_hub_status = try planned_graph.graphMetricStatus("hits_hub"); + defer planned_hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_status.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, planned_hub_status.state); + try std.testing.expectEqual(target_generation, planned_status.published_generation); + try std.testing.expectEqual(planned_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_hub_status.iterations_completed, planned_hub_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectEqual(local_hub_status.converged, planned_hub_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + try std.testing.expectApproxEqAbs(local_hub_status.delta, planned_hub_status.delta, 0.0000001); + try std.testing.expectEqual(@as(usize, 0), planned_status.build_pages.len); + try std.testing.expectEqual(@as(usize, 0), planned_hub_status.build_pages.len); + + { + var txn = try planned_graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try planned_graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, completed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try planned_graph.metricBuildLease(&txn, "hits_authority")) == null); + try std.testing.expect((try planned_graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) == null); + } + + const top_limit: usize = @intCast(planned_graph.node_count); + const local_authorities = try local_graph.graphMetricTopK("hits_authority", top_limit); + defer { + for (local_authorities) |*score| score.deinit(alloc); + alloc.free(local_authorities); + } + const planned_authorities = try planned_graph.graphMetricTopK("hits_authority", top_limit); + defer { + for (planned_authorities) |*score| score.deinit(alloc); + alloc.free(planned_authorities); + } + try std.testing.expectEqual(local_authorities.len, planned_authorities.len); + for (local_authorities, planned_authorities) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + const local_hubs = try local_graph.graphMetricTopK("hits_hub", top_limit); + defer { + for (local_hubs) |*score| score.deinit(alloc); + alloc.free(local_hubs); + } + const planned_hubs = try planned_graph.graphMetricTopK("hits_hub", top_limit); + defer { + for (planned_hubs) |*score| score.deinit(alloc); + alloc.free(planned_hubs); + } + try std.testing.expectEqual(local_hubs.len, planned_hubs.len); + for (local_hubs, planned_hubs) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "graph hits planned drain runs through metric-name worker and coordinator boundary" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-planned-drain"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-planned-drain"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + for (0..graph_metric_build_target_scan_page_units + 1) |i| { + var source_buf: [64]u8 = undefined; + const source = try std.fmt.bufPrint(&source_buf, "doc-hub-{d:0>3}", .{i}); + try graph.addEdge(source, "doc-authority", "cites", 1.0, 0, 0, ""); + } + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + const target_generation = graph.edge_generation; + const worker_ids = [_][]const u8{ "worker-a", "worker-b", "worker-c" }; + var status = try graph.runGraphMetricPlannedDrain("hits_authority", target_generation, .{ + .worker_ids = &worker_ids, + .max_steps = 256, + }); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(u32, 1), status.iterations_completed); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + + var hub_status = try graph.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expectEqual(status.published_generation, hub_status.published_generation); + try std.testing.expectEqual(status.iterations_completed, hub_status.iterations_completed); + try std.testing.expectEqual(status.converged, hub_status.converged); + + const authorities = try graph.graphMetricTopK("hits_authority", 3); + defer { + for (authorities) |*score| score.deinit(alloc); + alloc.free(authorities); + } + try std.testing.expect(authorities.len >= 2); + try std.testing.expectEqualStrings("doc-authority", authorities[0].node); + + const hubs = try graph.graphMetricTopK("hits_hub", 3); + defer { + for (hubs) |*score| score.deinit(alloc); + alloc.free(hubs); + } + try std.testing.expect(hubs.len >= 2); + try std.testing.expect(hubs[0].score >= hubs[1].score); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "hits_authority")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", completed_job.job_id)) == null); + } +} + +test "graph hits later iteration failed pages retry and advance" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-later-iteration-retry"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-later-iteration-retry"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence }) |expected_phase| { + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .reduce_ranks, 1); + const reduce_failed_claim = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", 3000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), reduce_failed_claim.attempt); + const reduce_failed = try graph.failGraphMetricBuildPage("hits_authority", active_job.job_id, .reduce_ranks, 1, 4, "worker-fail-reduce", "later hits reduce failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, reduce_failed.state); + try std.testing.expectEqualStrings("later hits reduce failed", reduce_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .reduce_ranks, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-retry-reduce"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 0), summary.failed_pages); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, job.phase); + } + + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 1); + const hub_reduce_failed_claim = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, "worker-fail-hub-reduce", 3750) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), hub_reduce_failed_claim.attempt); + const hub_reduce_failed = try graph.failGraphMetricBuildPage("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, hub_reduce_failed_claim.page_id, "worker-fail-hub-reduce", "later hits hub reduce failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, hub_reduce_failed.state); + try std.testing.expectEqualStrings("later hits hub reduce failed", hub_reduce_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + { + var phase_steps: usize = 0; + while (true) { + phase_steps += 1; + try std.testing.expect(phase_steps <= 4); + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-retry-hub-reduce"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, hub_reduce_failed_claim.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 0), summary.failed_pages); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, job.phase); + } + + const check_failed_claim = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .check_convergence, 1, 7, "worker-fail-check", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(@as(u64, 1), check_failed_claim.attempt); + const check_failed = try graph.failGraphMetricBuildPage("hits_authority", active_job.job_id, .check_convergence, 1, 7, "worker-fail-check", "later hits check failed"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, check_failed.state); + try std.testing.expectEqualStrings("later hits check failed", check_failed.last_error); + try std.testing.expect(!(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .check_convergence, 1))); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.failed, summary.state); + try std.testing.expectEqual(@as(u64, 1), summary.failed_pages); + } + { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-retry-check"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.check_convergence, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + try std.testing.expect(step.advanced_phase); + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .check_convergence, 1, 7) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectEqual(@as(u64, 2), page.attempt); + const summary = try graph.metricBuildPhaseSummary(&txn, "hits_authority", active_job.job_id, .check_convergence, 1) orelse return error.TestExpectedGraphMetricBuildPhaseSummary; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhaseState.complete, summary.state); + try std.testing.expectEqual(@as(u64, 0), summary.failed_pages); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.metricBuildIterationSummary(&txn, "hits_authority", active_job.job_id, 1) orelse return error.TestExpectedGraphMetricBuildIterationSummary; + } +} + +test "graph hits later iteration exhausted page fails pair and preserves prior published pair" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-later-iteration-exhausted-preserves-pair"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-later-iteration-exhausted-preserves-pair"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.0, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + var published = try graph.runHitsMetricPlanned("hits_authority"); + defer published.deinit(alloc); + var published_hub = try graph.graphMetricStatus("hits_hub"); + defer published_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published_hub.state); + try std.testing.expectEqual(published.published_generation, published_hub.published_generation); + const published_generation = published.published_generation; + + const before_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (before_authorities) |*score| score.deinit(alloc); + alloc.free(before_authorities); + } + const before_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (before_hubs) |*score| score.deinit(alloc); + alloc.free(before_hubs); + } + try std.testing.expectEqual(@as(usize, 3), before_authorities.len); + try std.testing.expectEqual(@as(usize, 3), before_hubs.len); + + try graph.addEdge("doc-new-hub", "doc-authority", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("hits_authority", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence }) |expected_phase| { + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + _ = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .reduce_ranks, 1, 4) orelse return error.TestExpectedGraphMetricBuildPage; + } + + try drainGraphMetricBuildToPublishForTest(&graph, "hits_authority", metrics[0], "worker-advance-to-hub-reduce", &.{.reduce_ranks}); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + var exhausted_hub_reduce_page_id: u64 = 0; + var attempt: u64 = 0; + while (attempt < graph_metric_build_max_page_attempts) : (attempt += 1) { + const worker_id = switch (attempt) { + 0 => "worker-a", + 1 => "worker-b", + else => "worker-c", + }; + try drainGraphMetricSummaryForTest(&graph, "hits_authority", active_job, .hits_hub_reduce_ranks, 1); + const page = try graph.claimNextGraphMetricBuildPageAt("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, worker_id, 2000 + attempt) orelse return error.TestExpectedGraphMetricBuildPage; + if (attempt == 0) { + exhausted_hub_reduce_page_id = page.page_id; + } else { + try std.testing.expectEqual(exhausted_hub_reduce_page_id, page.page_id); + } + try std.testing.expectEqual(attempt + 1, page.attempt); + _ = try graph.failGraphMetricBuildPage("hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, page.page_id, worker_id, "retryable later hits hub reduce failure"); + } + + graph.close(); + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .hits_hub_reduce_ranks, 1, exhausted_hub_reduce_page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, page.state); + try std.testing.expectEqual(@as(u64, graph_metric_build_max_page_attempts), page.attempt); + try std.testing.expectEqualStrings("retryable later hits hub reduce failure", page.last_error); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, job.phase); + try std.testing.expectEqual(@as(u32, 1), job.iteration); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_authority = try graph.graphMetricStatus("hits_authority"); + defer failed_authority.deinit(alloc); + var failed_hub = try graph.graphMetricStatus("hits_hub"); + defer failed_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_hub.state); + try std.testing.expectEqual(published_generation, failed_authority.published_generation); + try std.testing.expectEqual(published_generation, failed_hub.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_authority.build_job_id); + try std.testing.expectEqual(@as(u64, 0), failed_hub.build_job_id); + const exhaustion_reason = try std.fmt.allocPrint( + alloc, + "GraphMetricBuildPageAttemptsExhausted: phase=hits_hub_reduce_ranks, iteration=1, page_id={d}, attempt=3, cause=retryable later hits hub reduce failure", + .{exhausted_hub_reduce_page_id}, + ); + defer alloc.free(exhaustion_reason); + try std.testing.expectEqualStrings(exhaustion_reason, failed_authority.last_error); + try std.testing.expectEqualStrings(exhaustion_reason, failed_hub.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_authority.recent_failures.len); + try std.testing.expectEqual(@as(usize, 1), failed_hub.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_authority.recent_failures[0].job_id); + try std.testing.expectEqual(active_job.job_id, failed_hub.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, failed_authority.recent_failures[0].phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, failed_hub.recent_failures[0].phase); + try std.testing.expectEqual(@as(u32, 1), failed_authority.recent_failures[0].iteration); + try std.testing.expectEqual(@as(u32, 1), failed_hub.recent_failures[0].iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_authority.recent_failures[0].last_error); + try std.testing.expectEqualStrings(exhaustion_reason, failed_hub.recent_failures[0].last_error); + + const after_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (after_authorities) |*score| score.deinit(alloc); + alloc.free(after_authorities); + } + const after_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (after_hubs) |*score| score.deinit(alloc); + alloc.free(after_hubs); + } + try std.testing.expectEqual(before_authorities.len, after_authorities.len); + for (before_authorities, after_authorities) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + try std.testing.expectEqual(before_hubs.len, after_hubs.len); + for (before_hubs, after_hubs) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.hits_hub_reduce_ranks, failed_job.phase); + try std.testing.expectEqual(@as(u32, 1), failed_job.iteration); + try std.testing.expectEqualStrings(exhaustion_reason, failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_authority", active_job.score_generation)); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_hub", active_job.score_generation)); + } +} + +test "graph hits coordinator publish failure preserves prior published pair after reopen" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-publish-failure-preserves-pair"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-publish-failure-preserves-pair"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + var published = try graph.runHitsMetricPlanned("hits_authority"); + defer published.deinit(alloc); + var published_hub = try graph.graphMetricStatus("hits_hub"); + defer published_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published_hub.state); + try std.testing.expectEqual(published.published_generation, published_hub.published_generation); + const published_generation = published.published_generation; + + const before_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (before_authorities) |*score| score.deinit(alloc); + alloc.free(before_authorities); + } + const before_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (before_hubs) |*score| score.deinit(alloc); + alloc.free(before_hubs); + } + try std.testing.expectEqual(@as(usize, 3), before_authorities.len); + try std.testing.expectEqual(@as(usize, 3), before_hubs.len); + + try graph.addEdge("doc-new-hub", "doc-authority", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("hits_authority", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const active_job = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + break :blk job; + }; + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence }) |expected_phase| { + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + // Adjacency compaction may checkpoint before the page completes. + if (step.advanced_phase) break; + } + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + _ = try graph.verifyGraphMetricBuildPublishReady("hits_authority", active_job.job_id); + } + const missing_pair_rank_key = try graphMetricVectorKeyForNodeForTest(&graph, "hits_authority", active_job.job_id, "hub", active_job.iteration + 1, "doc-hub-a"); + defer alloc.free(missing_pair_rank_key); + const missing_pair_rank_value = blk: { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + break :blk try alloc.dupe(u8, try txn.get(missing_pair_rank_key)); + }; + defer alloc.free(missing_pair_rank_value); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.delete(missing_pair_rank_key); + try batch.commit(); + } + const rejected_pair_materialization = try graph.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-missing-pair-rank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, rejected_pair_materialization.phase); + try std.testing.expect(rejected_pair_materialization.claimed_page); + try std.testing.expect(!rejected_pair_materialization.completed_page); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .publish_generation, active_job.iteration, rejected_pair_materialization.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.failed, failed_page.state); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_page.last_error); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_authority", active_job.score_generation)); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_hub", active_job.score_generation)); + } + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try batch.put(missing_pair_rank_key, missing_pair_rank_value); + try batch.commit(); + } + _ = try materializeGraphMetricPublishPagesForTest(&graph, "hits_authority", "worker-publish-materialize"); + + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + var manifest = try graph.metricBuildManifest(&batch, "hits_authority", active_job.job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + manifest.config_fingerprint += 1; + try graph.putGraphMetricBuildManifestInBatch(&batch, "hits_authority", manifest); + try batch.commit(); + } + graph.close(); + + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, graph.verifyGraphMetricBuildPublishReady("hits_authority", active_job.job_id)); + } + + const failed_step = try graph.runGraphMetricPlannedCoordinatorStepForMetric("hits_authority"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_step.phase); + try std.testing.expect(failed_step.failed_build); + try std.testing.expect(!failed_step.advanced_phase); + try std.testing.expect(!failed_step.published); + + var failed_authority = try graph.graphMetricStatus("hits_authority"); + defer failed_authority.deinit(alloc); + var failed_hub = try graph.graphMetricStatus("hits_hub"); + defer failed_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_hub.state); + try std.testing.expectEqual(published_generation, failed_authority.published_generation); + try std.testing.expectEqual(published_generation, failed_hub.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_authority.build_job_id); + try std.testing.expectEqual(@as(u64, 0), failed_hub.build_job_id); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_authority.last_error); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_hub.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_authority.recent_failures.len); + try std.testing.expectEqual(@as(usize, 1), failed_hub.recent_failures.len); + try std.testing.expectEqual(active_job.job_id, failed_authority.recent_failures[0].job_id); + try std.testing.expectEqual(active_job.job_id, failed_hub.recent_failures[0].job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_authority.recent_failures[0].phase); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_hub.recent_failures[0].phase); + + const after_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (after_authorities) |*score| score.deinit(alloc); + alloc.free(after_authorities); + } + const after_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (after_hubs) |*score| score.deinit(alloc); + alloc.free(after_hubs); + } + try std.testing.expectEqual(before_authorities.len, after_authorities.len); + for (before_authorities, after_authorities) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + try std.testing.expectEqual(before_hubs.len, after_hubs.len); + for (before_hubs, after_hubs) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + try std.testing.expect((try graph.countGraphMetricScoreGeneration("hits_authority", active_job.score_generation)) > 0); + try std.testing.expect((try graph.countGraphMetricScoreGeneration("hits_hub", active_job.score_generation)) > 0); + try drainRetiredGraphMetricScoresForTest(&graph, "hits_authority"); + try drainRetiredGraphMetricScoresForTest(&graph, "hits_hub"); + + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(active_job.job_id, failed_job.job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, failed_job.phase); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_authority", active_job.score_generation)); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_hub", active_job.score_generation)); + } +} + +test "graph hits reclaimed contribution and reduce pages overwrite stale output" { + try expectGraphMetricOrdinalTakeoverForTest(.hits_authority); +} + +test "graph hits convergence page reclaim recomputes without stale partial summary" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-convergence-reclaim"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-convergence-reclaim"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); + defer graph.releaseGraphMetricBuildLease("hits_authority") catch {}; + + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); + + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks }) |expected_phase| { + var phase_steps: usize = 0; + while (true) { + phase_steps += 1; + try std.testing.expect(phase_steps <= 4); + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; + } + } + + _ = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .check_convergence, 0, 7, "worker-a", 4000) orelse return error.TestExpectedGraphMetricBuildPage; + var check_claim: GraphIndex.GraphMetricBuildPage = undefined; + var check_range_lower: []u8 = ""; + defer if (check_range_lower.len > 0) alloc.free(check_range_lower); + var check_range_upper: []u8 = ""; + defer if (check_range_upper.len > 0) alloc.free(check_range_upper); + var check_output_prefix: []u8 = ""; + defer if (check_output_prefix.len > 0) alloc.free(check_output_prefix); + var check_worker_id: []u8 = ""; + defer if (check_worker_id.len > 0) alloc.free(check_worker_id); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const stored = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .check_convergence, 0, 7) orelse return error.TestExpectedGraphMetricBuildPage; + check_range_lower = try alloc.dupe(u8, stored.range_lower); + check_range_upper = try alloc.dupe(u8, stored.range_upper); + check_output_prefix = try alloc.dupe(u8, stored.output_prefix); + check_worker_id = try alloc.dupe(u8, stored.worker_id); + check_claim = stored; + } + check_claim.range_lower = check_range_lower; + check_claim.range_upper = check_range_upper; + check_claim.output_prefix = check_output_prefix; + check_claim.worker_id = check_worker_id; + check_claim.completed_units = 1; + check_claim.cursor = "stale-hits-check"; + check_claim.max_delta = 99.0; + check_claim.total_delta = 99.0; + check_claim.rank_sum = 99.0; + check_claim.converged = true; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.putGraphMetricBuildPageInBatch(&batch, "hits_authority", check_claim); + try batch.commit(); + } + + var expires_at: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .check_convergence, 0, 7) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, page.state); + try std.testing.expectEqual(@as(u64, 1), page.completed_units); + try std.testing.expectApproxEqAbs(@as(f64, 99.0), page.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 99.0), page.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 99.0), page.rank_sum, 0.0); + try std.testing.expect(page.converged); + expires_at = page.lease_expires_at_ms; + } + + const reclaimed = try graph.claimGraphMetricBuildPageAt("hits_authority", active_job.job_id, .check_convergence, 0, 7, "worker-b", expires_at + 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, reclaimed.state); + try std.testing.expectEqual(@as(u64, 2), reclaimed.attempt); + try std.testing.expectEqual(@as(u64, 0), reclaimed.completed_units); + try std.testing.expectEqualStrings("", reclaimed.cursor); + try std.testing.expectEqual(@as(u64, 0), reclaimed.output_fingerprint); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.max_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.total_delta, 0.0); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), reclaimed.rank_sum, 0.0); + try std.testing.expect(!reclaimed.converged); + + const completed_check = try graph.executeHitsConvergenceBuildPage("hits_authority", metrics[0], active_job, reclaimed); + try std.testing.expect(completed_check.completed_page); + try std.testing.expectApproxEqAbs(1.0 / @sqrt(@as(f64, 3.0)), completed_check.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0) + 1.0 / @sqrt(@as(f64, 3.0)), completed_check.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0) + @sqrt(@as(f64, 3.0)), completed_check.rank_sum, 0.0000001); + try std.testing.expect(try graph.advanceGraphMetricBuildPhaseIfReady("hits_authority", active_job.job_id, .check_convergence, 0)); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .check_convergence, 0, 7) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, page.state); + try std.testing.expectApproxEqAbs(completed_check.max_delta, page.max_delta, 0.0000001); + try std.testing.expectApproxEqAbs(completed_check.total_delta, page.total_delta, 0.0000001); + try std.testing.expectApproxEqAbs(completed_check.rank_sum, page.rank_sum, 0.0000001); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, job.phase); + } +} + +test "graph hits cleanup page resumes after reopen with published pair visible" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-cleanup-cursor"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-cleanup-cursor"); + defer cleanupTmp(rev_path); - const owned_pairs = try self.mainStoreScanRange(alloc, range_lower, range_upper); - defer backend_scan.freeResults(alloc, owned_pairs); + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 1, + .tolerance = 0.000001, + }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); - var outgoing_batch = try self.beginWriteOutgoingBatch(); - errdefer outgoing_batch.abort(); - var reverse_txn = try self.beginWriteReverseTxn(); - errdefer reverse_txn.abort(); + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.acquireGraphMetricBuildLease("hits_authority", try graph.graphMetricCurrentGeneration("hits_authority")); - for (owned_pairs) |pair| { - var parsed = (try parseOutgoingEdgeKeyAlloc(alloc, pair.key)) orelse continue; - defer parsed.deinit(alloc); - if (!std.mem.eql(u8, parsed.index_name, self.index_name)) continue; + var job_txn = try graph.beginReadReverseTxn(); + const active_job = try graph.metricBuildJob(&job_txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_txn.abort(); - const rev_key = try reverseEdgeKeyAlloc(alloc, parsed.target, self.index_name, parsed.edge_type, parsed.source); - defer alloc.free(rev_key); - outgoing_batch.delete(pair.key) catch |err| switch (err) { - error.NotFound => {}, - else => return err, - }; - reverse_txn.delete(rev_key) catch |err| switch (err) { - error.NotFound => {}, - else => return err, - }; - removed += 1; + inline for (.{ .prepare_generation, .scan_edges_and_out_degree, .initialize_ranks, .iterate_contributions, .reduce_ranks, .hits_hub_contributions, .hits_hub_reduce_ranks, .check_convergence }) |expected_phase| { + while (true) { + const step = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(expected_phase, step.phase); + try std.testing.expect(step.claimed_page); + try std.testing.expect(step.completed_page); + if (step.advanced_phase) break; } - - // Reverse rows are projections of source-owned outgoing edges, not - // target-owned records. Keep projections whose target moved to another - // range; distributed incoming reads fan out across source owners. The - // loop above already removes the exact reverse projection for every - // outgoing edge whose source is leaving this range. - // - // Match normal graph batch publication order: make forward ownership - // authoritative first, then retire the corresponding projections. - try outgoing_batch.commit(); - try reverse_txn.commit(); - try self.rebuildCounterMetadata(); - return removed; } + const publish = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); - fn mainStoreScanPrefix(self: *GraphIndex, alloc: Allocator, prefix: []const u8) ![]backend_scan.OwnedKVPair { - return try backend_scan.scanPrefix(alloc, &self.outgoing_store, prefix); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expectEqual(active_job.job_id, job.job_id); + const hub_raw_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, 0) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, hub_raw_cleanup.range_kind); + try std.testing.expect(hub_raw_cleanup.output_prefix.len > 0); + const summary_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, 1) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, summary_cleanup.range_kind); + try std.testing.expect(summary_cleanup.output_prefix.len > 0); + const rank_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, 2) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, rank_cleanup.range_kind); + try std.testing.expect(rank_cleanup.output_prefix.len > 0); + const final_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, 3) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageRangeKind.job_control, final_cleanup.range_kind); + try std.testing.expect(final_cleanup.output_prefix.len > 0); + try std.testing.expect(!std.mem.eql(u8, hub_raw_cleanup.output_prefix, summary_cleanup.output_prefix)); + try std.testing.expect(!std.mem.eql(u8, summary_cleanup.output_prefix, rank_cleanup.output_prefix)); + try std.testing.expect(!std.mem.eql(u8, rank_cleanup.output_prefix, final_cleanup.output_prefix)); + try std.testing.expect((try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, 4)) == null); } - fn mainStoreScanRange(self: *GraphIndex, alloc: Allocator, lower: []const u8, upper: []const u8) ![]backend_scan.OwnedKVPair { - return try backend_scan.scanRange(alloc, &self.outgoing_store, lower, upper); + const visible_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (visible_authorities) |*score| score.deinit(alloc); + alloc.free(visible_authorities); + } + const visible_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (visible_hubs) |*score| score.deinit(alloc); + alloc.free(visible_hubs); } + try std.testing.expectEqual(@as(usize, 3), visible_authorities.len); + try std.testing.expectEqual(@as(usize, 3), visible_hubs.len); - fn containsBatchDelete( - deletes: []const BatchDelete, - source: []const u8, - target: []const u8, - edge_type: []const u8, - ) bool { - for (deletes) |delete| { - if (!std.mem.eql(u8, delete.source, source)) continue; - if (!std.mem.eql(u8, delete.target, target)) continue; - if (!std.mem.eql(u8, delete.edge_type, edge_type)) continue; - return true; + const abandoned_hub_attempt_key = try graph.graphMetricBuildAttemptHitsHubRawKeyAlloc("hits_authority", active_job.job_id, .hits_hub_contributions, 0, 999, 1, "doc-hub-abandoned"); + defer alloc.free(abandoned_hub_attempt_key); + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try GraphIndex.putF64(&batch, abandoned_hub_attempt_key, 123.0); + for (0..graph_metric_build_cleanup_delete_page_units + 6) |i| { + var node_buf: [64]u8 = undefined; + const node = try std.fmt.bufPrint(&node_buf, "cleanup-extra-{d:0>3}", .{i}); + const key = try graph.graphMetricBuildHitsHubRawKeyAlloc("hits_authority", active_job.job_id, 99, node, @intCast(i)); + defer alloc.free(key); + try GraphIndex.putF64(&batch, key, 1.0); } - return false; + try graph.putHitsHubRawSummaryInBatch(&batch, "hits_authority", active_job.job_id, 99, .{ + .count = 1, + .norm = 1.0, + .raw_fingerprint = 123, + .fingerprint = 456, + }); + const rank_key = try graph.graphMetricBuildHitsRankKeyAlloc("hits_authority", active_job.job_id, "authority", 99, "cleanup-rank-extra"); + defer alloc.free(rank_key); + try GraphIndex.putF64(&batch, rank_key, 1.0); + try batch.commit(); } - fn edgeOwnedBytes(edge: Edge) usize { - var total: usize = @sizeOf(Edge); - total = std.math.add(usize, total, edge.source.len) catch return std.math.maxInt(usize); - total = std.math.add(usize, total, edge.target.len) catch return std.math.maxInt(usize); - total = std.math.add(usize, total, edge.edge_type.len) catch return std.math.maxInt(usize); - return std.math.add(usize, total, edge.metadata.len) catch std.math.maxInt(usize); + const first_cleanup = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, first_cleanup.phase); + try std.testing.expect(first_cleanup.claimed_page); + try std.testing.expect(!first_cleanup.completed_page); + try std.testing.expect(!first_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const cleanup_page = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, first_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.leased, cleanup_page.state); + try std.testing.expect(cleanup_page.cursor.len > 0); + try std.testing.expect(cleanup_page.completed_units >= graph_metric_build_cleanup_delete_page_units); + _ = try txn.get(abandoned_hub_attempt_key); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, job.phase); + try std.testing.expectEqualStrings(cleanup_page.cursor, job.cursor); } - fn edgeScanCursorFromPhysicalKey( - alloc: Allocator, - direction: EdgeDirection, - type_index: u32, - edge: Edge, - ) !EdgeScanCursor { - const edge_type = try alloc.dupe(u8, edge.edge_type); - errdefer alloc.free(edge_type); - const adjacent_key = try alloc.dupe(u8, if (direction == .out) edge.target else edge.source); - return .{ - .direction = direction, - .type_index = type_index, - .edge_type = edge_type, - .adjacent_key = adjacent_key, - }; + const visible_after_partial_cleanup = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (visible_after_partial_cleanup) |*score| score.deinit(alloc); + alloc.free(visible_after_partial_cleanup); } + try std.testing.expectEqual(visible_authorities.len, visible_after_partial_cleanup.len); + for (visible_authorities, visible_after_partial_cleanup) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + } + graph.close(); - fn edgeScanStartCursor( - alloc: Allocator, - direction: EdgeDirection, - type_index: u32, - edge_type_name: []const u8, - ) !EdgeScanCursor { - const edge_type = try alloc.dupe(u8, edge_type_name); - errdefer alloc.free(edge_type); - return .{ - .direction = direction, - .type_index = type_index, - .edge_type = edge_type, - .adjacent_key = try alloc.alloc(u8, 0), - .at_phase_start = true, - }; + graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); + + const visible_after_reopen = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (visible_after_reopen) |*score| score.deinit(alloc); + alloc.free(visible_after_reopen); + } + try std.testing.expectEqual(visible_hubs.len, visible_after_reopen.len); + for (visible_hubs, visible_after_reopen) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + } + { + var status = try graph.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(active_job.job_id, status.build_job_id); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); } - /// Free an edge's allocated fields. - pub fn freeEdge(alloc: Allocator, edge: Edge) void { - alloc.free(edge.source); - alloc.free(edge.target); - alloc.free(edge.edge_type); - if (edge.metadata.len > 0) alloc.free(edge.metadata); + const raw_cleanup = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-clean"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, raw_cleanup.phase); + try std.testing.expect(raw_cleanup.claimed_page); + try std.testing.expect(raw_cleanup.completed_page); + try std.testing.expect(!raw_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const hub_raw_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, raw_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, hub_raw_cleanup.state); + const hub_raw_prefix = try graph.graphMetricBuildHitsHubRawNamespacePrefixAlloc("hits_authority", active_job.job_id); + defer alloc.free(hub_raw_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const maybe_raw = try cur.seekAtOrAfter(hub_raw_prefix); + if (maybe_raw) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, hub_raw_prefix)); + const hub_raw_summary_prefix = try graph.graphMetricBuildHitsHubRawSummaryNamespacePrefixAlloc("hits_authority", active_job.job_id); + defer alloc.free(hub_raw_summary_prefix); + const maybe_summary_before = try cur.seekAtOrAfter(hub_raw_summary_prefix); + try std.testing.expect(maybe_summary_before != null and std.mem.startsWith(u8, maybe_summary_before.?.key, hub_raw_summary_prefix)); } - /// Free a slice of edges returned by getEdges. - pub fn freeEdges(alloc: Allocator, edges: []Edge) void { - for (edges) |e| freeEdge(alloc, e); - alloc.free(edges); + const summary_cleanup = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-clean-summary"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, summary_cleanup.phase); + try std.testing.expect(summary_cleanup.claimed_page); + try std.testing.expect(summary_cleanup.completed_page); + try std.testing.expect(!summary_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const hub_raw_summary_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, summary_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, hub_raw_summary_cleanup.state); + const hub_raw_summary_prefix = try graph.graphMetricBuildHitsHubRawSummaryNamespacePrefixAlloc("hits_authority", active_job.job_id); + defer alloc.free(hub_raw_summary_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const maybe_summary = try cur.seekAtOrAfter(hub_raw_summary_prefix); + if (maybe_summary) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, hub_raw_summary_prefix)); + const hits_rank_prefix = try graph.graphMetricBuildHitsRankNamespacePrefixAlloc("hits_authority", active_job.job_id); + defer alloc.free(hits_rank_prefix); + const maybe_rank_before = try cur.seekAtOrAfter(hits_rank_prefix); + try std.testing.expect(maybe_rank_before != null and std.mem.startsWith(u8, maybe_rank_before.?.key, hits_rank_prefix)); } -}; -const RuntimeStoreHandle = struct { - store: backend_erased.Store, - owned: bool, -}; + const rank_cleanup = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-clean-rank"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, rank_cleanup.phase); + try std.testing.expect(rank_cleanup.claimed_page); + try std.testing.expect(rank_cleanup.completed_page); + try std.testing.expect(!rank_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const hits_rank_cleanup = try graph.metricBuildPage(&txn, "hits_authority", active_job.job_id, .cleanup_old_generations, 0, rank_cleanup.page_id) orelse return error.TestExpectedGraphMetricBuildPage; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPageState.complete, hits_rank_cleanup.state); + const hits_rank_prefix = try graph.graphMetricBuildHitsRankNamespacePrefixAlloc("hits_authority", active_job.job_id); + defer alloc.free(hits_rank_prefix); + var cur = try txn.openCursor(); + defer cur.close(); + const maybe_rank = try cur.seekAtOrAfter(hits_rank_prefix); + if (maybe_rank) |entry| try std.testing.expect(!std.mem.startsWith(u8, entry.key, hits_rank_prefix)); + } -fn initRuntimeStore(alloc: Allocator, store: anytype) !RuntimeStoreHandle { - const T = @TypeOf(store); - if (T == backend_erased.Store) return .{ .store = store, .owned = true }; - if (T == *backend_erased.Store) return .{ .store = store.*, .owned = false }; + const final_cleanup = try graph.runGraphMetricPlannedWorkerStep("hits_authority", metrics[0], "worker-clean-final"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, final_cleanup.phase); + try std.testing.expect(final_cleanup.claimed_page); + try std.testing.expect(final_cleanup.completed_page); + try std.testing.expect(final_cleanup.published); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const completed_job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.complete, completed_job.phase); + try std.testing.expect((try graph.metricBuildLease(&txn, "hits_authority")) == null); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", active_job.job_id)) == null); + try std.testing.expectError(error.NotFound, txn.get(abandoned_hub_attempt_key)); + } - switch (@typeInfo(T)) { - .pointer => |ptr| { - if (@typeInfo(ptr.child) == .@"struct" and @hasDecl(ptr.child, "backendStore")) { - return .{ - .store = try backend_erased.storeFrom(alloc, store.backendStore()), - .owned = true, - }; - } + var final_authority = try graph.graphMetricStatus("hits_authority"); + defer final_authority.deinit(alloc); + var final_hub = try graph.graphMetricStatus("hits_hub"); + defer final_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, final_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, final_hub.state); + try std.testing.expectEqual(final_authority.published_generation, final_hub.published_generation); +} + +test "graph hits failed planned build preserves prior published pair" { + const alloc = std.testing.allocator; + var store_buf: [256]u8 = undefined; + const store_path = tmpPath(&store_buf, "store-hits-planned-failure-preserves-pair"); + defer cleanupTmp(store_path); + var rev_buf: [256]u8 = undefined; + const rev_path = tmpPath(&rev_buf, "rev-hits-planned-failure-preserves-pair"); + defer cleanupTmp(rev_path); + + var store = try docstore.DocStore.open(alloc, store_path, .{}); + defer store.close(); + const metrics = [_]GraphMetricConfig{ + .{ + .name = "hits_authority", + .kind = .hits_authority, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, }, - .@"struct" => { - if (@hasDecl(T, "backendStore")) { - return .{ - .store = try backend_erased.storeFrom(alloc, store.backendStore()), - .owned = true, - }; - } + .{ + .name = "hits_hub", + .kind = .hits_hub, + .refresh = .manual, + .max_iterations = 2, + .tolerance = 0.000001, }, - else => {}, - } - - return .{ - .store = try backend_erased.storeFrom(alloc, store), - .owned = true, }; -} + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); + defer graph.close(); -// ============================================================================ -// Tests -// ============================================================================ + try graph.addEdge("doc-hub-a", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-hub-b", "doc-authority", "cites", 1.0, 0, 0, ""); + try graph.addEdge("doc-authority", "doc-authority", "cites", 1.0, 0, 0, ""); + + var published = try graph.runHitsMetricPlanned("hits_authority"); + defer published.deinit(alloc); + var published_hub = try graph.graphMetricStatus("hits_hub"); + defer published_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.fresh, published_hub.state); + try std.testing.expectEqual(published.published_generation, published_hub.published_generation); + const published_generation = published.published_generation; + + const before_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (before_authorities) |*score| score.deinit(alloc); + alloc.free(before_authorities); + } + const before_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (before_hubs) |*score| score.deinit(alloc); + alloc.free(before_hubs); + } + try std.testing.expectEqual(@as(usize, 3), before_authorities.len); + try std.testing.expectEqual(@as(usize, 3), before_hubs.len); + + try graph.addEdge("doc-new-hub", "doc-authority", "cites", 1.0, 0, 0, ""); + const rebuilding_generation = graph.edge_generation; + try std.testing.expect(rebuilding_generation > published_generation); + var building = try graph.ensureGraphMetricPlannedBuild("hits_authority", rebuilding_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(rebuilding_generation, building.building_generation); + + const prepare = try graph.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "worker-a"); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + try std.testing.expect(!prepare.advanced_phase); + + var job_id: u64 = 0; + var score_generation: u64 = 0; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + job_id = job.job_id; + score_generation = job.score_generation; + try std.testing.expectEqual(building.build_job_id, job.job_id); + try std.testing.expectEqual(rebuilding_generation, job.target_generation); + try std.testing.expectEqual(GraphIndex.GraphMetricBuildPhase.prepare_generation, job.phase); + _ = try graph.metricBuildManifest(&txn, "hits_authority", job_id) orelse return error.TestExpectedGraphMetricBuildManifest; + } -fn tmpPath(buf: []u8, label: []const u8) [*:0]const u8 { - const ns = platform_time.monotonicNs(); - const slice = std.fmt.bufPrint(buf, "/tmp/antfly-graph-{s}-{d}\x00", .{ label, ns }) catch unreachable; - var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); - defer io_impl.deinit(); - std.Io.Dir.cwd().createDirPath(io_impl.io(), std.mem.span(@as([*:0]const u8, @ptrCast(slice.ptr)))) catch {}; - return @ptrCast(slice.ptr); -} + var failed_authority = try graph.failGraphMetricPlannedBuild("hits_authority", error.InvalidGraphMetricScore); + defer failed_authority.deinit(alloc); + var failed_hub = try graph.graphMetricStatus("hits_hub"); + defer failed_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.failed, failed_hub.state); + try std.testing.expectEqual(published_generation, failed_authority.published_generation); + try std.testing.expectEqual(published_generation, failed_hub.published_generation); + try std.testing.expectEqual(@as(u64, 0), failed_authority.build_job_id); + try std.testing.expectEqual(@as(u64, 0), failed_hub.build_job_id); + try std.testing.expectEqual(@as(u64, 1), failed_authority.retry_count); + try std.testing.expectEqual(@as(u64, 1), failed_hub.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_authority.last_error); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_hub.last_error); + try std.testing.expectEqual(@as(usize, 1), failed_authority.recent_failures.len); + try std.testing.expectEqual(@as(usize, 1), failed_hub.recent_failures.len); + try std.testing.expectEqual(job_id, failed_authority.recent_failures[0].job_id); + try std.testing.expectEqual(job_id, failed_hub.recent_failures[0].job_id); + + const after_authorities = try graph.graphMetricTopK("hits_authority", 10); + defer { + for (after_authorities) |*score| score.deinit(alloc); + alloc.free(after_authorities); + } + const after_hubs = try graph.graphMetricTopK("hits_hub", 10); + defer { + for (after_hubs) |*score| score.deinit(alloc); + alloc.free(after_hubs); + } + try std.testing.expectEqual(before_authorities.len, after_authorities.len); + for (before_authorities, after_authorities) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } + try std.testing.expectEqual(before_hubs.len, after_hubs.len); + for (before_hubs, after_hubs) |before, after| { + try std.testing.expectEqualStrings(before.node, after.node); + try std.testing.expectApproxEqAbs(before.score, after.score, 0.0000001); + try std.testing.expect(!std.mem.eql(u8, after.node, "doc-new-hub")); + } -fn cleanupTmp(path: [*:0]const u8) void { - var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); - defer io_impl.deinit(); - std.Io.Dir.cwd().deleteTree(io_impl.io(), std.mem.span(path)) catch {}; + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + const failed_job = try graph.metricBuildJob(&txn, "hits_authority") orelse return error.TestExpectedGraphMetricBuildJob; + try std.testing.expectEqual(job_id, failed_job.job_id); + try std.testing.expectEqual(@as(u64, 1), failed_job.retry_count); + try std.testing.expectEqualStrings("InvalidGraphMetricScore", failed_job.last_error); + try std.testing.expect((try graph.metricBuildManifest(&txn, "hits_authority", job_id)) == null); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_authority", score_generation)); + try std.testing.expectEqual(@as(usize, 0), try graph.countGraphMetricScoreGeneration("hits_hub", score_generation)); + } + + try std.testing.expectError(error.GraphMetricBuildNotActive, graph.failGraphMetricPlannedBuild("hits_authority", error.InvalidGraphMetricScore)); } -test "graph addEdge and getEdges out" { +test "graph compatible HITS aliases share lifecycle controls and reject stale publication" { const alloc = std.testing.allocator; var store_buf: [256]u8 = undefined; - const store_path = tmpPath(&store_buf, "store"); + const store_path = tmpPath(&store_buf, "store-hits-shared-lifecycle"); defer cleanupTmp(store_path); var rev_buf: [256]u8 = undefined; - const rev_path = tmpPath(&rev_buf, "rev"); + const rev_path = tmpPath(&rev_buf, "rev-hits-shared-lifecycle"); defer cleanupTmp(rev_path); var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + const metrics = [_]GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority, .refresh = .manual }, + .{ .name = "hub", .kind = .hits_hub, .refresh = .manual }, + }; + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{ .metric_configs = &metrics }); defer graph.close(); - try graph.addEdge("doc1", "doc2", "cites", 0.9, 1000, 1001, "{}"); - try graph.addEdge("doc1", "doc3", "cites", 0.5, 1000, 1001, ""); - - const edges = try graph.getEdges(alloc, "doc1", "cites", .out); - defer GraphIndex.freeEdges(alloc, edges); + var scheduled_hub = try graph.ensureGraphMetricPlannedBuild("hub", 1); + defer scheduled_hub.deinit(alloc); + var scheduled_authority = try graph.ensureGraphMetricPlannedBuild("authority", 1); + defer scheduled_authority.deinit(alloc); + try std.testing.expectEqualStrings("hub", scheduled_hub.name); + try std.testing.expectEqual(scheduled_hub.build_job_id, scheduled_authority.build_job_id); + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expect((try graph.metricBuildJob(&txn, "authority")) != null); + try std.testing.expect((try graph.metricBuildJob(&txn, "hub")) == null); + } - try std.testing.expectEqual(@as(usize, 2), edges.len); - try std.testing.expectEqualStrings("doc1", edges[0].source); - try std.testing.expectApproxEqAbs(@as(f64, 0.9), edges[0].weight, 0.001); + var paused = try graph.pauseGraphMetricMaintenance("hub"); + defer paused.deinit(alloc); + var paused_authority = try graph.graphMetricStatus("authority"); + defer paused_authority.deinit(alloc); + try std.testing.expect(paused.maintenance_paused); + try std.testing.expect(paused_authority.maintenance_paused); + + var resumed = try graph.resumeGraphMetricMaintenance("authority"); + defer resumed.deinit(alloc); + var resumed_hub = try graph.graphMetricStatus("hub"); + defer resumed_hub.deinit(alloc); + try std.testing.expect(!resumed.maintenance_paused); + try std.testing.expect(!resumed_hub.maintenance_paused); + + try graph.deleteGraphMetricMaterialization("hub"); + var deleted_authority = try graph.graphMetricStatus("authority"); + defer deleted_authority.deinit(alloc); + var deleted_hub = try graph.graphMetricStatus("hub"); + defer deleted_hub.deinit(alloc); + try std.testing.expectEqual(GraphIndex.GraphMetricState.disabled, deleted_authority.state); + try std.testing.expectEqual(GraphIndex.GraphMetricState.disabled, deleted_hub.state); + + try graph.enableGraphMetric("authority"); + const meta_new = GraphIndex.GraphMetricMeta{ + .target_edge_generation = 2, + .config_fingerprint = GraphIndex.graphMetricConfigFingerprint(metrics[0]), + }; + { + var batch = try graph.beginWriteReverseBatch(); + errdefer batch.abort(); + try graph.publishGraphMetricPointerInBatch(&batch, "authority", 10, meta_new); + try batch.commit(); + } + { + var batch = try graph.beginWriteReverseBatch(); + defer batch.abort(); + var meta_old = meta_new; + meta_old.target_edge_generation = 1; + try std.testing.expectError(error.GraphMetricBuildSuperseded, graph.publishGraphMetricPointerInBatch(&batch, "authority", 11, meta_old)); + } + { + var txn = try graph.beginReadReverseTxn(); + defer txn.abort(); + try std.testing.expectEqual(@as(u64, 10), try graph.metricPublishedGeneration(&txn, "authority")); + try std.testing.expectEqual(@as(u64, 2), try graph.metricPublishedEdgeGeneration(&txn, "authority")); + } } test "graph both direction emits one physical self loop and preserves reciprocal edges" { @@ -1978,7 +33359,7 @@ test "graph both direction emits one physical self loop and preserves reciprocal var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); try graph.addEdge("same", "same", "loop", 1, 0, 0, "{}"); @@ -1988,11 +33369,9 @@ test "graph both direction emits one physical self loop and preserves reciprocal const outgoing = try graph.getEdges(alloc, "same", "", .out); defer GraphIndex.freeEdges(alloc, outgoing); try std.testing.expectEqual(@as(usize, 2), outgoing.len); - const incoming = try graph.getEdges(alloc, "same", "", .in); defer GraphIndex.freeEdges(alloc, incoming); try std.testing.expectEqual(@as(usize, 2), incoming.len); - const both = try graph.getEdges(alloc, "same", "", .both); defer GraphIndex.freeEdges(alloc, both); try std.testing.expectEqual(@as(usize, 3), both.len); @@ -2015,13 +33394,10 @@ test "graph durable writes reject invalid edge types before mutation" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); - try std.testing.expectError( - error.InvalidGraphEdges, - graph.addEdge("a", "b", "", 1, 0, 0, ""), - ); + try std.testing.expectError(error.InvalidGraphEdges, graph.addEdge("a", "b", "", 1, 0, 0, "")); try std.testing.expectError( error.InvalidGraphEdges, graph.addEdge("a", "b", "x" ** (edge_type_mod.max_bytes + 1), 1, 0, 0, ""), @@ -2042,7 +33418,7 @@ test "graph bounded adjacency pages preserve order and fail before budget overfl var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); for (0..5) |i| { @@ -2085,6 +33461,7 @@ test "graph bounded adjacency pages preserve order and fail before budget overfl try std.testing.expectEqual(@as(usize, 5), exact.len); var cites_bytes: usize = 0; for (exact) |edge| cites_bytes += GraphIndex.edgeOwnedBytes(edge); + try graph.addEdge("root", "doc-r", "refs", 1, 0, 0, "{}"); var typed_first = try graph.getEdgesByTypesPage( alloc, @@ -2098,6 +33475,7 @@ test "graph bounded adjacency pages preserve order and fail before budget overfl try std.testing.expectEqual(@as(usize, 5), typed_first.edges.len); try std.testing.expect(typed_first.next_cursor.?.at_phase_start); try std.testing.expectEqual(@as(u32, 1), typed_first.next_cursor.?.type_index); + var typed_second = try graph.getEdgesByTypesPage( alloc, "root", @@ -2109,6 +33487,7 @@ test "graph bounded adjacency pages preserve order and fail before budget overfl defer typed_second.deinit(alloc); try std.testing.expectEqual(@as(usize, 1), typed_second.edges.len); try std.testing.expectEqualStrings("refs", typed_second.edges[0].edge_type); + const deduplicated_types = try graph.getEdgesByTypesBounded( alloc, "root", @@ -2136,7 +33515,7 @@ test "graph addEdge and getEdges in (reverse index)" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); try graph.addEdge("a", "b", "knows", 1.0, 100, 100, ""); @@ -2169,7 +33548,7 @@ test "graph exact edge probes stay aligned and preserve payloads" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); try graph.addEdge("post:2", "tag", "HAS_TAG", 0.75, 10, 11, "{\"rank\":1}"); try graph.addEdge("post:1", "other", "HAS_TAG", 1, 10, 11, ""); @@ -2218,7 +33597,7 @@ test "graph edge keys support arbitrary document ids and edge types" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g\x00:i:", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g\x00:i:", .{}); defer graph.close(); const source = "doc\x00:i:\xff"; @@ -2267,7 +33646,7 @@ test "graph deleteEdge removes both directions" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); try graph.addEdge("x", "y", "rel", 1.0, 0, 0, ""); @@ -2293,7 +33672,7 @@ test "graph batchApply applies writes and deletes together" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "links", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "links", .{}); defer graph.close(); try graph.addEdge("a", "b", "knows", 1.0, 0, 0, ""); @@ -2327,6 +33706,7 @@ test "graph edge encoding round-trip" { try std.testing.expectEqual(@as(u64, 1234567890), decoded.created_at); try std.testing.expectEqual(@as(u64, 1234567891), decoded.updated_at); try std.testing.expectEqualStrings("{\"key\":\"val\"}", decoded.metadata); + try std.testing.expectError(error.InvalidGraphEdgeValue, decodeEdgeValue(encoded[0..23])); try std.testing.expectError( error.InvalidGraphEdges, @@ -2339,6 +33719,22 @@ test "graph edge encoding round-trip" { try std.testing.expectError(error.InvalidGraphEdgeValue, decodeEdgeValue(invalid)); } +test "graph metric reverse edge parser borrows ordinary keys and owns escaped components" { + const alloc = std.testing.allocator; + const encoded = try reverseEdgeKeyAlloc(alloc, "target", "links", "ci\x00tes", "source"); + defer alloc.free(encoded); + var parsed = (try parseMetricReverseEdgeKeyView(alloc, encoded, "links")) orelse + return error.TestUnexpectedResult; + defer parsed.deinit(alloc); + try std.testing.expectEqualStrings("source", parsed.source.bytes); + try std.testing.expectEqualStrings("ci\x00tes", parsed.edge_type.bytes); + try std.testing.expectEqualStrings("target", parsed.target.bytes); + try std.testing.expect(!parsed.source.owned); + try std.testing.expect(parsed.edge_type.owned); + try std.testing.expect(!parsed.target.owned); + try std.testing.expect((try parseMetricReverseEdgeKeyView(alloc, encoded, "other")) == null); +} + test "graph storage rejects non-finite edge weights" { const alloc = std.testing.allocator; for ([_]f64{ std.math.nan(f64), std.math.inf(f64), -std.math.inf(f64) }) |weight| { @@ -2377,7 +33773,7 @@ test "graph index persists metadata larger than the former stack buffer" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, reverse_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, reverse_path, "g", .{}); defer graph.close(); try graph.addEdge("source", "target", "references", 1.0, 10, 11, metadata); @@ -2398,7 +33794,7 @@ test "graph getEdges with edge type filter" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); try graph.addEdge("n1", "n2", "likes", 1.0, 0, 0, ""); @@ -2427,7 +33823,7 @@ test "graph deleteEdgesForDoc cleanup" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); try graph.addEdge("doc1", "doc2", "ref", 1.0, 0, 0, ""); @@ -2456,7 +33852,7 @@ test "graph rebuildReverseFromOwnedOutgoingEdges reconstructs incoming index" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); const edge_val = try encodeEdgeValueAlloc(alloc, 1.0, 10, 11, ""); @@ -2489,7 +33885,7 @@ test "graph rebuildReverseFromOwnedOutgoingEdges respects split ownership bounds var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); const edge_val = try encodeEdgeValueAlloc(alloc, 1.0, 10, 11, ""); @@ -2533,7 +33929,7 @@ test "graph pruneOwnedRange preserves reverse edges for retained cross-range sou var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); try graph.addEdge("doc:a", "doc:z", "ref", 1.0, 0, 0, ""); @@ -2577,7 +33973,7 @@ test "tree topology rejects second outgoing edge" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .edge_type_configs = &.{.{ .name = "parent", .topology = .tree }}, }); defer graph.close(); @@ -2607,7 +34003,7 @@ test "tree topology allows update to same target" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .edge_type_configs = &.{.{ .name = "parent", .topology = .tree }}, }); defer graph.close(); @@ -2637,7 +34033,7 @@ test "graph mode allows multiple outgoing edges" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); // "parent" is tree, "likes" is graph (default) - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .edge_type_configs = &.{.{ .name = "parent", .topology = .tree }}, }); defer graph.close(); @@ -2662,7 +34058,7 @@ test "graph reverse backend adapters expose txn cursor and batch operations" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); { @@ -2709,7 +34105,7 @@ test "graph stats summary counts unique nodes from reverse edges" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); try graph.addEdge("doc:a", "doc:b", "links", 1.0, 0, 0, ""); @@ -2731,7 +34127,7 @@ test "graph reverse store opens concrete txn and batch handles" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{}); + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{}); defer graph.close(); const reverse_store = graph.reverseStore(); @@ -2773,7 +34169,7 @@ test "graph reverse store persists on durable lsm backend across reopen" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .reverse_backend = .lsm, }); defer graph.close(); @@ -2787,7 +34183,7 @@ test "graph reverse store persists on durable lsm backend across reopen" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .reverse_backend = .lsm, }); defer graph.close(); @@ -2818,7 +34214,7 @@ test "graph reverse lsm durable boundary checkpoint retires retained wal" { var store = try docstore.DocStore.open(alloc, store_path, .{}); defer store.close(); - var graph = try GraphIndex.open(alloc, &store, rev_path, "g", .{ + var graph = try openTestGraphIndex(alloc, &store, rev_path, "g", .{ .reverse_backend = .lsm, .reverse_lsm_options = .{ .flush_threshold = 1024 }, }); @@ -2841,3 +34237,35 @@ test "graph reverse lsm durable boundary checkpoint retires retained wal" { }; try std.testing.expectEqual(@as(u64, 0), after.wal_retained_bytes); } + +test "graph owned build helpers remain leak-free across allocation failures" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var graph: GraphIndex = undefined; + graph.alloc = alloc; + + var values = std.StringHashMapUnmanaged(u64).empty; + defer graph.freeStringHashMapKeys(u64, &values); + const inserted = try graph.getOrPutOwnedStringMap(u64, &values, "node-a"); + inserted.value_ptr.* = 1; + const existing = try graph.getOrPutOwnedStringMap(u64, &values, "node-a"); + try std.testing.expect(existing.found_existing); + try std.testing.expectEqual(@as(u64, 1), existing.value_ptr.*); + + var cursor: []u8 = ""; + defer if (cursor.len > 0) alloc.free(cursor); + try graph.replaceOwnedBytes(&cursor, "first"); + try graph.replaceOwnedBytes(&cursor, "second"); + try std.testing.expectEqualStrings("second", cursor); + + var owned = std.ArrayListUnmanaged([]u8).empty; + defer { + for (owned.items) |item| alloc.free(item); + owned.deinit(alloc); + } + try graph.appendOwnedBytes(&owned, "value"); + try std.testing.expectEqualStrings("value", owned.items[0]); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} diff --git a/zig/pkg/antfly/src/graph/membership.zig b/zig/pkg/antfly/src/graph/membership.zig new file mode 100644 index 0000000000..2f999814c0 --- /dev/null +++ b/zig/pkg/antfly/src/graph/membership.zig @@ -0,0 +1,112 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Canonical membership, sealed by completed initialization leaves. Blocks +//! are addressed by row number, so missing output cannot silently omit nodes. +const std = @import("std"); +pub const capacity = 256; +pub const Row = struct { node: []const u8, slot: u64 }; +pub const Block = struct { + rows: [capacity]Row = undefined, + len: usize = 0, +}; +const header_len = 16; +const checksum_seed: u64 = 0xA17F_4D45_4D42_0001; + +pub fn encodeAlloc(alloc: std.mem.Allocator, rows: []const Row) ![]u8 { + if (rows.len == 0 or rows.len > capacity) return error.InvalidGraphMetricBuildManifest; + var size: usize = header_len; + for (rows, 0..) |row, i| { + if (row.node.len == 0 or row.node.len > std.math.maxInt(u32) or row.slot == 0 or + (i != 0 and (row.slot <= rows[i - 1].slot or std.mem.order(u8, rows[i - 1].node, row.node) != .lt))) + return error.InvalidGraphMetricBuildManifest; + size = try std.math.add(usize, size, try std.math.add(usize, 12, row.node.len)); + } + const bytes = try alloc.alloc(u8, size); + @memcpy(bytes[0..4], "GMB1"); + std.mem.writeInt(u32, bytes[4..8], @intCast(rows.len), .little); + var pos: usize = header_len; + for (rows) |row| { + std.mem.writeInt(u64, bytes[pos..][0..8], row.slot, .little); + std.mem.writeInt(u32, bytes[pos + 8 ..][0..4], @intCast(row.node.len), .little); + pos += 12; + @memcpy(bytes[pos..][0..row.node.len], row.node); + pos += row.node.len; + } + std.mem.writeInt(u64, bytes[8..16], std.hash.Wyhash.hash(checksum_seed, bytes[header_len..]), .little); + return bytes; +} + +/// Rows borrow the transaction's bytes. No per-node decode allocations. +pub fn decode(bytes: []const u8) !Block { + if (bytes.len < header_len or !std.mem.eql(u8, bytes[0..4], "GMB1")) return error.InvalidGraphMetricBuildManifest; + const count = std.mem.readInt(u32, bytes[4..8], .little); + if (count == 0 or count > capacity or std.mem.readInt(u64, bytes[8..16], .little) != std.hash.Wyhash.hash(checksum_seed, bytes[header_len..])) + return error.InvalidGraphMetricBuildManifest; + var result = Block{ .len = count }; + var pos: usize = header_len; + for (result.rows[0..count], 0..) |*row, i| { + if (bytes.len - pos < 12) return error.InvalidGraphMetricBuildManifest; + const slot = std.mem.readInt(u64, bytes[pos..][0..8], .little); + const len = std.mem.readInt(u32, bytes[pos + 8 ..][0..4], .little); + pos += 12; + if (slot == 0 or len == 0 or len > bytes.len - pos) return error.InvalidGraphMetricBuildManifest; + row.* = .{ .slot = slot, .node = bytes[pos..][0..len] }; + if (i != 0 and (slot <= result.rows[i - 1].slot or std.mem.order(u8, result.rows[i - 1].node, row.node) != .lt)) + return error.InvalidGraphMetricBuildManifest; + pos += len; + } + if (pos != bytes.len) return error.InvalidGraphMetricBuildManifest; + return result; +} + +/// Bind framing to the addressed block and the completed leaf's exact count. +/// A valid checksum alone cannot detect a block copied to the wrong key. +pub fn decodeSealed(bytes: []const u8, leaf: u64, block_id: u64, count: u64, lower: []const u8, upper: []const u8) !Block { + const start = std.math.mul(u64, block_id, capacity) catch return error.InvalidGraphMetricBuildManifest; + if (start >= count or count > std.math.maxInt(u32) or leaf >= std.math.maxInt(u32)) return error.InvalidGraphMetricBuildManifest; + const block = try decode(bytes); + if (block.len != @min(capacity, count - start)) return error.InvalidGraphMetricBuildManifest; + for (block.rows[0..block.len], 0..) |row, i| { + if (row.slot != (((leaf + 1) << 32) | (start + i))) return error.InvalidGraphMetricBuildManifest; + if (std.mem.order(u8, row.node, lower) == .lt or + (upper.len != 0 and std.mem.order(u8, row.node, upper) != .lt)) return error.InvalidGraphMetricBuildManifest; + } + return block; +} + +test "graph metric membership blocks own framing and reject omissions or corruption" { + const alloc = std.testing.allocator; + const raw = try encodeAlloc(alloc, &.{ .{ .node = "a", .slot = 1 }, .{ .node = "z", .slot = 256 } }); + defer alloc.free(raw); + const block = try decode(raw); + try std.testing.expectEqual(@as(usize, 2), block.len); + try std.testing.expectEqualStrings("z", block.rows[1].node); + try std.testing.expectEqual(@as(u64, 256), block.rows[1].slot); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decode(raw[0 .. raw.len - 1])); + raw[raw.len - 1] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decode(raw)); +} + +test "graph metric membership seals reject misplaced blocks and truncated canonical counts" { + const alloc = std.testing.allocator; + const raw = try encodeAlloc(alloc, &.{ .{ .node = "a", .slot = @as(u64, 1) << 32 }, .{ .node = "z", .slot = (@as(u64, 1) << 32) + 1 } }); + defer alloc.free(raw); + _ = try decodeSealed(raw, 0, 0, 2, "", ""); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeSealed(raw, 1, 0, 2, "", "")); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeSealed(raw, 0, 1, 258, "", "")); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeSealed(raw, 0, 0, 3, "", "")); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeSealed(raw, 0, 0, 2, "b", "")); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeSealed(raw, 0, 0, 2, "", "z")); +} diff --git a/zig/pkg/antfly/src/graph/metric_cost.zig b/zig/pkg/antfly/src/graph/metric_cost.zig new file mode 100644 index 0000000000..a948f5f34b --- /dev/null +++ b/zig/pkg/antfly/src/graph/metric_cost.zig @@ -0,0 +1,78 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy at +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Shared graph-metric kernel cost model. Admission and execution must use the +//! same model so a configured work ceiling remains a real isolation boundary. + +const std = @import("std"); + +pub const Kind = enum { + degree, + pagerank, + eigenvector, + hits, +}; + +fn scaled(count: usize, passes: u64) !u64 { + return std.math.mul(u64, @intCast(count), passes) catch + error.GraphMetricBuildBudgetExceeded; +} + +fn add(left: u64, right: u64) !u64 { + return std.math.add(u64, left, right) catch + error.GraphMetricBuildBudgetExceeded; +} + +/// Conservative logical work performed by a kernel, including dense-vector +/// setup. A work item is one visited vertex or edge in a full kernel pass. +pub fn kernelWorkItems(kind: Kind, node_count: usize, edge_count: usize, iterations: u32) !u64 { + const setup_node_passes: u64 = switch (kind) { + .degree => 0, + .pagerank => 2, // reciprocal out-degree and initial rank + .eigenvector => 1, + .hits => 2, + }; + const iteration_node_passes: u64 = switch (kind) { + .degree => 1, + .pagerank => 2, // sink mass, fused adjacency fill + delta + .eigenvector => 3, // adjacency fill, norm, fused scale + delta + .hits => 5, // two fills, two norms, fused paired scale/delta/copy + }; + const iteration_edge_passes: u64 = switch (kind) { + .degree => 0, + .pagerank, .eigenvector => 1, + .hits => 2, + }; + const effective_iterations: u64 = if (kind == .degree) 1 else iterations; + const setup = try scaled(node_count, setup_node_passes); + const nodes_per_iteration = try scaled(node_count, iteration_node_passes); + const edges_per_iteration = try scaled(edge_count, iteration_edge_passes); + const per_iteration = try add(nodes_per_iteration, edges_per_iteration); + return try add(setup, std.math.mul(u64, per_iteration, effective_iterations) catch + return error.GraphMetricBuildBudgetExceeded); +} + +test "kernel work model accounts for algorithm-specific vector and edge passes" { + try std.testing.expectEqual(@as(u64, 10), try kernelWorkItems(.degree, 10, 20, 50)); + try std.testing.expectEqual(@as(u64, 100), try kernelWorkItems(.pagerank, 10, 20, 2)); + try std.testing.expectEqual(@as(u64, 110), try kernelWorkItems(.eigenvector, 10, 20, 2)); + try std.testing.expectEqual(@as(u64, 200), try kernelWorkItems(.hits, 10, 20, 2)); +} + +test "kernel work model rejects overflow" { + try std.testing.expectError( + error.GraphMetricBuildBudgetExceeded, + kernelWorkItems(.hits, std.math.maxInt(usize), std.math.maxInt(usize), 1_000), + ); +} diff --git a/zig/pkg/antfly/src/graph/metric_rerank.zig b/zig/pkg/antfly/src/graph/metric_rerank.zig new file mode 100644 index 0000000000..88ef40a25b --- /dev/null +++ b/zig/pkg/antfly/src/graph/metric_rerank.zig @@ -0,0 +1,154 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +pub const Config = struct { + base_weight: f64, + metric_weight: f64, + missing_score: f64, +}; + +/// A selected candidate borrows its id from the caller's hit array. The +/// caller uses original_index to move only the requested page into its result +/// and allocates explanation strings only for those surviving hits. +pub const Selection = struct { + original_index: usize, + id: []const u8, + base_score: f64, + metric_score: ?f64, + metric_score_used: f64, + final_score: f32, +}; + +fn comesBefore(left: Selection, right: Selection) bool { + if (left.final_score == right.final_score) + return std.mem.lessThan(u8, left.id, right.id); + return left.final_score > right.final_score; +} + +fn worstFirst(_: void, left: Selection, right: Selection) std.math.Order { + if (comesBefore(left, right)) return .gt; + if (comesBefore(right, left)) return .lt; + return .eq; +} + +fn lessThan(_: void, left: Selection, right: Selection) bool { + return comesBefore(left, right); +} + +pub fn clampF64ToF32(value: f64) f32 { + const max = std.math.floatMax(f32); + if (value > max) return max; + if (value < -max) return -max; + return @floatCast(value); +} + +/// Select one reranked page in O(N log(offset + limit)) time and +/// O(offset + limit) memory. `hits` may be any SearchHit-compatible slice with +/// `id` and optional `score` fields, keeping this policy independent of the +/// native and serverless storage implementations. +pub fn selectPageAlloc( + alloc: Allocator, + hits: anytype, + metric_scores: []const ?f64, + config: Config, + offset: u32, + limit: u32, +) ![]Selection { + if (hits.len != metric_scores.len) return error.InvalidGraphMetricScores; + if (!std.math.isFinite(config.base_weight) or + !std.math.isFinite(config.metric_weight) or + !std.math.isFinite(config.missing_score)) + return error.InvalidGraphMetricScores; + if (limit == 0 or hits.len == 0) return try alloc.alloc(Selection, 0); + if (@as(usize, offset) >= hits.len) return try alloc.alloc(Selection, 0); + const requested = std.math.add(usize, offset, limit) catch + return error.InvalidGraphMetricRerankWindow; + const capacity = @min(requested, hits.len); + var selected = std.PriorityQueue(Selection, void, worstFirst).initContext({}); + defer selected.deinit(alloc); + try selected.ensureTotalCapacity(alloc, capacity); + + for (hits, metric_scores, 0..) |hit, metric_score, index| { + const base_score: f64 = if (hit.score) |score| @floatCast(score) else 0; + const metric_score_used = metric_score orelse config.missing_score; + if (!std.math.isFinite(base_score) or !std.math.isFinite(metric_score_used)) + return error.InvalidGraphMetricScores; + const final_score = config.base_weight * base_score + config.metric_weight * metric_score_used; + if (!std.math.isFinite(final_score)) return error.InvalidGraphMetricScores; + const candidate = Selection{ + .original_index = index, + .id = hit.id, + .base_score = base_score, + .metric_score = metric_score, + .metric_score_used = metric_score_used, + .final_score = clampF64ToF32(final_score), + }; + if (selected.count() < capacity) { + try selected.push(alloc, candidate); + } else if (comesBefore(candidate, selected.peek().?)) { + _ = selected.pop(); + try selected.push(alloc, candidate); + } + } + + std.mem.sort(Selection, selected.items, {}, lessThan); + const start = @min(@as(usize, offset), selected.items.len); + const count = @min(@as(usize, limit), selected.items.len - start); + return try alloc.dupe(Selection, selected.items[start .. start + count]); +} + +test "bounded rerank rejects non-finite inputs and skips impossible offsets" { + const Hit = struct { id: []const u8, score: ?f32 }; + const hits = [_]Hit{.{ .id = "a", .score = 1 }}; + const scores = [_]?f64{0.5}; + const empty = try selectPageAlloc(std.testing.allocator, &hits, &scores, .{ + .base_weight = 1, + .metric_weight = 1, + .missing_score = 0, + }, 2, 1); + defer std.testing.allocator.free(empty); + try std.testing.expectEqual(@as(usize, 0), empty.len); + + try std.testing.expectError(error.InvalidGraphMetricScores, selectPageAlloc( + std.testing.allocator, + &hits, + &scores, + .{ .base_weight = std.math.nan(f64), .metric_weight = 1, .missing_score = 0 }, + 0, + 1, + )); +} + +test "bounded rerank selects the requested stable page" { + const Hit = struct { id: []const u8, score: ?f32 }; + const hits = [_]Hit{ + .{ .id = "c", .score = 1 }, + .{ .id = "a", .score = 1 }, + .{ .id = "b", .score = 1 }, + .{ .id = "d", .score = 1 }, + }; + const scores = [_]?f64{ 0.2, 0.5, 0.5, null }; + const selected = try selectPageAlloc(std.testing.allocator, &hits, &scores, .{ + .base_weight = 1, + .metric_weight = 1, + .missing_score = 0, + }, 1, 2); + defer std.testing.allocator.free(selected); + try std.testing.expectEqual(@as(usize, 2), selected.len); + try std.testing.expectEqualStrings("b", selected[0].id); + try std.testing.expectEqualStrings("c", selected[1].id); +} diff --git a/zig/pkg/antfly/src/graph/metrics.zig b/zig/pkg/antfly/src/graph/metrics.zig new file mode 100644 index 0000000000..8e8a2f027e --- /dev/null +++ b/zig/pkg/antfly/src/graph/metrics.zig @@ -0,0 +1,1211 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the Elastic License 2.0 is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See +// the Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Storage-independent, bounded graph metric kernels. Persistence and work +//! scheduling deliberately live outside this module so the same algorithms can +//! be used by embedded and immutable lake-native graph implementations. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const CancellationToken = @import("../common/cancellation.zig").CancellationToken; +const metric_cost = @import("metric_cost.zig"); +pub const warm_start = @import("warm_start.zig"); + +pub const Edge = struct { + // Materializers cap graphs well below u32 addressability. Keeping the + // immutable compute edge at eight bytes halves the hottest topology array + // on 64-bit hosts and gives serverless builds a stable cross-platform wire + // width instead of leaking usize into retained state. + source: u32, + target: u32, +}; + +pub const AdjacencyLane = enum(u2) { + none, + degrees, + neighbors, +}; + +/// Describes the smallest topology a kernel family needs. Keeping this in the +/// storage-independent layer lets materializers plan one union topology for a +/// compatible metric group without paying for adjacency lanes no consumer +/// reads. +pub const TopologyRequirements = struct { + incoming: AdjacencyLane = .none, + outgoing: AdjacencyLane = .none, + + pub const degree = TopologyRequirements{ .incoming = .degrees, .outgoing = .degrees }; + pub const pagerank = TopologyRequirements{ .incoming = .neighbors, .outgoing = .degrees }; + pub const eigenvector = TopologyRequirements{ .incoming = .neighbors }; + pub const hits = TopologyRequirements{ .incoming = .neighbors, .outgoing = .neighbors }; + pub const full = hits; + + pub fn merge(self: TopologyRequirements, other: TopologyRequirements) TopologyRequirements { + return .{ + .incoming = @enumFromInt(@max(@intFromEnum(self.incoming), @intFromEnum(other.incoming))), + .outgoing = @enumFromInt(@max(@intFromEnum(self.outgoing), @intFromEnum(other.outgoing))), + }; + } + + pub fn satisfies(self: TopologyRequirements, required: TopologyRequirements) bool { + return @intFromEnum(self.incoming) >= @intFromEnum(required.incoming) and + @intFromEnum(self.outgoing) >= @intFromEnum(required.outgoing); + } +}; + +/// Compact target-owned and source-owned adjacency. Iterative kernels write +/// one output ordinal at a time, avoiding random scatter writes and providing +/// a race-free partition boundary for runtime-backed parallel execution. +pub const Topology = struct { + node_count: usize, + edge_count: usize, + requirements: TopologyRequirements, + incoming_offsets: []u32, + incoming_sources: []u32, + outgoing_offsets: []u32, + outgoing_targets: []u32, + + pub fn initAlloc(alloc: Allocator, node_count: usize, edges: []const Edge, cancellation: CancellationToken) !Topology { + return initAllocFor(alloc, node_count, edges, .full, cancellation); + } + + pub fn initAllocFor( + alloc: Allocator, + node_count: usize, + edges: []const Edge, + requirements: TopologyRequirements, + cancellation: CancellationToken, + ) !Topology { + return initFromSourceAlloc(alloc, node_count, edges.len, SliceEdges{ .edges = edges }, requirements, cancellation); + } + + const SliceEdges = struct { + edges: []const Edge, + index: usize = 0, + + pub fn next(self: *@This()) ?Edge { + if (self.index == self.edges.len) return null; + defer self.index += 1; + return self.edges[self.index]; + } + }; + + /// Two deterministic passes over a replayable edge source. The source is + /// copied for each pass and must yield the same immutable, ordered edges. + /// Projections can remap ordinals here without retaining an O(E) edge copy. + pub fn initFromSourceAlloc( + alloc: Allocator, + node_count: usize, + edge_count: usize, + source: anytype, + requirements: TopologyRequirements, + cancellation: CancellationToken, + ) !Topology { + try cancellation.check(); + if (node_count > std.math.maxInt(u32) or edge_count > std.math.maxInt(u32)) + return error.GraphMetricBuildBudgetExceeded; + const offset_count = std.math.add(usize, node_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + const has_incoming = requirements.incoming != .none; + const has_outgoing = requirements.outgoing != .none; + const incoming_offsets = if (has_incoming) try alloc.alloc(u32, offset_count) else @constCast(&[_]u32{}); + errdefer if (has_incoming) alloc.free(incoming_offsets); + const outgoing_offsets = if (has_outgoing) try alloc.alloc(u32, offset_count) else @constCast(&[_]u32{}); + errdefer if (has_outgoing) alloc.free(outgoing_offsets); + if (has_incoming) @memset(incoming_offsets, 0); + if (has_outgoing) @memset(outgoing_offsets, 0); + var census = source; + var edge_index: usize = 0; + while (census.next()) |edge| : (edge_index += 1) { + if (edge_index % 4096 == 0) try cancellation.check(); + if (edge_index == edge_count) return error.InvalidGraphMetricEdge; + if (@as(usize, edge.source) >= node_count or @as(usize, edge.target) >= node_count) + return error.InvalidGraphMetricEdge; + if (has_incoming) incoming_offsets[@as(usize, edge.target) + 1] += 1; + if (has_outgoing) outgoing_offsets[@as(usize, edge.source) + 1] += 1; + } + if (edge_index != edge_count) return error.InvalidGraphMetricEdge; + for (1..offset_count) |i| { + if (i % 4096 == 0) try cancellation.check(); + if (has_incoming) incoming_offsets[i] = std.math.add(u32, incoming_offsets[i], incoming_offsets[i - 1]) catch + return error.GraphMetricBuildBudgetExceeded; + if (has_outgoing) outgoing_offsets[i] = std.math.add(u32, outgoing_offsets[i], outgoing_offsets[i - 1]) catch + return error.GraphMetricBuildBudgetExceeded; + } + const owns_incoming_sources = requirements.incoming == .neighbors; + const owns_outgoing_targets = requirements.outgoing == .neighbors; + const incoming_sources = if (owns_incoming_sources) try alloc.alloc(u32, edge_count) else @constCast(&[_]u32{}); + errdefer if (owns_incoming_sources) alloc.free(incoming_sources); + const outgoing_targets = if (owns_outgoing_targets) try alloc.alloc(u32, edge_count) else @constCast(&[_]u32{}); + errdefer if (owns_outgoing_targets) alloc.free(outgoing_targets); + const incoming_cursors = if (owns_incoming_sources) try alloc.dupe(u32, incoming_offsets[0..node_count]) else @constCast(&[_]u32{}); + defer if (owns_incoming_sources) alloc.free(incoming_cursors); + const outgoing_cursors = if (owns_outgoing_targets) try alloc.dupe(u32, outgoing_offsets[0..node_count]) else @constCast(&[_]u32{}); + defer if (owns_outgoing_targets) alloc.free(outgoing_cursors); + if (owns_incoming_sources or owns_outgoing_targets) { + var fill = source; + edge_index = 0; + while (fill.next()) |edge| : (edge_index += 1) { + if (edge_index % 4096 == 0) try cancellation.check(); + if (edge_index == edge_count or edge.source >= node_count or edge.target >= node_count) return error.InvalidGraphMetricEdge; + if (owns_incoming_sources) { + const incoming_position = incoming_cursors[edge.target]; + if (incoming_position >= incoming_offsets[edge.target + 1]) return error.InvalidGraphMetricEdge; + incoming_sources[incoming_position] = edge.source; + incoming_cursors[edge.target] += 1; + } + if (owns_outgoing_targets) { + const outgoing_position = outgoing_cursors[edge.source]; + if (outgoing_position >= outgoing_offsets[edge.source + 1]) return error.InvalidGraphMetricEdge; + outgoing_targets[outgoing_position] = edge.target; + outgoing_cursors[edge.source] += 1; + } + } + if (edge_index != edge_count) return error.InvalidGraphMetricEdge; + } + return .{ + .node_count = node_count, + .edge_count = edge_count, + .requirements = requirements, + .incoming_offsets = incoming_offsets, + .incoming_sources = incoming_sources, + .outgoing_offsets = outgoing_offsets, + .outgoing_targets = outgoing_targets, + }; + } + + pub fn deinit(self: *Topology, alloc: Allocator) void { + if (self.requirements.incoming != .none) alloc.free(self.incoming_offsets); + if (self.requirements.incoming == .neighbors) alloc.free(self.incoming_sources); + if (self.requirements.outgoing != .none) alloc.free(self.outgoing_offsets); + if (self.requirements.outgoing == .neighbors) alloc.free(self.outgoing_targets); + self.* = undefined; + } + + pub fn nodeCount(self: Topology) usize { + return self.node_count; + } + + pub fn edgeCount(self: Topology) usize { + return self.edge_count; + } +}; + +pub const Options = struct { + damping: f64 = 0.85, + tolerance: f64 = 0.000001, + max_iterations: u32 = 50, + max_nodes: usize = 1_000_000, + max_edges: usize = 10_000_000, + max_work_items: u64 = 500_000_000, + cancellation: CancellationToken = .none, + io: ?std.Io = null, + max_parallelism: usize = 1, + /// PageRank-only ordinal-aligned seed from a compatible publication. + /// Spectral kernels reject supplied seeds: normalization alone cannot + /// guarantee support on a newly dominant disconnected component. + initial_scores: ?[]const f64 = null, + initial_authorities: ?[]const f64 = null, + initial_hubs: ?[]const f64 = null, +}; + +const parallel_edge_threshold: usize = 128 * 1024; +const parallel_vector_threshold: usize = 32 * 1024; +const max_kernel_parallelism: usize = 16; +const reduction_partitions: usize = 16; + +fn parallelWidth(topology: Topology, options: Options) usize { + if (options.io == null or options.max_parallelism < 2 or + topology.edgeCount() < parallel_edge_threshold or topology.nodeCount() == 0) + { + return 1; + } + return @min(max_kernel_parallelism, options.max_parallelism); +} + +fn vectorParallelWidth(len: usize, options: Options) usize { + if (options.io == null or options.max_parallelism < 2 or len < parallel_vector_threshold) + return 1; + return @min(max_kernel_parallelism, @min(options.max_parallelism, len)); +} + +fn vectorBoundary(len: usize, part: usize, parts: usize) usize { + const whole = len / parts; + const remainder = len % parts; + return whole * part + @min(part, remainder); +} + +fn logicalReductionParts(len: usize) usize { + if (len < parallel_vector_threshold) return 1; + return @min(reduction_partitions, len); +} + +fn graphReductionParts(topology: Topology) usize { + // Stable across runtime worker counts, but sensitive to edge work. + return if (topology.edgeCount() >= parallel_edge_threshold) + reduction_partitions + else + logicalReductionParts(topology.nodeCount()); +} + +/// Locate a work coordinate in CSR's interleaved vertex/edge stream. Unlike +/// vertex-only boundaries, a coordinate may fall inside a high-degree row. +fn workOrdinal(offsets: []const u32, coordinate: usize) usize { + const node_count = offsets.len - 1; + var lower: usize = 0; + var upper: usize = node_count; + while (lower < upper) { + const middle = lower + (upper - lower) / 2; + const work = middle + @as(usize, offsets[middle]); + if (work <= coordinate) lower = middle + 1 else upper = middle; + } + return lower -| 1; +} + +pub const Result = struct { + scores: []f64, + iterations_completed: u32, + converged: bool, + delta: f64, + + pub fn deinit(self: *Result, alloc: Allocator) void { + alloc.free(self.scores); + self.* = undefined; + } +}; + +pub const HitsResult = struct { + authorities: []f64, + hubs: []f64, + iterations_completed: u32, + converged: bool, + delta: f64, + + pub fn deinit(self: *HitsResult, alloc: Allocator) void { + alloc.free(self.authorities); + alloc.free(self.hubs); + self.* = undefined; + } +}; + +fn validateInputBoundsAndOptions(node_count: usize, edge_count: usize, options: Options) !void { + if (node_count > options.max_nodes or edge_count > options.max_edges) return error.GraphMetricBuildBudgetExceeded; + if (!std.math.isFinite(options.damping) or options.damping < 0 or options.damping >= 1 or + !std.math.isFinite(options.tolerance) or options.tolerance < 0 or + options.max_iterations == 0 or options.max_iterations > 1_000 or + options.max_nodes == 0 or options.max_edges == 0 or options.max_work_items == 0 or + options.max_parallelism == 0 or options.max_parallelism > max_kernel_parallelism) + { + return error.InvalidGraphMetricOptions; + } +} + +fn admitWork(kind: metric_cost.Kind, node_count: usize, edge_count: usize, iterations: u32, max_work_items: u64) !void { + const work = try metric_cost.kernelWorkItems(kind, node_count, edge_count, iterations); + if (work > max_work_items) return error.GraphMetricBuildBudgetExceeded; +} + +pub fn degreeAlloc(alloc: Allocator, node_count: usize, edges: []const Edge, options: Options) !Result { + try validateInputBoundsAndOptions(node_count, edges.len, options); + try admitWork(.degree, node_count, edges.len, 1, options.max_work_items); + var topology = try Topology.initAllocFor(alloc, node_count, edges, .degree, options.cancellation); + defer topology.deinit(alloc); + return try degreeTopologyAlloc(alloc, topology, options); +} + +fn validateTopology(topology: Topology, required: TopologyRequirements, options: Options) !void { + const node_count = topology.nodeCount(); + const edge_count = topology.edgeCount(); + if (node_count > options.max_nodes or edge_count > options.max_edges) return error.GraphMetricBuildBudgetExceeded; + if (!topology.requirements.satisfies(required)) return error.InvalidGraphMetricEdge; + if (!std.math.isFinite(options.damping) or options.damping < 0 or options.damping >= 1 or + !std.math.isFinite(options.tolerance) or options.tolerance < 0 or + options.max_iterations == 0 or options.max_iterations > 1_000 or + options.max_nodes == 0 or options.max_edges == 0 or options.max_work_items == 0 or + options.max_parallelism == 0 or options.max_parallelism > max_kernel_parallelism) + { + return error.InvalidGraphMetricOptions; + } + const expected_offsets = node_count + 1; + if (topology.requirements.incoming != .none and + (topology.incoming_offsets.len != expected_offsets or topology.incoming_offsets[0] != 0 or + @as(usize, topology.incoming_offsets[node_count]) != edge_count)) return error.InvalidGraphMetricEdge; + if (topology.requirements.outgoing != .none and + (topology.outgoing_offsets.len != expected_offsets or topology.outgoing_offsets[0] != 0 or + @as(usize, topology.outgoing_offsets[node_count]) != edge_count)) return error.InvalidGraphMetricEdge; + if (topology.requirements.incoming == .neighbors and topology.incoming_sources.len != edge_count) + return error.InvalidGraphMetricEdge; + if (topology.requirements.outgoing == .neighbors and topology.outgoing_targets.len != edge_count) + return error.InvalidGraphMetricEdge; + for (0..node_count) |i| { + if ((topology.requirements.incoming != .none and topology.incoming_offsets[i] > topology.incoming_offsets[i + 1]) or + (topology.requirements.outgoing != .none and topology.outgoing_offsets[i] > topology.outgoing_offsets[i + 1])) + { + return error.InvalidGraphMetricEdge; + } + } + // Endpoint ordinals were validated while the immutable topology was + // constructed. Revalidating O(E) neighbors for every metric sharing the + // same projection defeats topology reuse and is not a useful trust boundary. +} + +pub fn degreeTopologyAlloc(alloc: Allocator, topology: Topology, options: Options) !Result { + try validateTopology(topology, .degree, options); + try admitWork(.degree, topology.nodeCount(), topology.edgeCount(), 1, options.max_work_items); + const node_count = topology.nodeCount(); + const scores = try alloc.alloc(f64, node_count); + errdefer alloc.free(scores); + for (scores, 0..) |*score, i| { + if (i % 4096 == 0) try options.cancellation.check(); + const incoming = topology.incoming_offsets[i + 1] - topology.incoming_offsets[i]; + const outgoing = topology.outgoing_offsets[i + 1] - topology.outgoing_offsets[i]; + score.* = @floatFromInt(@as(u64, incoming) + @as(u64, outgoing)); + } + return .{ .scores = scores, .iterations_completed = 1, .converged = true, .delta = 0 }; +} + +fn fillPageRankNext( + topology: Topology, + scores: []const f64, + source_scale: []const f64, + next: []f64, + base: f64, + options: Options, +) !f64 { + return fillTiledAdjacency(topology, scores, next, true, source_scale, 1, base, options); +} + +fn fillAdjacencySums( + topology: Topology, + input: []const f64, + output: []f64, + incoming: bool, + input_divisor: f64, + options: Options, +) !void { + _ = try fillTiledAdjacency(topology, input, output, incoming, null, if (input_divisor > 0) 1.0 / input_divisor else 1.0, 0, options); +} + +/// Fixed logical edge tiles, independent of executor width. Interior vertices +/// retain exclusive output ownership. Only the two boundary rows of each tile +/// need partial sums: at most 32 records, on the stack, for any graph size. +/// This bounds worker work by ceil((N + E) / 16), even for a single giant hub. +fn fillTiledAdjacency( + topology: Topology, + input: []const f64, + output: []f64, + incoming: bool, + source_scale: ?[]const f64, + input_scale: f64, + base: f64, + options: Options, +) !f64 { + const Boundary = struct { ordinal: usize, sum: f64 }; + const Partial = struct { + boundaries: [2]Boundary = undefined, + count: usize = 0, + delta: f64 = 0, + }; + const Worker = struct { + fn run( + offsets: []const u32, + neighbors: []const u32, + current: []const f64, + scale: ?[]const f64, + scalar: f64, + result: []f64, + base_score: f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]Partial, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + const total = offsets.len - 1 + neighbors.len; + const start = vectorBoundary(total, part, parts); + const end = vectorBoundary(total, part + 1, parts); + if (start == end) continue; + var ordinal = workOrdinal(offsets, start); + var visited: usize = 0; + while (ordinal < offsets.len - 1) : (ordinal += 1) { + const row_start = ordinal + @as(usize, offsets[ordinal]); + if (row_start >= end) break; + if (visited % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + visited += 1; + const row_end = ordinal + 1 + @as(usize, offsets[ordinal + 1]); + const complete = row_start >= start and row_end <= end; + const edge_start = offsets[ordinal] + (@max(start, row_start + 1) - (row_start + 1)); + const edge_end = offsets[ordinal] + (@min(end, row_end) - (row_start + 1)); + var value: f64 = if (complete) base_score else 0; + for (neighbors[edge_start..edge_end], 0..) |source, edge_index| { + if (edge_index % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + value += current[source] * (if (scale) |scales| scales[source] else scalar); + } + if (complete) { + result[ordinal] = value; + if (scale != null) partials[part].delta += @abs(value - current[ordinal]); + } else { + const partial = &partials[part]; + partial.boundaries[partial.count] = .{ .ordinal = ordinal, .sum = value }; + partial.count += 1; + } + } + } + } + }; + const parts = graphReductionParts(topology); + var partials: [reduction_partitions]Partial = @splat(.{}); + const width = @min(parallelWidth(topology, options), parts); + const offsets = if (incoming) topology.incoming_offsets else topology.outgoing_offsets; + const neighbors = if (incoming) topology.incoming_sources else topology.outgoing_targets; + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(offsets, neighbors, input, source_scale, input_scale, output, base, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + offsets, neighbors, input, source_scale, input_scale, output, base, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var delta: f64 = 0; + var boundary_ordinal: ?usize = null; + var boundary_sum: f64 = base; + for (partials[0..parts]) |partial| { + delta += partial.delta; + for (partial.boundaries[0..partial.count]) |boundary| { + if (boundary_ordinal) |ordinal| { + if (ordinal != boundary.ordinal) { + output[ordinal] = boundary_sum; + if (source_scale != null) delta += @abs(boundary_sum - input[ordinal]); + boundary_sum = base; + } + } + boundary_ordinal = boundary.ordinal; + boundary_sum += boundary.sum; + } + } + if (boundary_ordinal) |ordinal| { + output[ordinal] = boundary_sum; + if (source_scale != null) delta += @abs(boundary_sum - input[ordinal]); + } + return delta; +} + +pub fn pageRankAlloc(alloc: Allocator, node_count: usize, edges: []const Edge, options: Options) !Result { + try validateInputBoundsAndOptions(node_count, edges.len, options); + try admitWork(.pagerank, node_count, edges.len, options.max_iterations, options.max_work_items); + var topology = try Topology.initAllocFor(alloc, node_count, edges, .pagerank, options.cancellation); + defer topology.deinit(alloc); + return try pageRankTopologyAlloc(alloc, topology, options); +} + +pub fn pageRankTopologyAlloc(alloc: Allocator, topology: Topology, options: Options) !Result { + try validateTopology(topology, .pagerank, options); + try admitWork(.pagerank, topology.nodeCount(), topology.edgeCount(), options.max_iterations, options.max_work_items); + const node_count = topology.nodeCount(); + var scores = try alloc.alloc(f64, node_count); + errdefer alloc.free(scores); + if (node_count == 0) return .{ .scores = scores, .iterations_completed = 0, .converged = true, .delta = 0 }; + var next = try alloc.alloc(f64, node_count); + defer alloc.free(next); + // Reuse one dense vector for the damped reciprocal out-degree. This moves + // division out of the O(E * iterations) edge loop without increasing peak + // memory over the previous degree vector. + const source_scale = try alloc.alloc(f64, node_count); + defer alloc.free(source_scale); + for (source_scale, 0..) |*scale, i| { + if (i % 4096 == 0) try options.cancellation.check(); + const out_degree = topology.outgoing_offsets[i + 1] - topology.outgoing_offsets[i]; + scale.* = if (out_degree == 0) 0 else options.damping / @as(f64, @floatFromInt(out_degree)); + } + const count: f64 = @floatFromInt(node_count); + if (options.initial_scores) |initial| { + try initializeProbabilityVector(scores, initial, options.cancellation); + } else { + @memset(scores, 1.0 / count); + } + + var iteration: u32 = 0; + var delta: f64 = 0; + while (iteration < options.max_iterations) { + try options.cancellation.check(); + iteration += 1; + const sink_mass = try pageRankSinkMass(scores, source_scale, options); + const base = (1.0 - options.damping + options.damping * sink_mass) / count; + delta = try fillPageRankNext(topology, scores, source_scale, next, base, options); + const previous = scores; + scores = next; + next = previous; + if (!std.math.isFinite(delta)) return error.InvalidGraphMetricScore; + if (delta <= options.tolerance) return .{ .scores = scores, .iterations_completed = iteration, .converged = true, .delta = delta }; + } + return .{ .scores = scores, .iterations_completed = iteration, .converged = false, .delta = delta }; +} + +pub fn eigenvectorAlloc(alloc: Allocator, node_count: usize, edges: []const Edge, options: Options) !Result { + try validateInputBoundsAndOptions(node_count, edges.len, options); + try admitWork(.eigenvector, node_count, edges.len, options.max_iterations, options.max_work_items); + var topology = try Topology.initAllocFor(alloc, node_count, edges, .eigenvector, options.cancellation); + defer topology.deinit(alloc); + return try eigenvectorTopologyAlloc(alloc, topology, options); +} + +pub fn eigenvectorTopologyAlloc(alloc: Allocator, topology: Topology, options: Options) !Result { + if (options.initial_scores != null) return error.InvalidGraphMetricWarmStart; + try validateTopology(topology, .eigenvector, options); + try admitWork(.eigenvector, topology.nodeCount(), topology.edgeCount(), options.max_iterations, options.max_work_items); + const node_count = topology.nodeCount(); + var scores = try alloc.alloc(f64, node_count); + errdefer alloc.free(scores); + if (node_count == 0) return .{ .scores = scores, .iterations_completed = 0, .converged = true, .delta = 0 }; + var next = try alloc.alloc(f64, node_count); + defer alloc.free(next); + @memset(scores, 1.0 / @sqrt(@as(f64, @floatFromInt(node_count)))); + var iteration: u32 = 0; + var delta: f64 = 0; + while (iteration < options.max_iterations) { + try options.cancellation.check(); + iteration += 1; + try fillAdjacencySums(topology, scores, next, true, 1, options); + delta = try normalizeSwapAndDelta(&scores, &next, options); + if (!std.math.isFinite(delta)) return error.InvalidGraphMetricScore; + if (delta <= options.tolerance) return .{ .scores = scores, .iterations_completed = iteration, .converged = true, .delta = delta }; + } + return .{ .scores = scores, .iterations_completed = iteration, .converged = false, .delta = delta }; +} + +pub fn hitsAlloc(alloc: Allocator, node_count: usize, edges: []const Edge, options: Options) !HitsResult { + try validateInputBoundsAndOptions(node_count, edges.len, options); + try admitWork(.hits, node_count, edges.len, options.max_iterations, options.max_work_items); + var topology = try Topology.initAllocFor(alloc, node_count, edges, .hits, options.cancellation); + defer topology.deinit(alloc); + return try hitsTopologyAlloc(alloc, topology, options); +} + +pub fn hitsTopologyAlloc(alloc: Allocator, topology: Topology, options: Options) !HitsResult { + if (options.initial_authorities != null or options.initial_hubs != null) return error.InvalidGraphMetricWarmStart; + try validateTopology(topology, .hits, options); + try admitWork(.hits, topology.nodeCount(), topology.edgeCount(), options.max_iterations, options.max_work_items); + const node_count = topology.nodeCount(); + const authorities = try alloc.alloc(f64, node_count); + errdefer alloc.free(authorities); + const hubs = try alloc.alloc(f64, node_count); + errdefer alloc.free(hubs); + if (node_count == 0) return .{ .authorities = authorities, .hubs = hubs, .iterations_completed = 0, .converged = true, .delta = 0 }; + const next_authorities = try alloc.alloc(f64, node_count); + defer alloc.free(next_authorities); + const next_hubs = try alloc.alloc(f64, node_count); + defer alloc.free(next_hubs); + const initial = 1.0 / @sqrt(@as(f64, @floatFromInt(node_count))); + @memset(authorities, initial); + @memset(hubs, initial); + var iteration: u32 = 0; + var delta: f64 = 0; + while (iteration < options.max_iterations) { + try options.cancellation.check(); + iteration += 1; + try fillAdjacencySums(topology, hubs, next_authorities, true, 1, options); + const authority_norm = @sqrt(try normSquared(next_authorities, options)); + try fillAdjacencySums(topology, next_authorities, next_hubs, false, authority_norm, options); + const hub_norm = @sqrt(try normSquared(next_hubs, options)); + delta = try replaceHitsAndDelta(authorities, hubs, next_authorities, next_hubs, authority_norm, hub_norm, options); + if (!std.math.isFinite(delta)) return error.InvalidGraphMetricScore; + if (delta <= options.tolerance) return .{ .authorities = authorities, .hubs = hubs, .iterations_completed = iteration, .converged = true, .delta = delta }; + } + return .{ .authorities = authorities, .hubs = hubs, .iterations_completed = iteration, .converged = false, .delta = delta }; +} + +fn initializeProbabilityVector(destination: []f64, seed: []const f64, cancellation: CancellationToken) !void { + if (seed.len != destination.len or seed.len == 0) return error.InvalidGraphMetricWarmStart; + var mass = warm_start.Mass{}; + for (seed, 0..) |value, i| { + if (i % 4096 == 0) try cancellation.check(); + try mass.add(value); + } + const total = try mass.total(); + for (seed, destination, 0..) |value, *out, i| { + if (i % 4096 == 0) try cancellation.check(); + out.* = try warm_start.normalized(value, total, destination.len); + } +} + +fn pageRankSinkMass(scores: []const f64, source_scale: []const f64, options: Options) !f64 { + const Worker = struct { + fn run( + values: []const f64, + scales: []const f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]f64, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + var sum: f64 = 0; + const start = vectorBoundary(values.len, part, parts); + const end = vectorBoundary(values.len, part + 1, parts); + for (values[start..end], scales[start..end], 0..) |value, scale, i| { + if (i % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + if (scale == 0) sum += value; + } + partials[part] = sum; + } + } + }; + if (scores.len != source_scale.len) return error.InvalidGraphMetricScore; + const parts = logicalReductionParts(scores.len); + var partials: [reduction_partitions]f64 = @splat(0); + const width = @min(vectorParallelWidth(scores.len, options), parts); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(scores, source_scale, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + scores, source_scale, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var total: f64 = 0; + for (partials[0..parts]) |partial| total += partial; + return total; +} + +fn normSquared(values: []const f64, options: Options) !f64 { + const Worker = struct { + fn run( + input: []const f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]f64, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + var sum: f64 = 0; + const start = vectorBoundary(input.len, part, parts); + const end = vectorBoundary(input.len, part + 1, parts); + for (input[start..end], 0..) |value, i| { + if (i % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + sum += value * value; + } + partials[part] = sum; + } + } + }; + const parts = logicalReductionParts(values.len); + var partials: [reduction_partitions]f64 = @splat(0); + const width = @min(vectorParallelWidth(values.len, options), parts); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(values, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + values, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var total: f64 = 0; + for (partials[0..parts]) |partial| total += partial; + return total; +} + +fn scaleValues(values: []f64, denominator: f64, options: Options) !void { + const Worker = struct { + fn run( + output: []f64, + divisor: f64, + start: usize, + end: usize, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + for (output[start..end], 0..) |*value, i| { + if (i % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + value.* /= divisor; + } + } + }; + const width = vectorParallelWidth(values.len, options); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(values, denominator, 0, values.len, options.cancellation, &failure); + if (failure) |err| return err; + return; + } + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |part| group.async(io, Worker.run, .{ + values, + denominator, + vectorBoundary(values.len, part, width), + vectorBoundary(values.len, part + 1, width), + options.cancellation, + &failures[part], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; +} + +fn normalize(values: []f64, options: Options) !void { + const norm_sq = try normSquared(values, options); + const norm = @sqrt(norm_sq); + if (norm > 0) try scaleValues(values, norm, options); +} + +/// Normalize the newly computed vector and calculate convergence in the same +/// cache pass. Iterative eigenvector builds previously streamed the full +/// vector once to normalize and again to compare it with the prior vector. +fn normalizeSwapAndDelta(current: *[]f64, next: *[]f64, options: Options) !f64 { + if (current.*.len != next.*.len) return error.InvalidGraphMetricScore; + const norm = @sqrt(try normSquared(next.*, options)); + const Worker = struct { + fn run( + old_values: []const f64, + new_values: []f64, + divisor: f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]f64, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + var sum: f64 = 0; + const start = vectorBoundary(old_values.len, part, parts); + const end = vectorBoundary(old_values.len, part + 1, parts); + for (start..end) |i| { + if ((i - start) % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + if (divisor > 0) new_values[i] /= divisor; + sum += @abs(new_values[i] - old_values[i]); + } + partials[part] = sum; + } + } + }; + const parts = logicalReductionParts(current.*.len); + var partials: [reduction_partitions]f64 = @splat(0); + const width = @min(vectorParallelWidth(current.*.len, options), parts); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(current.*, next.*, norm, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + current.*, next.*, norm, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var delta: f64 = 0; + for (partials[0..parts]) |partial| delta += partial; + const previous = current.*; + current.* = next.*; + next.* = previous; + return delta; +} + +fn swapAndDelta(current: *[]f64, next: *[]f64, options: Options) !f64 { + const Worker = struct { + fn run( + old_values: []const f64, + new_values: []const f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]f64, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + var sum: f64 = 0; + const start = vectorBoundary(old_values.len, part, parts); + const end = vectorBoundary(old_values.len, part + 1, parts); + for (old_values[start..end], new_values[start..end], 0..) |old, new, i| { + if (i % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + sum += @abs(new - old); + } + partials[part] = sum; + } + } + }; + if (current.*.len != next.*.len) return error.InvalidGraphMetricScore; + const parts = logicalReductionParts(current.*.len); + var partials: [reduction_partitions]f64 = @splat(0); + const width = @min(vectorParallelWidth(current.*.len, options), parts); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(current.*, next.*, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + current.*, next.*, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var delta: f64 = 0; + for (partials[0..parts]) |partial| delta += partial; + const previous = current.*; + current.* = next.*; + next.* = previous; + return delta; +} + +fn replaceHitsAndDelta( + authorities: []f64, + hubs: []f64, + next_authorities: []const f64, + next_hubs: []const f64, + authority_norm: f64, + hub_norm: f64, + options: Options, +) !f64 { + const Worker = struct { + fn run( + authority_values: []f64, + hub_values: []f64, + new_authorities: []const f64, + new_hubs: []const f64, + authority_divisor: f64, + hub_divisor: f64, + parts: usize, + worker: usize, + width: usize, + partials: *[reduction_partitions]f64, + cancellation: CancellationToken, + failure: *?anyerror, + ) void { + var part = worker; + while (part < parts) : (part += width) { + var sum: f64 = 0; + const start = vectorBoundary(authority_values.len, part, parts); + const end = vectorBoundary(authority_values.len, part + 1, parts); + for (start..end) |i| { + if ((i - start) % 4096 == 0) cancellation.check() catch |err| { + failure.* = err; + return; + }; + const new_authority = if (authority_divisor > 0) new_authorities[i] / authority_divisor else new_authorities[i]; + const new_hub = if (hub_divisor > 0) new_hubs[i] / hub_divisor else new_hubs[i]; + sum += @abs(new_authority - authority_values[i]); + sum += @abs(new_hub - hub_values[i]); + authority_values[i] = new_authority; + hub_values[i] = new_hub; + } + partials[part] = sum; + } + } + }; + if (authorities.len != hubs.len or authorities.len != next_authorities.len or authorities.len != next_hubs.len) + return error.InvalidGraphMetricScore; + const parts = logicalReductionParts(authorities.len); + var partials: [reduction_partitions]f64 = @splat(0); + const width = @min(vectorParallelWidth(authorities.len, options), parts); + if (width == 1) { + var failure: ?anyerror = null; + Worker.run(authorities, hubs, next_authorities, next_hubs, authority_norm, hub_norm, parts, 0, 1, &partials, options.cancellation, &failure); + if (failure) |err| return err; + } else { + const io = options.io.?; + var failures: [max_kernel_parallelism]?anyerror = @splat(null); + var group: std.Io.Group = .init; + for (0..width) |worker| group.async(io, Worker.run, .{ + authorities, hubs, next_authorities, next_hubs, authority_norm, hub_norm, parts, worker, width, &partials, options.cancellation, &failures[worker], + }); + try group.await(io); + for (failures[0..width]) |failure| if (failure) |err| return err; + } + var delta: f64 = 0; + for (partials[0..parts]) |partial| delta += partial; + return delta; +} + +test "serverless bounded graph metric kernels compute all supported metrics" { + const edges = [_]Edge{ .{ .source = 0, .target = 1 }, .{ .source = 2, .target = 1 }, .{ .source = 1, .target = 0 } }; + var degree = try degreeAlloc(std.testing.allocator, 3, &edges, .{}); + defer degree.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(f64, 3), degree.scores[1]); + var pagerank = try pageRankAlloc(std.testing.allocator, 3, &edges, .{}); + defer pagerank.deinit(std.testing.allocator); + try std.testing.expect(pagerank.scores[1] > pagerank.scores[2]); + var eigenvector = try eigenvectorAlloc(std.testing.allocator, 3, &edges, .{}); + defer eigenvector.deinit(std.testing.allocator); + try std.testing.expect(eigenvector.iterations_completed > 0); + var hits = try hitsAlloc(std.testing.allocator, 3, &edges, .{}); + defer hits.deinit(std.testing.allocator); + try std.testing.expect(hits.authorities[1] > hits.authorities[2]); +} + +test "serverless graph metric kernels reject unbounded work before allocating" { + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, pageRankAlloc(std.testing.allocator, 2, &.{}, .{ .max_nodes = 1 })); + try std.testing.expectError(error.InvalidGraphMetricEdge, degreeAlloc(std.testing.allocator, 1, &.{.{ .source = 0, .target = 1 }}, .{})); +} + +test "serverless graph metric replayed CSR preserves exact adjacency order and unwinds allocation failures" { + const Source = struct { + index: usize = 0, + pub fn next(self: *@This()) ?Edge { + const edges = [_]Edge{ + .{ .source = 2, .target = 1 }, .{ .source = 1, .target = 1 }, + .{ .source = 0, .target = 2 }, .{ .source = 2, .target = 1 }, + }; + if (self.index == edges.len) return null; + defer self.index += 1; + return edges[self.index]; + } + fn run(alloc: Allocator) !void { + for ([_]TopologyRequirements{ .degree, .pagerank, .eigenvector, .hits }) |requirements| { + var topology = try Topology.initFromSourceAlloc(alloc, 3, 4, @This(){}, requirements, .none); + defer topology.deinit(alloc); + if (requirements.incoming != .none) try std.testing.expectEqualSlices(u32, &.{ 0, 0, 3, 4 }, topology.incoming_offsets); + if (requirements.incoming == .neighbors) try std.testing.expectEqualSlices(u32, &.{ 2, 1, 2, 0 }, topology.incoming_sources); + if (requirements.outgoing != .none) try std.testing.expectEqualSlices(u32, &.{ 0, 1, 2, 4 }, topology.outgoing_offsets); + if (requirements.outgoing == .neighbors) try std.testing.expectEqualSlices(u32, &.{ 2, 1, 1, 1 }, topology.outgoing_targets); + } + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Source.run, .{}); + try std.testing.expectError(error.InvalidGraphMetricEdge, Topology.initFromSourceAlloc(std.testing.allocator, 3, 3, Source{}, .hits, .none)); + try std.testing.expectError(error.InvalidGraphMetricEdge, Topology.initFromSourceAlloc(std.testing.allocator, 3, 5, Source{}, .hits, .none)); +} + +test "serverless graph metric kernels normalize compatible warm starts and reject malformed seeds" { + const alloc = std.testing.allocator; + const edges = [_]Edge{ + .{ .source = 0, .target = 1 }, + .{ .source = 1, .target = 2 }, + .{ .source = 2, .target = 0 }, + .{ .source = 2, .target = 1 }, + }; + var topology = try Topology.initAlloc(alloc, 3, &edges, .none); + defer topology.deinit(alloc); + + const cold_options = Options{ .tolerance = 1e-12, .max_iterations = 100 }; + var cold = try pageRankTopologyAlloc(alloc, topology, cold_options); + defer cold.deinit(alloc); + var warm_options = cold_options; + warm_options.initial_scores = cold.scores; + var warm = try pageRankTopologyAlloc(alloc, topology, warm_options); + defer warm.deinit(alloc); + try std.testing.expect(warm.iterations_completed <= cold.iterations_completed); + try std.testing.expect(warm.delta <= cold_options.tolerance); + + const scaled_seed = [_]f64{ 2, 4, 4 }; + try std.testing.expectError(error.InvalidGraphMetricWarmStart, eigenvectorTopologyAlloc(alloc, topology, .{ + .max_iterations = 1, + .initial_scores = &scaled_seed, + })); + + try std.testing.expectError(error.InvalidGraphMetricWarmStart, pageRankTopologyAlloc(alloc, topology, .{ + .initial_scores = &.{ 1, 2 }, + })); + try std.testing.expectError(error.InvalidGraphMetricWarmStart, hitsTopologyAlloc(alloc, topology, .{ + .initial_authorities = &.{ 0, 0, 0 }, + })); +} + +test "graph metric topology materializes only requested adjacency lanes" { + const edges = [_]Edge{ .{ .source = 0, .target = 1 }, .{ .source = 1, .target = 0 } }; + var degree_topology = try Topology.initAllocFor(std.testing.allocator, 2, &edges, .degree, .none); + defer degree_topology.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 3), degree_topology.incoming_offsets.len); + try std.testing.expectEqual(@as(usize, 0), degree_topology.incoming_sources.len); + try std.testing.expectEqual(@as(usize, 3), degree_topology.outgoing_offsets.len); + try std.testing.expectEqual(@as(usize, 0), degree_topology.outgoing_targets.len); + try std.testing.expectError(error.InvalidGraphMetricEdge, pageRankTopologyAlloc(std.testing.allocator, degree_topology, .{})); + + var eigenvector_topology = try Topology.initAllocFor(std.testing.allocator, 2, &edges, .eigenvector, .none); + defer eigenvector_topology.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, edges.len), eigenvector_topology.incoming_sources.len); + try std.testing.expectEqual(@as(usize, 0), eigenvector_topology.outgoing_offsets.len); + var eigenvector = try eigenvectorTopologyAlloc(std.testing.allocator, eigenvector_topology, .{}); + defer eigenvector.deinit(std.testing.allocator); +} + +test "serverless graph metric spectral rebuild rejects support-deficient warm starts" { + const alloc = std.testing.allocator; + const edges = [_]Edge{ + .{ .source = 0, .target = 1 }, .{ .source = 1, .target = 0 }, + .{ .source = 2, .target = 3 }, .{ .source = 2, .target = 4 }, + .{ .source = 3, .target = 2 }, .{ .source = 3, .target = 4 }, + .{ .source = 4, .target = 2 }, .{ .source = 4, .target = 3 }, + }; + const seed = [_]f64{ 0.7071067811865476, 0.7071067811865476, 0, 0, 0 }; + try std.testing.expectError(error.InvalidGraphMetricWarmStart, eigenvectorAlloc(alloc, 5, &edges, .{ .initial_scores = &seed })); + try std.testing.expectError(error.InvalidGraphMetricWarmStart, hitsAlloc(alloc, 5, &edges, .{ .initial_authorities = &seed, .initial_hubs = &seed })); + var eigen = try eigenvectorAlloc(alloc, 5, &edges, .{}); + defer eigen.deinit(alloc); + var hits = try hitsAlloc(alloc, 5, &edges, .{}); + defer hits.deinit(alloc); + try std.testing.expect(eigen.converged and hits.converged); + try std.testing.expect(eigen.scores[2] > 0.57 and hits.authorities[2] > 0.57 and hits.hubs[2] > 0.57); +} + +test "serverless graph metric runtime fanout preserves deterministic target-owned results" { + const alloc = std.testing.allocator; + for ([_]usize{ 4096, parallel_vector_threshold }) |node_count| { + const edge_count: usize = parallel_edge_threshold; + const edges = try alloc.alloc(Edge, edge_count); + defer alloc.free(edges); + for (edges, 0..) |*edge, i| { + edge.* = .{ + .source = @intCast(i % node_count), + .target = @intCast((i * 17 + 3) % node_count), + }; + } + var topology = try Topology.initAlloc(alloc, node_count, edges, .none); + defer topology.deinit(alloc); + try std.testing.expectEqual(@as(usize, reduction_partitions), graphReductionParts(topology)); + const options = Options{ .max_iterations = 3, .max_work_items = 10_000_000 }; + var serial = try pageRankTopologyAlloc(alloc, topology, options); + defer serial.deinit(alloc); + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + var parallel_options = options; + parallel_options.io = io_impl.io(); + parallel_options.max_parallelism = 4; + var parallel = try pageRankTopologyAlloc(alloc, topology, parallel_options); + defer parallel.deinit(alloc); + try std.testing.expectEqualSlices(f64, serial.scores, parallel.scores); + + const values = try alloc.alloc(f64, node_count); + defer alloc.free(values); + const serial_incoming = try alloc.alloc(f64, node_count); + defer alloc.free(serial_incoming); + const parallel_incoming = try alloc.alloc(f64, node_count); + defer alloc.free(parallel_incoming); + const serial_outgoing = try alloc.alloc(f64, node_count); + defer alloc.free(serial_outgoing); + const parallel_outgoing = try alloc.alloc(f64, node_count); + defer alloc.free(parallel_outgoing); + for (values, 0..) |*value, i| value.* = @floatFromInt(i % 31); + try fillAdjacencySums(topology, values, serial_incoming, true, 1, options); + try fillAdjacencySums(topology, values, parallel_incoming, true, 1, parallel_options); + try std.testing.expectEqualSlices(f64, serial_incoming, parallel_incoming); + try fillAdjacencySums(topology, values, serial_outgoing, false, 1, options); + try fillAdjacencySums(topology, values, parallel_outgoing, false, 1, parallel_options); + try std.testing.expectEqualSlices(f64, serial_outgoing, parallel_outgoing); + + try std.testing.expectEqual( + try normSquared(values, options), + try normSquared(values, parallel_options), + ); + const serial_normalized = try alloc.dupe(f64, values); + defer alloc.free(serial_normalized); + const parallel_normalized = try alloc.dupe(f64, values); + defer alloc.free(parallel_normalized); + try normalize(serial_normalized, options); + try normalize(parallel_normalized, parallel_options); + try std.testing.expectEqualSlices(f64, serial_normalized, parallel_normalized); + } +} + +test "serverless graph metric edge tiles split hubs with deterministic bounded reductions" { + const alloc = std.testing.allocator; + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + for ([_]usize{ 2, parallel_edge_threshold + 1 }) |n| { + const edges = try alloc.alloc(Edge, parallel_edge_threshold); + defer alloc.free(edges); + for (edges, 0..) |*edge, i| edge.* = .{ .source = @intCast(i % (n - 1)), .target = @intCast(n - 1) }; + var topology = try Topology.initAlloc(alloc, n, edges, .none); + defer topology.deinit(alloc); + const total = n + edges.len; + const hub_start = n - 1; + var hub_tiles: usize = 0; + for (0..reduction_partitions) |part| { + const start = vectorBoundary(total, part, reduction_partitions); + const end = vectorBoundary(total, part + 1, reduction_partitions); + try std.testing.expect(end - start <= (total + reduction_partitions - 1) / reduction_partitions); + if (end > hub_start) hub_tiles += 1; + } + try std.testing.expect(hub_tiles >= 8); + const options = Options{ .max_iterations = 3 }; + var serial = try pageRankTopologyAlloc(alloc, topology, options); + defer serial.deinit(alloc); + const input = try alloc.alloc(f64, n); + defer alloc.free(input); + @memset(input, 1); + const output = try alloc.alloc(f64, n); + defer alloc.free(output); + for ([_]usize{ 1, 2, 4, 16 }) |width| { + var parallel = options; + parallel.io = io_impl.io(); + parallel.max_parallelism = width; + var rank = try pageRankTopologyAlloc(alloc, topology, parallel); + defer rank.deinit(alloc); + try std.testing.expectEqualSlices(f64, serial.scores, rank.scores); + try std.testing.expectEqual(serial.delta, rank.delta); + for ([_]bool{ true, false }) |incoming| { + try fillAdjacencySums(topology, input, output, incoming, 1, parallel); + const offsets = if (incoming) topology.incoming_offsets else topology.outgoing_offsets; + for (output, 0..) |value, i| try std.testing.expectEqual(@as(f64, @floatFromInt(offsets[i + 1] - offsets[i])), value); + } + } + } +} diff --git a/zig/pkg/antfly/src/graph/ordinal.zig b/zig/pkg/antfly/src/graph/ordinal.zig new file mode 100644 index 0000000000..0a70595250 --- /dev/null +++ b/zig/pkg/antfly/src/graph/ordinal.zig @@ -0,0 +1,214 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Job-local immutable adjacency, bounded fold state, and fixture value records. +//! These bounded blocks use +//! numeric ordinals throughout iteration; document IDs are resolved only when +//! compiling topology or crossing the public score boundary. +const std = @import("std"); +pub const max_edges = 4096; +pub const fold_entries = 256; + +/// One bounded adjacency fold. Cursor and compensated accumulators commit +/// together; attempt identity prevents a replacement from mixing executions. +pub const Fold = struct { + attempt: u64 = 0, + prior: u64 = 0, + fingerprint: u64 = 0, + position: u16 = 0, + count: u16 = 0, + cursor: []const u8 = "", + sums: [fold_entries]f64 = @splat(0), + corrections: [fold_entries]f64 = @splat(0), + + const fixed_len = 36 + fold_entries * 16; + + pub fn encode(self: *const Fold, alloc: std.mem.Allocator) ![]u8 { + if (self.count > fold_entries or self.position > self.count or self.cursor.len > std.math.maxInt(u32)) return error.InvalidGraphMetricBuildManifest; + const raw = try alloc.alloc(u8, fixed_len + self.cursor.len); + errdefer alloc.free(raw); + @memcpy(raw[0..4], "GOF1"); + std.mem.writeInt(u64, raw[4..12], self.attempt, .little); + std.mem.writeInt(u64, raw[12..20], self.prior, .little); + std.mem.writeInt(u64, raw[20..28], self.fingerprint, .little); + std.mem.writeInt(u16, raw[28..30], self.position, .little); + std.mem.writeInt(u16, raw[30..32], self.count, .little); + std.mem.writeInt(u32, raw[32..36], @intCast(self.cursor.len), .little); + for (self.sums, self.corrections, 0..) |sum, correction, i| { + if (!std.math.isFinite(sum) or sum < 0 or !std.math.isFinite(correction)) return error.InvalidGraphMetricScore; + std.mem.writeInt(u64, raw[36 + i * 16 ..][0..8], @bitCast(sum), .little); + std.mem.writeInt(u64, raw[44 + i * 16 ..][0..8], @bitCast(correction), .little); + } + @memcpy(raw[fixed_len..], self.cursor); + return raw; + } + + /// The cursor borrows raw; the caller owns its lifetime. + pub fn decode(raw: []const u8) !Fold { + if (raw.len < fixed_len or !std.mem.eql(u8, raw[0..4], "GOF1") or std.mem.readInt(u32, raw[32..36], .little) != raw.len - fixed_len) return error.InvalidGraphMetricBuildManifest; + var result = Fold{ + .attempt = std.mem.readInt(u64, raw[4..12], .little), + .prior = std.mem.readInt(u64, raw[12..20], .little), + .fingerprint = std.mem.readInt(u64, raw[20..28], .little), + .position = std.mem.readInt(u16, raw[28..30], .little), + .count = std.mem.readInt(u16, raw[30..32], .little), + .cursor = raw[fixed_len..], + }; + if (result.count > fold_entries or result.position > result.count) return error.InvalidGraphMetricBuildManifest; + for (&result.sums, &result.corrections, 0..) |*sum, *correction, i| { + sum.* = @bitCast(std.mem.readInt(u64, raw[36 + i * 16 ..][0..8], .little)); + correction.* = @bitCast(std.mem.readInt(u64, raw[44 + i * 16 ..][0..8], .little)); + if (!std.math.isFinite(sum.*) or sum.* < 0 or !std.math.isFinite(correction.*)) return error.InvalidGraphMetricScore; + } + return result; + } +}; +pub const Edge = struct { source: u64, target: u64 }; +pub const Value = struct { + ordinal: u64, + value: f64, + pub fn lessThan(_: void, a: Value, b: Value) bool { + return a.ordinal < b.ordinal; + } +}; + +pub const Topology = struct { + edges: []Edge, + cursor: []u8, + scanned: u64, + complete: bool, + pub fn deinit(self: *Topology, alloc: std.mem.Allocator) void { + alloc.free(self.edges); + alloc.free(self.cursor); + self.* = undefined; + } +}; + +pub fn encodeTopology(alloc: std.mem.Allocator, topology: Topology) ![]u8 { + if (topology.edges.len > max_edges or topology.scanned > max_edges or topology.edges.len > topology.scanned or topology.cursor.len > std.math.maxInt(u32)) return error.InvalidGraphMetricBuildManifest; + const out = try alloc.alloc(u8, 17 + topology.cursor.len + topology.edges.len * 16); + @memcpy(out[0..4], "GTO1"); + std.mem.writeInt(u64, out[4..12], topology.scanned, .little); + out[12] = @intFromBool(topology.complete); + std.mem.writeInt(u32, out[13..17], @intCast(topology.cursor.len), .little); + @memcpy(out[17..][0..topology.cursor.len], topology.cursor); + for (topology.edges, 0..) |edge, i| { + const offset = 17 + topology.cursor.len + i * 16; + std.mem.writeInt(u64, out[offset..][0..8], edge.source, .little); + std.mem.writeInt(u64, out[offset + 8 ..][0..8], edge.target, .little); + } + return out; +} + +/// Validated, unaligned wire view. The caller keeps the source bytes alive. +pub const TopologyView = struct { + data: []const u8, + cursor: []const u8, + scanned: u64, + complete: bool, + + pub fn len(self: TopologyView) usize { + return self.data.len / 16; + } + pub fn edge(self: TopologyView, i: usize) Edge { + std.debug.assert(i < self.len()); + return .{ .source = std.mem.readInt(u64, self.data[i * 16 ..][0..8], .little), .target = std.mem.readInt(u64, self.data[i * 16 + 8 ..][0..8], .little) }; + } +}; + +pub fn decodeTopologyView(raw: []const u8) !TopologyView { + if (raw.len < 17 or !std.mem.eql(u8, raw[0..4], "GTO1") or raw[12] > 1) return error.InvalidGraphMetricBuildManifest; + const cursor_len = std.mem.readInt(u32, raw[13..17], .little); + if (cursor_len > raw.len - 17) return error.InvalidGraphMetricBuildManifest; + const data = raw[17 + cursor_len ..]; + const scanned = std.mem.readInt(u64, raw[4..12], .little); + if (data.len % 16 != 0 or data.len / 16 > max_edges or scanned > max_edges or data.len / 16 > scanned) return error.InvalidGraphMetricBuildManifest; + const view = TopologyView{ .data = data, .cursor = raw[17..][0..cursor_len], .scanned = scanned, .complete = raw[12] == 1 }; + for (0..view.len()) |i| { + const value = view.edge(i); + if (value.source == 0 or value.target == 0) return error.InvalidGraphMetricBuildManifest; + } + return view; +} + +pub fn decodeTopology(alloc: std.mem.Allocator, raw: []const u8) !Topology { + const view = try decodeTopologyView(raw); + const edges = try alloc.alloc(Edge, view.len()); + errdefer alloc.free(edges); + for (edges, 0..) |*edge, i| edge.* = view.edge(i); + return .{ .edges = edges, .cursor = try alloc.dupe(u8, view.cursor), .scanned = view.scanned, .complete = view.complete }; +} + +pub fn encodeValues(alloc: std.mem.Allocator, values: []const Value) ![]u8 { + if (values.len > max_edges) return error.InvalidGraphMetricBuildManifest; + const out = try alloc.alloc(u8, values.len * 16); + errdefer alloc.free(out); + for (values, 0..) |value, i| { + if (value.ordinal == 0 or !std.math.isFinite(value.value) or value.value < 0 or (i > 0 and value.ordinal <= values[i - 1].ordinal)) return error.InvalidGraphMetricScore; + std.mem.writeInt(u64, out[i * 16 ..][0..8], value.ordinal, .little); + std.mem.writeInt(u64, out[i * 16 + 8 ..][0..8], @bitCast(value.value), .little); + } + return out; +} + +pub fn decodeValues(alloc: std.mem.Allocator, raw: []const u8) ![]Value { + if (raw.len % 16 != 0 or raw.len / 16 > max_edges) return error.InvalidGraphMetricBuildManifest; + const values = try alloc.alloc(Value, raw.len / 16); + errdefer alloc.free(values); + for (values, 0..) |*value, i| { + value.* = .{ .ordinal = std.mem.readInt(u64, raw[i * 16 ..][0..8], .little), .value = @bitCast(std.mem.readInt(u64, raw[i * 16 + 8 ..][0..8], .little)) }; + if (value.ordinal == 0 or !std.math.isFinite(value.value) or value.value < 0 or (i > 0 and value.ordinal <= values[i - 1].ordinal)) return error.InvalidGraphMetricScore; + } + return values; +} + +test "ordinal blocks round trip and reject malformed input" { + const alloc = std.testing.allocator; + const encoded = try encodeValues(alloc, &.{ .{ .ordinal = 1, .value = 0 }, .{ .ordinal = 900, .value = 0.7 } }); + defer alloc.free(encoded); + const values = try decodeValues(alloc, encoded); + defer alloc.free(values); + try std.testing.expectEqual(@as(u64, 900), values[1].ordinal); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeValues(alloc, encoded[1..])); + try std.testing.expectError(error.InvalidGraphMetricScore, encodeValues(alloc, &.{.{ .ordinal = 1, .value = std.math.nan(f64) }})); + var edges = [_]Edge{.{ .source = 9, .target = 4 }}; + var cursor = [_]u8{ 0, 255 }; + const bytes = try encodeTopology(alloc, .{ .edges = &edges, .cursor = &cursor, .scanned = 2, .complete = false }); + defer alloc.free(bytes); + var decoded = try decodeTopology(alloc, bytes); + defer decoded.deinit(alloc); + try std.testing.expectEqualSlices(u8, &cursor, decoded.cursor); + try std.testing.expectEqual(@as(u64, 9), decoded.edges[0].source); + try std.testing.expect(!decoded.complete); + const view = try decodeTopologyView(bytes); + try std.testing.expectEqual(@as(usize, 1), view.len()); + try std.testing.expectEqual(decoded.edges[0], view.edge(0)); + try std.testing.expectEqual(bytes[17..].ptr, view.cursor.ptr); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeTopologyView(bytes[0 .. bytes.len - 1])); + std.mem.writeInt(u64, bytes[19..][0..8], 0, .little); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, decodeTopologyView(bytes)); + std.mem.writeInt(u64, bytes[19..][0..8], 9, .little); + var fold = Fold{ .attempt = 2, .prior = 256, .count = 2, .position = 1, .cursor = "checkpoint" }; + fold.sums[0] = 1.0e16; + fold.corrections[0] = 1; + const state = try fold.encode(alloc); + defer alloc.free(state); + const restored = try Fold.decode(state); + try std.testing.expectEqual(@as(u64, 2), restored.attempt); + try std.testing.expectEqual(@as(f64, 1), restored.corrections[0]); + try std.testing.expectEqualStrings("checkpoint", restored.cursor); + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, Fold.decode(state[0 .. state.len - 1])); + fold.position = 3; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, fold.encode(alloc)); +} diff --git a/zig/pkg/antfly/src/graph/partition_census.zig b/zig/pkg/antfly/src/graph/partition_census.zig new file mode 100644 index 0000000000..8b577d86a8 --- /dev/null +++ b/zig/pkg/antfly/src/graph/partition_census.zig @@ -0,0 +1,304 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Generation-fenced partition census checkpoints. They are shared by every +//! metric on an index, and bounded by partition count rather than graph size. +const std = @import("std"); +const Allocator = std.mem.Allocator; +const checksum_seed: u64 = 0xA17F_4345_4E53_0001; +pub const max_boundaries = 256; +const max_key_bytes = 1024 * 1024; +const header_len = 60; + +pub const State = struct { + generation: u64, + edge_count: u64, + node_count: u64, + edges_seen: u64 = 0, + nodes_seen: u64 = 0, + edges_done: bool = false, + phase: u8 = 0, + edge_cursor: []u8 = &.{}, + node_cursor: []u8 = &.{}, + edge_boundaries: std.ArrayListUnmanaged([]u8) = .empty, + node_boundaries: std.ArrayListUnmanaged([]u8) = .empty, + persisted_edges: usize = 0, + persisted_nodes: usize = 0, + materialized: bool = false, + + pub fn boundaryCount(self: State, nodes: bool) usize { + const pending = if (nodes) self.node_boundaries.items.len else self.edge_boundaries.items.len; + return pending + if (self.materialized) @as(usize, 0) else if (nodes) self.persisted_nodes else self.persisted_edges; + } + + /// Canonical identity of the addressed boundary records. Call only after + /// materialization, on the read side of the transaction fence. + pub fn boundaryDigest(self: State) [32]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update("antfly:partition-boundaries:v1"); + for ([_][]const []u8{ self.edge_boundaries.items, self.node_boundaries.items }) |list| { + var size: [8]u8 = undefined; + std.mem.writeInt(u64, &size, list.len, .little); + hash.update(&size); + for (list) |key| { + std.mem.writeInt(u64, &size, key.len, .little); + hash.update(&size); + hash.update(key); + } + } + return hash.finalResult(); + } + + fn boundaryKeyAlloc(alloc: Allocator, parent: []const u8, nodes: bool, index: usize) ![]u8 { + var suffix = [_]u8{ '/', @intFromBool(nodes), 0, 0 }; + std.mem.writeInt(u16, suffix[2..4], @intCast(index), .big); + return std.mem.concat(alloc, u8, &.{ parent, &suffix }); + } + + fn boundaryChecksum(generation: u64, nodes: bool, index: usize, key: []const u8) u64 { + return std.hash.Wyhash.hash(checksum_seed ^ generation ^ (@as(u64, @intFromBool(nodes)) << 32) ^ index, key); + } + + /// Only the new page's boundaries are written. Slots are reused on a + /// generation restart, so churn cannot create unbounded abandoned records. + pub fn persistBoundaries(self: State, alloc: Allocator, batch: anytype, parent: []const u8) !void { + for ([_]bool{ false, true }) |nodes| { + const count = if (nodes) self.persisted_nodes else self.persisted_edges; + const list = if (nodes) self.node_boundaries.items else self.edge_boundaries.items; + const pending = if (self.materialized) list[count..] else list; + if (count + pending.len > max_boundaries) return error.InvalidGraphMetricPartitionCensus; + for (pending, count..) |key, index| { + if (key.len == 0 or key.len > max_key_bytes) return error.InvalidGraphMetricPartitionCensus; + const storage_key = try boundaryKeyAlloc(alloc, parent, nodes, index); + defer alloc.free(storage_key); + const raw = try alloc.alloc(u8, 16 + key.len); + defer alloc.free(raw); + std.mem.writeInt(u64, raw[0..8], self.generation, .little); + std.mem.writeInt(u64, raw[8..16], boundaryChecksum(self.generation, nodes, index, key), .little); + @memcpy(raw[16..], key); + try batch.put(storage_key, raw); + } + } + } + + /// Called once, on completion, outside the write transaction. Ordinary + /// progress steps never read or allocate previously persisted boundaries. + pub fn materializeBoundaries(self: *State, alloc: Allocator, txn: anytype, parent: []const u8) !void { + std.debug.assert(!self.materialized); + var loaded: [2]std.ArrayListUnmanaged([]u8) = .{ .empty, .empty }; + defer for (&loaded) |*list| { + for (list.items) |key| alloc.free(key); + list.deinit(alloc); + }; + for ([_]bool{ false, true }, &loaded) |nodes, *list| { + const count = if (nodes) self.persisted_nodes else self.persisted_edges; + const pending = if (nodes) &self.node_boundaries else &self.edge_boundaries; + try list.ensureTotalCapacity(alloc, count + pending.items.len); + for (0..count) |index| { + const key = try boundaryKeyAlloc(alloc, parent, nodes, index); + defer alloc.free(key); + const raw = txn.get(key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricPartitionCensus, + else => return err, + }; + if (raw.len <= 16 or raw.len > 16 + max_key_bytes or + std.mem.readInt(u64, raw[0..8], .little) != self.generation or + std.mem.readInt(u64, raw[8..16], .little) != boundaryChecksum(self.generation, nodes, index, raw[16..])) + return error.InvalidGraphMetricPartitionCensus; + list.appendAssumeCapacity(try alloc.dupe(u8, raw[16..])); + } + for (pending.items) |key| list.appendAssumeCapacity(try alloc.dupe(u8, key)); + for (list.items, 0..) |key, index| { + if (index > 0 and std.mem.order(u8, list.items[index - 1], key) != .lt) return error.InvalidGraphMetricPartitionCensus; + } + } + for ([_]*std.ArrayListUnmanaged([]u8){ &self.edge_boundaries, &self.node_boundaries }, &loaded) |target, *source| { + std.mem.swap(std.ArrayListUnmanaged([]u8), target, source); + } + self.materialized = true; + } + + pub fn deleteBoundaries(alloc: Allocator, batch: anytype, parent: []const u8) !void { + for ([_]bool{ false, true }) |nodes| for (0..max_boundaries) |index| { + const key = try boundaryKeyAlloc(alloc, parent, nodes, index); + defer alloc.free(key); + batch.delete(key) catch |err| if (err != error.NotFound) return err; + }; + } + + pub fn deinit(self: *State, alloc: Allocator) void { + alloc.free(self.edge_cursor); + alloc.free(self.node_cursor); + for (self.edge_boundaries.items) |key| alloc.free(key); + for (self.node_boundaries.items) |key| alloc.free(key); + self.edge_boundaries.deinit(alloc); + self.node_boundaries.deinit(alloc); + self.* = undefined; + } + + pub fn identifies(self: State, generation: u64, edges: u64, nodes: u64) bool { + return self.generation == generation and self.edge_count == edges and self.node_count == nodes; + } + + pub fn encodeAlloc(self: State, alloc: Allocator) ![]u8 { + if (self.phase > 2) return error.InvalidGraphMetricPartitionCensus; + if (self.boundaryCount(false) > max_boundaries or self.boundaryCount(true) > max_boundaries) + return error.InvalidGraphMetricPartitionCensus; + var size: usize = header_len + 8; + for ([_][]const u8{ self.edge_cursor, self.node_cursor }) |key| { + if (key.len > max_key_bytes) return error.InvalidGraphMetricPartitionCensus; + size += key.len; + } + const raw = try alloc.alloc(u8, size); + @memset(raw[0..header_len], 0); + @memcpy(raw[0..4], "GPC2"); + raw[4] = @intFromBool(self.edges_done); + raw[5] = self.phase; + for ([_]u64{ self.generation, self.edge_count, self.node_count, self.edges_seen, self.nodes_seen }, 0..) |value, i| + std.mem.writeInt(u64, raw[8 + i * 8 ..][0..8], value, .little); + std.mem.writeInt(u16, raw[48..50], @intCast(self.boundaryCount(false)), .little); + std.mem.writeInt(u16, raw[50..52], @intCast(self.boundaryCount(true)), .little); + std.mem.writeInt(u32, raw[52..56], @intCast(self.edge_cursor.len), .little); + std.mem.writeInt(u32, raw[56..60], @intCast(self.node_cursor.len), .little); + var pos: usize = header_len; + for ([_][]const u8{ self.edge_cursor, self.node_cursor }) |key| { + @memcpy(raw[pos..][0..key.len], key); + pos += key.len; + } + std.mem.writeInt(u64, raw[pos..][0..8], std.hash.Wyhash.hash(checksum_seed, raw[0..pos]), .little); + return raw; + } + + pub fn decodeAlloc(alloc: Allocator, raw: []const u8) !?State { + if (raw.len < header_len + 8 or !std.mem.eql(u8, raw[0..4], "GPC2") or raw[4] > 1 or raw[5] > 2 or + !std.mem.eql(u8, raw[6..8], &.{ 0, 0 })) return null; + const end = raw.len - 8; + if (std.hash.Wyhash.hash(checksum_seed, raw[0..end]) != std.mem.readInt(u64, raw[end..][0..8], .little)) return null; + var state = State{ + .generation = std.mem.readInt(u64, raw[8..16], .little), + .edge_count = std.mem.readInt(u64, raw[16..24], .little), + .node_count = std.mem.readInt(u64, raw[24..32], .little), + .edges_seen = std.mem.readInt(u64, raw[32..40], .little), + .nodes_seen = std.mem.readInt(u64, raw[40..48], .little), + .edges_done = raw[4] != 0, + .phase = raw[5], + .persisted_edges = std.mem.readInt(u16, raw[48..50], .little), + .persisted_nodes = std.mem.readInt(u16, raw[50..52], .little), + }; + var owned = true; + defer if (owned) state.deinit(alloc); + if (state.edges_seen > state.edge_count or state.nodes_seen > state.node_count or + (state.edges_done and state.edges_seen != state.edge_count)) return null; + var pos: usize = header_len; + for ([_]*[]u8{ &state.edge_cursor, &state.node_cursor }, [_]usize{ 52, 56 }) |cursor, offset| { + const len = std.mem.readInt(u32, raw[offset..][0..4], .little); + if (len > max_key_bytes or len > end - pos) return null; + cursor.* = try alloc.dupe(u8, raw[pos..][0..len]); + pos += len; + } + if (state.persisted_edges > max_boundaries or state.persisted_nodes > max_boundaries) return null; + if (pos != end) return null; + owned = false; + return state; + } +}; + +test "partition census owns bounded checkpoints and rejects corruption" { + const alloc = std.testing.allocator; + var state = State{ .generation = 7, .edge_count = 10, .node_count = 4, .edges_seen = 2 }; + defer state.deinit(alloc); + try state.edge_boundaries.append(alloc, try alloc.dupe(u8, "edge-a")); + state.edge_cursor = try alloc.dupe(u8, "edge-b"); + const raw = try state.encodeAlloc(alloc); + defer alloc.free(raw); + var decoded = (try State.decodeAlloc(alloc, raw)).?; + defer decoded.deinit(alloc); + try std.testing.expect(decoded.identifies(7, 10, 4)); + try std.testing.expectEqualStrings("edge-b", decoded.edge_cursor); + raw[16] ^= 1; + try std.testing.expect((try State.decodeAlloc(alloc, raw)) == null); +} + +test "partition census owns bounded checkpoints without replaying accumulated boundary bytes" { + const alloc = std.testing.allocator; + const Store = struct { + values: std.StringHashMapUnmanaged([]u8) = .empty, + writes: usize = 0, + reads: usize = 0, + + fn deinit(self: *@This()) void { + var entries = self.values.iterator(); + while (entries.next()) |entry| { + std.testing.allocator.free(entry.key_ptr.*); + std.testing.allocator.free(entry.value_ptr.*); + } + self.values.deinit(std.testing.allocator); + } + + pub fn put(self: *@This(), key: []const u8, value: []const u8) !void { + const owned = try std.testing.allocator.dupe(u8, value); + errdefer std.testing.allocator.free(owned); + const result = try self.values.getOrPut(std.testing.allocator, key); + if (result.found_existing) std.testing.allocator.free(result.value_ptr.*) else { + errdefer _ = self.values.remove(key); + result.key_ptr.* = try std.testing.allocator.dupe(u8, key); + } + result.value_ptr.* = owned; + self.writes += 1; + } + + pub fn get(self: *@This(), key: []const u8) anyerror![]const u8 { + self.reads += 1; + return self.values.get(key) orelse error.NotFound; + } + + pub fn delete(self: *@This(), key: []const u8) !void { + const entry = self.values.fetchRemove(key) orelse return error.NotFound; + std.testing.allocator.free(entry.key); + std.testing.allocator.free(entry.value); + } + }; + var store = Store{}; + defer store.deinit(); + var state = State{ .generation = 7, .edge_count = 1048576, .node_count = 1048577 }; + defer state.deinit(alloc); + // Legal long relationship names formerly made this progress record 32 MiB. + const key = try alloc.alloc(u8, 128 * 1024); + defer alloc.free(key); + @memset(key, 'a'); + for (0..max_boundaries) |i| { + std.mem.writeInt(u32, key[key.len - 4 ..][0..4], @intCast(i), .big); + try state.edge_boundaries.append(alloc, try alloc.dupe(u8, key)); + } + state.edge_cursor = try alloc.dupe(u8, key); + try state.persistBoundaries(alloc, &store, "census"); + try std.testing.expectEqual(max_boundaries, store.writes); + const raw = try state.encodeAlloc(alloc); + defer alloc.free(raw); + try std.testing.expectEqual(@as(usize, 128 * 1024 + header_len + 8), raw.len); + var resumed = (try State.decodeAlloc(alloc, raw)).?; + defer resumed.deinit(alloc); + try resumed.persistBoundaries(alloc, &store, "census"); + try std.testing.expectEqual(max_boundaries, store.writes); + try std.testing.expectEqual(@as(usize, 0), store.reads); + try std.testing.expectEqual(@as(usize, 0), resumed.edge_boundaries.items.len); + try resumed.materializeBoundaries(alloc, &store, "census"); + try std.testing.expectEqual(max_boundaries, store.reads); + for (resumed.edge_boundaries.items, state.edge_boundaries.items) |a, b| try std.testing.expectEqualSlices(u8, a, b); + try State.deleteBoundaries(alloc, &store, "census"); + try std.testing.expectEqual(@as(usize, 0), store.values.count()); + var missing = (try State.decodeAlloc(alloc, raw)).?; + defer missing.deinit(alloc); + try std.testing.expectError(error.InvalidGraphMetricPartitionCensus, missing.materializeBoundaries(alloc, &store, "census")); +} diff --git a/zig/pkg/antfly/src/graph/query.zig b/zig/pkg/antfly/src/graph/query.zig index 9fdc4b934e..c019679073 100644 --- a/zig/pkg/antfly/src/graph/query.zig +++ b/zig/pkg/antfly/src/graph/query.zig @@ -350,6 +350,73 @@ pub const QueryParams = struct { node_filter: pattern_mod.NodeFilter = .{}, }; +/// Exact graph-metric filtering and ordering must observe the full candidate +/// set. Bound that set explicitly so a broad traversal cannot turn a small +/// requested page into unbounded memory and sort work. +pub const graph_metric_candidate_limit: u32 = 100_000; +/// Public and internal graph metric query bounds. These limits keep request +/// parsing, metric materialization, and comparison work predictable even when +/// an internal caller bypasses the OpenAPI layer. +pub const graph_metric_projection_limit: usize = 16; +pub const graph_metric_order_limit: usize = 8; +pub const graph_metric_filter_limit: usize = 32; +pub const graph_metric_dependency_limit: usize = 16; + +/// Backend-independent late-materialization plan. Names borrow the validated +/// query; each stage is deduplicated and ordering remains user-defined. +pub const MetricReadPlan = struct { + const Names = struct { + buffer: [graph_metric_dependency_limit][]const u8 = undefined, + len: usize = 0, + + pub fn slice(self: *const @This()) []const []const u8 { + return self.buffer[0..self.len]; + } + fn append(self: *@This(), name: []const u8) void { + for (self.slice()) |prior| if (std.mem.eql(u8, name, prior)) return; + std.debug.assert(self.len < self.buffer.len); + self.buffer[self.len] = name; + self.len += 1; + } + }; + dependencies: Names = .{}, + filters: Names = .{}, + orders: Names = .{}, + projections: Names = .{}, + policies: [graph_metric_dependency_limit]graph_mod.GraphIndex.GraphMetricColumnReadPolicy = @splat(.{}), + + fn require(self: *@This(), name: []const u8, freshness: GraphMetricFreshness, published: bool) void { + for (self.dependencies.slice(), 0..) |dependency, i| { + if (!std.mem.eql(u8, dependency, name)) continue; + self.policies[i].require_published = self.policies[i].require_published or published; + self.policies[i].require_fresh = self.policies[i].require_fresh or freshness == .fresh; + return; + } + unreachable; + } + + pub fn init(query: GraphQuery) !MetricReadPlan { + try validateGraphMetricQueryShape(query); + var plan = MetricReadPlan{}; + for (query.metrics) |metric| { + plan.dependencies.append(metric.name); + plan.projections.append(metric.name); + plan.require(metric.name, metric.freshness, false); + } + for (query.order_by) |order| { + plan.dependencies.append(order.name); + plan.orders.append(order.name); + plan.require(order.name, order.freshness, true); + } + for (query.where_metric) |filter| { + plan.dependencies.append(filter.name); + plan.filters.append(filter.name); + plan.require(filter.name, filter.freshness, true); + } + return plan; + } +}; + pub fn nodeFilterActive(filter: pattern_mod.NodeFilter) bool { return filter.filter_prefix.len > 0 or filter.filter_query_json != null; } @@ -390,6 +457,10 @@ pub const GraphQuery = struct { include_documents: bool = false, fields: []const []const u8 = &.{}, include_all_fields: bool = true, + metrics: []const GraphMetricRead = &.{}, + order_by: []const GraphMetricOrder = &.{}, + where_metric: []const GraphMetricFilter = &.{}, + include_metric_status: bool = false, }; pub const NamedCountAggregate = struct { @@ -400,6 +471,71 @@ pub const NamedCountAggregate = struct { pub const ExpandStrategy = enum { @"union", intersection }; +pub const GraphMetricFreshness = enum { published, fresh }; + +pub const GraphMetricRead = struct { + name: []const u8, + freshness: GraphMetricFreshness = .published, +}; + +pub const GraphMetricOrderDirection = enum { asc, desc }; +pub const GraphMetricNullOrder = enum { first, last }; + +pub const GraphMetricOrder = struct { + name: []const u8, + direction: GraphMetricOrderDirection = .desc, + nulls: GraphMetricNullOrder = .last, + freshness: GraphMetricFreshness = .published, +}; + +pub const GraphMetricFilterOp = enum { gt, gte, lt, lte, eq, neq }; + +pub const GraphMetricFilter = struct { + name: []const u8, + op: GraphMetricFilterOp, + value: f64, + freshness: GraphMetricFreshness = .published, +}; + +/// Validate metric cardinality and comparison semantics before graph fan-out. +pub fn validateGraphMetricQueryShape(query: GraphQuery) !void { + if (query.metrics.len > graph_metric_projection_limit or + query.order_by.len > graph_metric_order_limit or + query.where_metric.len > graph_metric_filter_limit) + return error.InvalidQueryRequest; + + var dependency_names: [graph_metric_dependency_limit][]const u8 = undefined; + var dependency_count: usize = 0; + for (query.metrics, 0..) |metric, i| { + if (metric.name.len == 0) return error.InvalidQueryRequest; + for (query.metrics[0..i]) |previous| + if (std.mem.eql(u8, previous.name, metric.name)) return error.InvalidQueryRequest; + try appendGraphMetricDependencyName(&dependency_names, &dependency_count, metric.name); + } + for (query.order_by, 0..) |order, i| { + if (order.name.len == 0) return error.InvalidQueryRequest; + for (query.order_by[0..i]) |previous| + if (std.mem.eql(u8, previous.name, order.name)) return error.InvalidQueryRequest; + try appendGraphMetricDependencyName(&dependency_names, &dependency_count, order.name); + } + for (query.where_metric) |filter| { + if (filter.name.len == 0 or !std.math.isFinite(filter.value)) return error.InvalidQueryRequest; + try appendGraphMetricDependencyName(&dependency_names, &dependency_count, filter.name); + } +} + +fn appendGraphMetricDependencyName( + names: *[graph_metric_dependency_limit][]const u8, + count: *usize, + name: []const u8, +) !void { + for (names[0..count.*]) |existing| + if (std.mem.eql(u8, existing, name)) return; + if (count.* == names.len) return error.InvalidQueryRequest; + names[count.*] = name; + count.* += 1; +} + // ============================================================================ // Result types // ============================================================================ @@ -429,10 +565,16 @@ pub const GraphResultNode = struct { /// node is same-table (hydrated locally). Lets the api hydrate a cross-table /// entity node from its own table instead of failing closed. table: ?[]const u8 = null, + metrics: []GraphMetricValue = &.{}, + /// False when metrics is a row view into GraphQueryResult's contiguous + /// metric_values_slab. Values still own any promoted names individually. + metrics_owned: bool = true, pub fn deinit(self: *GraphResultNode, alloc: Allocator) void { alloc.free(self.key); if (self.table) |t| alloc.free(t); + for (self.metrics) |*metric| metric.deinit(alloc); + if (self.metrics_owned and self.metrics.len > 0) alloc.free(self.metrics); if (self.path) |p| { for (p) |s| alloc.free(s); alloc.free(p); @@ -532,6 +674,109 @@ test "canonical graph result node path is self-consistent" { })); } +pub const GraphMetricValue = struct { + name: []const u8, + score: ?f64 = null, + name_owned: bool = true, + + pub fn ensureNameOwned(self: *GraphMetricValue, alloc: Allocator) !void { + if (self.name_owned) return; + self.name = try alloc.dupe(u8, self.name); + self.name_owned = true; + } + + pub fn deinit(self: *GraphMetricValue, alloc: Allocator) void { + if (self.name_owned) alloc.free(self.name); + self.* = undefined; + } +}; + +pub const GraphMetricStatus = struct { + name: []const u8, + state: graph_mod.GraphIndex.GraphMetricState = .not_ready, + phase: graph_mod.GraphIndex.GraphMetricBuildPhase = .idle, + edge_filter: graph_mod.GraphMetricEdgeFilter = .{}, + metadata_version: u32 = 0, + config_fingerprint: u64 = 0, + maintenance_paused: bool = false, + build_queued: bool = false, + published_generation: u64 = 0, + published_edge_generation: u64 = 0, + edge_generation: u64 = 0, + target_edge_generation: u64 = 0, + queued_generation: u64 = 0, + building_generation: u64 = 0, + build_job_id: u64 = 0, + build_started_at_ms: u64 = 0, + build_iteration: u32 = 0, + build_lease_expires_at_ms: u64 = 0, + build_worker_id: []const u8 = "", + retry_count: u64 = 0, + last_error: []const u8 = "", + progress: f64 = 0, + converged: bool = false, + iterations_completed: u32 = 0, + delta: f64 = 0, + computed_at_ms: u64 = 0, + last_event: ?graph_mod.GraphIndex.GraphMetricEvent = null, + recent_events: []graph_mod.GraphIndex.GraphMetricEvent = &.{}, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.name); + self.edge_filter.deinit(alloc); + if (self.build_worker_id.len > 0) alloc.free(self.build_worker_id); + if (self.last_error.len > 0) alloc.free(self.last_error); + if (self.recent_events.len > 0) alloc.free(self.recent_events); + self.* = undefined; + } +}; + +fn cloneGraphMetricStatus(alloc: Allocator, source: graph_mod.GraphIndex.GraphMetricStatus) !GraphMetricStatus { + const name = try alloc.dupe(u8, source.name); + errdefer alloc.free(name); + var edge_filter = try source.edge_filter.cloneAlloc(alloc); + errdefer edge_filter.deinit(alloc); + const recent_events = if (source.recent_events.len > 0) + try alloc.dupe(graph_mod.GraphIndex.GraphMetricEvent, source.recent_events) + else + @constCast((&[_]graph_mod.GraphIndex.GraphMetricEvent{})[0..]); + errdefer if (recent_events.len > 0) alloc.free(recent_events); + const last_error = if (source.last_error.len > 0) try alloc.dupe(u8, source.last_error) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + const build_worker_id = if (source.build_worker_id.len > 0) try alloc.dupe(u8, source.build_worker_id) else ""; + errdefer if (build_worker_id.len > 0) alloc.free(build_worker_id); + return .{ + .name = name, + .state = source.state, + .phase = source.phase, + .edge_filter = edge_filter, + .metadata_version = source.metadata_version, + .config_fingerprint = source.config_fingerprint, + .maintenance_paused = source.maintenance_paused, + .build_queued = source.build_queued, + .published_generation = source.published_generation, + .published_edge_generation = source.published_edge_generation, + .edge_generation = source.edge_generation, + .target_edge_generation = source.target_edge_generation, + .queued_generation = source.queued_generation, + .building_generation = source.building_generation, + .build_job_id = source.build_job_id, + .build_started_at_ms = source.build_started_at_ms, + .build_iteration = source.build_iteration, + .build_lease_expires_at_ms = source.build_lease_expires_at_ms, + .build_worker_id = build_worker_id, + .retry_count = source.retry_count, + .last_error = last_error, + .progress = source.progress, + .converged = source.converged, + .iterations_completed = source.iterations_completed, + .delta = source.delta, + .computed_at_ms = source.computed_at_ms, + .last_event = source.last_event, + .recent_events = recent_events, + }; +} + test "graph result node JSON accepts omitted optional path fields" { var parsed = try std.json.parseFromSlice( GraphResultNode, @@ -549,11 +794,19 @@ test "graph result node JSON accepts omitted optional path fields" { pub const GraphQueryResult = struct { nodes: []GraphResultNode, matches: []pattern_mod.PatternMatch = &.{}, + metric_status: []GraphMetricStatus = &.{}, + metric_values_slab: []GraphMetricValue = &.{}, + metric_value_names: [][]u8 = &.{}, pub fn deinit(self: *GraphQueryResult, alloc: Allocator) void { for (self.nodes) |*node| node.deinit(alloc); alloc.free(self.nodes); + if (self.metric_values_slab.len > 0) alloc.free(self.metric_values_slab); + for (self.metric_value_names) |name| alloc.free(name); + if (self.metric_value_names.len > 0) alloc.free(self.metric_value_names); pattern_mod.freeMatches(alloc, self.matches); + for (self.metric_status) |*status| status.deinit(alloc); + if (self.metric_status.len > 0) alloc.free(self.metric_status); } }; @@ -576,19 +829,583 @@ pub const GraphQueryEngine = struct { gq: GraphQuery, resolved_keys: []const []const u8, ) !GraphQueryResult { - return switch (gq.query_type) { - .traverse => self.executeTraverse(graph_index, gq.params, resolved_keys, resolveTargetKeys(gq)), + try validateGraphMetricQueryShape(gq); + const defer_result_limit = graphMetricPostProcessingNeedsFullCandidateSet(gq); + var execution_params = gq.params; + if (defer_result_limit) execution_params.max_results = graph_metric_candidate_limit + 1; + + var execution_query = gq; + execution_query.params = execution_params; + + var result = try switch (execution_query.query_type) { + .traverse => self.executeTraverse(graph_index, execution_params, resolved_keys, resolveTargetKeys(gq)), .neighbors => blk: { - var params = gq.params; + var params = execution_params; params.max_depth = 1; break :blk self.executeTraverse(graph_index, params, resolved_keys, resolveTargetKeys(gq)); }, - .shortest_path => self.executeShortestPath(graph_index, gq, resolved_keys), - .k_shortest_paths => self.executeKShortestPaths(graph_index, gq, resolved_keys), - .pattern => self.executePattern(graph_index, gq, resolved_keys), + .shortest_path => self.executeShortestPath(graph_index, execution_query, resolved_keys), + .k_shortest_paths => self.executeKShortestPaths(graph_index, execution_query, resolved_keys), + .pattern => self.executePattern(graph_index, execution_query, resolved_keys), + }; + errdefer result.deinit(self.alloc); + if (defer_result_limit and result.nodes.len > graph_metric_candidate_limit) { + return error.QueryCandidateBudgetExceeded; + } + if (gq.metrics.len != 0 or gq.order_by.len != 0 or gq.where_metric.len != 0) { + try self.applyMetricDependenciesColumnar(graph_index, gq, defer_result_limit, &result); + } + return result; + } + + fn graphMetricPostProcessingNeedsFullCandidateSet(gq: GraphQuery) bool { + return gq.where_metric.len > 0 or gq.order_by.len > 0; + } + + fn metricFilterMatches(score: f64, filter: GraphMetricFilter) bool { + return switch (filter.op) { + .gt => score > filter.value, + .gte => score >= filter.value, + .lt => score < filter.value, + .lte => score <= filter.value, + .eq => score == filter.value, + .neq => score != filter.value, }; } + const MetricColumnSortContext = struct { + orders: []const GraphMetricOrder, + metric_indexes: []const usize, + score_columns: []const []?f64, + }; + + fn metricColumnCandidateLessThan(context: MetricColumnSortContext, left: usize, right: usize) bool { + for (context.orders, context.metric_indexes) |order, metric_index| { + const cmp = compareOptionalMetricScore( + context.score_columns[metric_index][left], + context.score_columns[metric_index][right], + order, + ); + if (cmp) |less| return less; + } + return left < right; + } + + fn siftWorstMetricCandidateUp(context: MetricColumnSortContext, heap: []usize, start: usize) void { + var child = start; + while (child > 0) { + const parent = (child - 1) / 2; + if (!metricColumnCandidateLessThan(context, heap[parent], heap[child])) break; + std.mem.swap(usize, &heap[parent], &heap[child]); + child = parent; + } + } + + fn siftWorstMetricCandidateDown(context: MetricColumnSortContext, heap: []usize, start: usize) void { + var parent = start; + while (true) { + const left = parent * 2 + 1; + if (left >= heap.len) return; + const right = left + 1; + var worse_child = left; + if (right < heap.len and metricColumnCandidateLessThan(context, heap[left], heap[right])) { + worse_child = right; + } + if (!metricColumnCandidateLessThan(context, heap[parent], heap[worse_child])) return; + std.mem.swap(usize, &heap[parent], &heap[worse_child]); + parent = worse_child; + } + } + + /// Select and order only the externally observable prefix. This changes a + /// 100k-candidate query with a 100-row limit from O(N log N) to O(N log K) + /// while retaining deterministic original-order tie breaking. + fn retainOrderedMetricCandidatePrefix( + candidates: []usize, + keep_count: usize, + context: MetricColumnSortContext, + ) []usize { + const keep = @min(candidates.len, keep_count); + if (keep == 0) return candidates[0..0]; + if (keep == candidates.len) { + std.mem.sort(usize, candidates, context, metricColumnCandidateLessThan); + return candidates; + } + const heap = candidates[0..keep]; + for (1..heap.len) |i| siftWorstMetricCandidateUp(context, heap, i); + for (candidates[keep..]) |candidate| { + if (!metricColumnCandidateLessThan(context, candidate, heap[0])) continue; + heap[0] = candidate; + siftWorstMetricCandidateDown(context, heap, 0); + } + std.mem.sort(usize, heap, context, metricColumnCandidateLessThan); + return heap; + } + + fn compareOptionalMetricScore(left: ?f64, right: ?f64, order: GraphMetricOrder) ?bool { + if (left == null and right == null) return null; + if (left == null) return order.nulls == .first; + if (right == null) return order.nulls != .first; + if (left.? == right.?) return null; + return if (order.direction == .desc) left.? > right.? else left.? < right.?; + } + + fn metricColumnNameIndex(dependency_names: []const []const u8, name: []const u8) ?usize { + for (dependency_names, 0..) |dependency_name, i| { + if (std.mem.eql(u8, dependency_name, name)) return i; + } + return null; + } + + fn metricCandidatePassesFilters( + candidate_index: usize, + filters: []const GraphMetricFilter, + filter_metric_indexes: []const usize, + score_columns: []const []?f64, + ) bool { + for (filters, filter_metric_indexes) |filter, metric_index| { + const score = score_columns[metric_index][candidate_index] orelse return false; + if (!metricFilterMatches(score, filter)) return false; + } + return true; + } + + /// Returns source row indexes after metric filtering, ordering, and the + /// optional response limit. Serverless uses this to compact and reuse + /// already-fetched columns between its filter/order/projection stages. + pub fn selectLoadedMetricCandidateIndexesAlloc( + alloc: Allocator, + dependency_names: []const []const u8, + score_columns: []const []?f64, + query: GraphQuery, + apply_result_limit: bool, + node_count: usize, + ) ![]usize { + try validateGraphMetricQueryShape(query); + if (dependency_names.len != score_columns.len) return error.InvalidQueryRequest; + for (score_columns, dependency_names, 0..) |column, dependency_name, i| { + if (column.len != node_count) return error.InvalidQueryRequest; + for (dependency_names[0..i]) |prior_name| { + if (std.mem.eql(u8, prior_name, dependency_name)) return error.InvalidQueryRequest; + } + } + + var filter_index_buffer: [graph_metric_filter_limit]usize = undefined; + const filter_metric_indexes = filter_index_buffer[0..query.where_metric.len]; + for (query.where_metric, 0..) |filter, i| { + filter_metric_indexes[i] = metricColumnNameIndex(dependency_names, filter.name) orelse + return error.InvalidQueryRequest; + } + const candidate_indexes = try alloc.alloc(usize, node_count); + defer alloc.free(candidate_indexes); + var candidate_count: usize = 0; + for (0..node_count) |original_index| { + if (!metricCandidatePassesFilters( + original_index, + query.where_metric, + filter_metric_indexes, + score_columns, + )) continue; + candidate_indexes[candidate_count] = original_index; + candidate_count += 1; + } + var selected = candidate_indexes[0..candidate_count]; + const requested_limit: usize = if (apply_result_limit and query.params.max_results != 0) + @intCast(query.params.max_results) + else + selected.len; + if (query.order_by.len > 0 and selected.len > 0) { + var order_index_buffer: [graph_metric_order_limit]usize = undefined; + const order_metric_indexes = order_index_buffer[0..query.order_by.len]; + for (query.order_by, 0..) |order, i| { + order_metric_indexes[i] = metricColumnNameIndex(dependency_names, order.name) orelse + return error.InvalidQueryRequest; + } + selected = retainOrderedMetricCandidatePrefix(selected, requested_limit, .{ + .orders = query.order_by, + .metric_indexes = order_metric_indexes, + .score_columns = score_columns, + }); + } else if (selected.len > requested_limit) { + selected = selected[0..requested_limit]; + } + return try alloc.dupe(usize, selected); + } + + /// Storage-independent graph-metric post-processing. Callers retain score + /// columns in their native storage representation until this routine has + /// filtered and selected the externally visible prefix. Metric objects are + /// allocated only for surviving rows. + pub fn applyLoadedMetricColumns( + alloc: Allocator, + dependency_names: []const []const u8, + metric_value_names: []const []const u8, + score_columns: []const []?f64, + query: GraphQuery, + apply_result_limit: bool, + nodes: *[]GraphResultNode, + ) ![]GraphMetricValue { + try validateGraphMetricQueryShape(query); + if (dependency_names.len != score_columns.len or metric_value_names.len != score_columns.len) + return error.InvalidQueryRequest; + for (score_columns, dependency_names, metric_value_names, 0..) |column, dependency_name, metric_value_name, i| { + if (column.len != nodes.*.len or !std.mem.eql(u8, dependency_name, metric_value_name)) + return error.InvalidQueryRequest; + for (dependency_names[0..i]) |prior_name| { + if (std.mem.eql(u8, prior_name, dependency_name)) return error.InvalidQueryRequest; + } + } + + const selected = try selectLoadedMetricCandidateIndexesAlloc( + alloc, + dependency_names, + score_columns, + query, + apply_result_limit, + nodes.*.len, + ); + defer alloc.free(selected); + + return try applySelectedMetricColumns( + alloc, + dependency_names, + metric_value_names, + score_columns, + query, + selected, + nodes, + ); + } + + /// Materializes a selection previously produced by + /// `selectLoadedMetricCandidateIndexesAlloc`. Keeping selection separate + /// lets staged backends reuse the exact same indexes for node mutation and + /// score-column compaction instead of repeating filtering and top-k work. + pub fn applySelectedMetricColumns( + alloc: Allocator, + dependency_names: []const []const u8, + metric_value_names: []const []const u8, + score_columns: []const []?f64, + query: GraphQuery, + selected: []const usize, + nodes: *[]GraphResultNode, + ) ![]GraphMetricValue { + try validateGraphMetricQueryShape(query); + if (dependency_names.len != score_columns.len or metric_value_names.len != score_columns.len) + return error.InvalidQueryRequest; + for (score_columns, dependency_names, metric_value_names, 0..) |column, dependency_name, metric_value_name, i| { + if (column.len != nodes.*.len or !std.mem.eql(u8, dependency_name, metric_value_name)) + return error.InvalidQueryRequest; + for (dependency_names[0..i]) |prior_name| { + if (std.mem.eql(u8, prior_name, dependency_name)) return error.InvalidQueryRequest; + } + } + + var projection_index_buffer: [graph_metric_projection_limit]usize = undefined; + const projection_metric_indexes = projection_index_buffer[0..query.metrics.len]; + for (query.metrics, 0..) |metric, i| { + projection_metric_indexes[i] = metricColumnNameIndex(dependency_names, metric.name) orelse + return error.InvalidQueryRequest; + } + + var selected_mask = try std.DynamicBitSetUnmanaged.initEmpty(alloc, nodes.*.len); + defer selected_mask.deinit(alloc); + for (selected) |source_index| { + if (source_index >= nodes.*.len or selected_mask.isSet(source_index)) return error.InvalidQueryRequest; + selected_mask.set(source_index); + } + + const slab_len = std.math.mul(usize, selected.len, query.metrics.len) catch return error.QueryCandidateBudgetExceeded; + const metric_values_slab = if (slab_len == 0) + @constCast((&[_]GraphMetricValue{})[0..]) + else + try alloc.alloc(GraphMetricValue, slab_len); + errdefer if (metric_values_slab.len > 0) alloc.free(metric_values_slab); + for (selected, 0..) |source_index, row_index| { + const row = metric_values_slab[row_index * query.metrics.len ..][0..query.metrics.len]; + for (row, projection_metric_indexes) |*value, metric_index| { + value.* = .{ + .name = metric_value_names[metric_index], + .score = score_columns[metric_index][source_index], + .name_owned = false, + }; + } + } + + const final_nodes = try alloc.alloc(GraphResultNode, selected.len); + for (selected, 0..) |source_index, out_index| { + var node = nodes.*[source_index]; + for (node.metrics) |*metric| metric.deinit(alloc); + if (node.metrics_owned and node.metrics.len > 0) alloc.free(node.metrics); + node.metrics = metric_values_slab[out_index * query.metrics.len ..][0..query.metrics.len]; + node.metrics_owned = false; + final_nodes[out_index] = node; + } + for (nodes.*, 0..) |*node, source_index| if (!selected_mask.isSet(source_index)) node.deinit(alloc); + alloc.free(nodes.*); + nodes.* = final_nodes; + return metric_values_slab; + } + + /// Installs an already-selected row set whose score columns are aligned + /// with output order rather than the original candidate array. Staged + /// backends use this to carry stable source ordinals through filtering and + /// ordering, then move nodes and materialize projected values exactly once. + pub fn materializeSelectedMetricColumns( + alloc: Allocator, + metric_value_names: []const []const u8, + aligned_score_columns: []const []?f64, + selected_source_indexes: []const usize, + nodes: *[]GraphResultNode, + ) ![]GraphMetricValue { + return materializeSelectedMetricColumnsWithAllocators(alloc, alloc, alloc, metric_value_names, aligned_score_columns, selected_source_indexes, nodes); + } + + fn materializeSelectedMetricColumnsWithAllocators( + node_alloc: Allocator, + alloc: Allocator, + scratch: Allocator, + metric_value_names: []const []const u8, + aligned_score_columns: []const []?f64, + selected_source_indexes: []const usize, + nodes: *[]GraphResultNode, + ) ![]GraphMetricValue { + if (metric_value_names.len != aligned_score_columns.len) + return error.InvalidQueryRequest; + for (aligned_score_columns, metric_value_names, 0..) |column, name, i| { + if (column.len != selected_source_indexes.len or name.len == 0) + return error.InvalidQueryRequest; + for (metric_value_names[0..i]) |prior_name| { + if (std.mem.eql(u8, prior_name, name)) return error.InvalidQueryRequest; + } + } + + var selected_mask = try std.DynamicBitSetUnmanaged.initEmpty(scratch, nodes.*.len); + defer selected_mask.deinit(scratch); + for (selected_source_indexes) |source_index| { + if (source_index >= nodes.*.len or selected_mask.isSet(source_index)) + return error.InvalidQueryRequest; + selected_mask.set(source_index); + } + + const slab_len = std.math.mul(usize, selected_source_indexes.len, metric_value_names.len) catch + return error.QueryCandidateBudgetExceeded; + const metric_values_slab = if (slab_len == 0) + @constCast((&[_]GraphMetricValue{})[0..]) + else + try alloc.alloc(GraphMetricValue, slab_len); + errdefer if (metric_values_slab.len > 0) alloc.free(metric_values_slab); + for (selected_source_indexes, 0..) |_, row_index| { + const row = metric_values_slab[row_index * metric_value_names.len ..][0..metric_value_names.len]; + for (row, metric_value_names, aligned_score_columns) |*value, name, column| { + value.* = .{ .name = name, .score = column[row_index], .name_owned = false }; + } + } + + const final_nodes = try alloc.alloc(GraphResultNode, selected_source_indexes.len); + for (selected_source_indexes, 0..) |source_index, out_index| { + var node = nodes.*[source_index]; + for (node.metrics) |*metric| metric.deinit(node_alloc); + if (node.metrics_owned and node.metrics.len > 0) node_alloc.free(node.metrics); + node.metrics = metric_values_slab[out_index * metric_value_names.len ..][0..metric_value_names.len]; + node.metrics_owned = false; + final_nodes[out_index] = node; + } + for (nodes.*, 0..) |*node, source_index| if (!selected_mask.isSet(source_index)) node.deinit(node_alloc); + node_alloc.free(nodes.*); + nodes.* = final_nodes; + return metric_values_slab; + } + + /// Keep metric scores columnar through filtering, ordering, and limiting. + /// Per-node public metric objects are created only for the surviving result + /// page, avoiding candidate-count heap fragmentation and needless copies. + fn applyMetricDependenciesColumnar( + self: *GraphQueryEngine, + graph_index: *graph_mod.GraphIndex, + query: GraphQuery, + apply_result_limit: bool, + result: *GraphQueryResult, + ) !void { + var scratch_budget = work_budget_mod.RetainedAllocator{ .backing = self.alloc, .budget = self.work_budget }; + var output_budget = work_budget_mod.RetainedAllocator{ .backing = self.alloc, .budget = self.work_budget }; + defer std.debug.assert(scratch_budget.live_bytes == 0 and output_budget.live_bytes == 0); + self.applyStagedMetricDependencies(graph_index, query, apply_result_limit, result, scratch_budget.allocator(), output_budget.allocator()) catch |err| { + if (err == error.OutOfMemory and (scratch_budget.denied or output_budget.denied)) return error.GraphWorkBudgetExceeded; + return err; + }; + output_budget.detach(); + } + + fn applyStagedMetricDependencies( + self: *GraphQueryEngine, + graph_index: *graph_mod.GraphIndex, + query: GraphQuery, + apply_result_limit: bool, + result: *GraphQueryResult, + scratch: Allocator, + output: Allocator, + ) !void { + const plan = try MetricReadPlan.init(query); + var session = try graph_index.openGraphMetricReadSessionAlloc(scratch, plan.dependencies.slice(), plan.policies[0..plan.dependencies.len]); + defer session.deinit(); + var workspace = try MetricStageWorkspace.init(scratch, plan, result.nodes.len); + defer workspace.deinit(); + if (plan.filters.len != 0) { + try workspace.ensure(&session, plan.filters.slice(), result.nodes); + var filter_query = query; + filter_query.metrics = &.{}; + filter_query.order_by = &.{}; + try workspace.select(filter_query, apply_result_limit and plan.orders.len == 0, plan.filters.slice(), plan.orders.slice(), plan.projections.slice()); + } + if (plan.orders.len != 0) { + try workspace.ensure(&session, plan.orders.slice(), result.nodes); + var order_query = query; + order_query.metrics = &.{}; + order_query.where_metric = &.{}; + try workspace.select(order_query, apply_result_limit, plan.orders.slice(), plan.projections.slice(), &.{}); + } + try workspace.ensure(&session, plan.projections.slice(), result.nodes); + + const statuses = try output.alloc(GraphMetricStatus, plan.dependencies.len); + var initialized_statuses: usize = 0; + errdefer { + for (statuses[0..initialized_statuses]) |*status| status.deinit(output); + output.free(statuses); + } + for (session.statuses, statuses) |status, *out| { + out.* = try cloneGraphMetricStatus(output, status); + initialized_statuses += 1; + } + const metric_value_names = try output.alloc([]u8, plan.projections.len); + var initialized_metric_names: usize = 0; + errdefer { + for (metric_value_names[0..initialized_metric_names]) |name| output.free(name); + output.free(metric_value_names); + } + for (plan.projections.slice(), metric_value_names) |name, *out| { + out.* = try output.dupe(u8, name); + initialized_metric_names += 1; + } + var columns: [graph_metric_dependency_limit][]?f64 = undefined; + workspace.columnsFor(plan.projections.slice(), columns[0..plan.projections.len]); + const slab = try materializeSelectedMetricColumnsWithAllocators( + self.alloc, + output, + scratch, + metric_value_names, + columns[0..plan.projections.len], + workspace.rows, + &result.nodes, + ); + if (result.metric_values_slab.len > 0) self.alloc.free(result.metric_values_slab); + for (result.metric_value_names) |name| self.alloc.free(name); + if (result.metric_value_names.len > 0) self.alloc.free(result.metric_value_names); + result.metric_values_slab = slab; + result.metric_value_names = metric_value_names; + for (result.metric_status) |*status| status.deinit(self.alloc); + if (result.metric_status.len > 0) self.alloc.free(result.metric_status); + result.metric_status = statuses; + } + + /// A stable source-row selection and only the columns needed by future + /// stages. The reader is snapshot-owned; this executor never opens storage. + pub const MetricStageWorkspace = struct { + alloc: Allocator, + plan: MetricReadPlan, + rows: []usize, + columns: [graph_metric_dependency_limit]?[]?f64 = @splat(null), + + pub fn init(alloc: Allocator, plan: MetricReadPlan, node_count: usize) !@This() { + const rows = try alloc.alloc(usize, node_count); + for (rows, 0..) |*row, i| row.* = i; + return .{ .alloc = alloc, .plan = plan, .rows = rows }; + } + + pub fn deinit(self: *@This()) void { + for (self.columns) |column| if (column) |scores| self.alloc.free(scores); + self.alloc.free(self.rows); + } + + fn index(self: *const @This(), name: []const u8) usize { + return metricColumnNameIndex(self.plan.dependencies.slice(), name).?; + } + + pub fn columnsFor(self: *const @This(), names: []const []const u8, out: [][]?f64) void { + for (names, out) |name, *column| column.* = self.columns[self.index(name)].?; + } + + pub fn ensure(self: *@This(), reader: anytype, names: []const []const u8, nodes: []const GraphResultNode) !void { + var missing: MetricReadPlan.Names = .{}; + for (names) |name| if (self.columns[self.index(name)] == null) { + missing.append(name); + }; + if (missing.len == 0) return; + var local_count: usize = 0; + for (self.rows) |row| local_count += @intFromBool(nodes[row].table == null); + const keys = try self.alloc.alloc([]const u8, local_count); + defer self.alloc.free(keys); + var local_index: usize = 0; + for (self.rows) |row| if (nodes[row].table == null) { + keys[local_index] = nodes[row].key; + local_index += 1; + }; + var columns: [graph_metric_dependency_limit][]?f64 = undefined; + var local_columns: [graph_metric_dependency_limit][]?f64 = undefined; + var initialized: usize = 0; + errdefer for (columns[0..initialized]) |column| self.alloc.free(column); + for (columns[0..missing.len]) |*column| { + column.* = try self.alloc.alloc(?f64, self.rows.len); + local_columns[initialized] = column.*[0..local_count]; + initialized += 1; + } + try reader.readColumns(self.alloc, missing.slice(), keys, local_columns[0..missing.len]); + // Expand backwards in-place: qualified identities must not alias a + // local document with the same key. No second score slab is needed. + if (local_count != self.rows.len) for (columns[0..missing.len]) |column| { + var source = local_count; + var target = self.rows.len; + while (target != 0) { + target -= 1; + if (nodes[self.rows[target]].table == null) { + source -= 1; + column[target] = column[source]; + } else column[target] = null; + } + }; + for (missing.slice(), columns[0..missing.len]) |name, column| self.columns[self.index(name)] = column; + } + + pub fn select(self: *@This(), query: GraphQuery, apply_limit: bool, names: []const []const u8, future: []const []const u8, later: []const []const u8) !void { + var columns: [graph_metric_dependency_limit][]?f64 = undefined; + self.columnsFor(names, columns[0..names.len]); + const selected = try selectLoadedMetricCandidateIndexesAlloc(self.alloc, names, columns[0..names.len], query, apply_limit, self.rows.len); + defer self.alloc.free(selected); + const rows = try self.alloc.alloc(usize, selected.len); + errdefer self.alloc.free(rows); + for (selected, rows) |parent, *row| row.* = self.rows[parent]; + var replacements: [graph_metric_dependency_limit]?[]?f64 = @splat(null); + errdefer for (replacements) |column| if (column) |scores| self.alloc.free(scores); + for (self.plan.dependencies.slice(), self.columns[0..self.plan.dependencies.len], 0..) |name, *maybe_column, i| { + const column = maybe_column.* orelse continue; + if (metricColumnNameIndex(future, name) == null and metricColumnNameIndex(later, name) == null) { + self.alloc.free(column); + maybe_column.* = null; + continue; + } + const rebased = try self.alloc.alloc(?f64, selected.len); + for (selected, rebased) |parent, *value| value.* = column[parent]; + replacements[i] = rebased; + } + for (&self.columns, replacements) |*column, replacement| if (replacement) |scores| { + self.alloc.free(column.*.?); + column.* = scores; + }; + self.alloc.free(self.rows); + self.rows = rows; + } + }; + fn executeTraverse( self: *GraphQueryEngine, graph_index: *graph_mod.GraphIndex, @@ -826,11 +1643,12 @@ pub const GraphQueryEngine = struct { defer if (admitted_starts) |mask| self.alloc.free(mask); if (self.node_admission != null and admitted_starts == null) return null; - for (start_keys, 0..) |start_key, start_index| { + outer: for (start_keys, 0..) |start_key, start_index| { if (admitted_starts) |mask| if (!mask[start_index]) continue; for (target_keys) |target_key| { if (std.mem.eql(u8, start_key, target_key)) { try all_results.append(self.alloc, try trivialPathResultNode(self.alloc, start_key)); + if (params.max_results > 0 and all_results.items.len >= params.max_results) break :outer; continue; } @@ -863,6 +1681,7 @@ pub const GraphQueryEngine = struct { if (reached.len != 1) return null; const node = (try algebraicShortestPathResultNodeAlloc(self.alloc, graph_index, params, start_key, target_key, reached[0])) orelse return null; try all_results.append(self.alloc, node); + if (params.max_results > 0 and all_results.items.len >= params.max_results) break :outer; } } @@ -1952,19 +2771,30 @@ const TestCtx = struct { } }; -fn setupGraph(alloc: Allocator, store_label: []const u8, rev_label: []const u8, sb: *[256]u8, rb: *[256]u8) !*TestCtx { +fn setupGraphWithOptions( + alloc: Allocator, + store_label: []const u8, + rev_label: []const u8, + sb: *[256]u8, + rb: *[256]u8, + opts: graph_mod.GraphIndexOptions, +) !*TestCtx { const sp = tmpPath(sb, store_label); const rp = tmpPath(rb, rev_label); const ctx = try alloc.create(TestCtx); errdefer alloc.destroy(ctx); ctx.store = try docstore.DocStore.open(alloc, sp, .{}); errdefer ctx.store.close(); - ctx.graph = try graph_mod.GraphIndex.open(alloc, &ctx.store, rp, "test", .{}); + ctx.graph = try graph_mod.GraphIndex.open(alloc, &ctx.store, rp, "test", opts); ctx.sp = sp; ctx.rp = rp; return ctx; } +fn setupGraph(alloc: Allocator, store_label: []const u8, rev_label: []const u8, sb: *[256]u8, rb: *[256]u8) !*TestCtx { + return try setupGraphWithOptions(alloc, store_label, rev_label, sb, rb, .{}); +} + fn expectAlgebraicTraversalReject(proof: AlgebraicTraversalProof, reason: AlgebraicTraversalRejectReason) !void { switch (proof) { .proven => return error.TestExpectedEqual, @@ -2890,6 +3720,499 @@ test "shortest_path algebraic provenance applies exact edge weight filters" { try std.testing.expectEqualStrings("E\x1fok\x1fD", provenance[2]); } +test "graph metric query shape bounds clauses and unique dependencies" { + const names = [_][]const u8{ + "m00", "m01", "m02", "m03", "m04", "m05", "m06", "m07", "m08", + "m09", "m10", "m11", "m12", "m13", "m14", "m15", "m16", + }; + var reads: [graph_metric_projection_limit + 1]GraphMetricRead = undefined; + for (&reads, names) |*read, name| read.* = .{ .name = name }; + + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &reads, + })); + + var too_many_orders: [graph_metric_order_limit + 1]GraphMetricOrder = undefined; + for (&too_many_orders) |*order| order.* = .{ .name = "pagerank" }; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &too_many_orders, + })); + + var too_many_filters: [graph_metric_filter_limit + 1]GraphMetricFilter = undefined; + for (&too_many_filters) |*filter| filter.* = .{ .name = "pagerank", .op = .gte, .value = 0.1 }; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .where_metric = &too_many_filters, + })); + + const extra_order = [_]GraphMetricOrder{.{ .name = names[graph_metric_dependency_limit] }}; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = reads[0..graph_metric_dependency_limit], + .order_by = &extra_order, + })); + + const duplicate_reads = [_]GraphMetricRead{ .{ .name = "pagerank" }, .{ .name = "pagerank" } }; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .metrics = &duplicate_reads, + })); + + const duplicate_orders = [_]GraphMetricOrder{ .{ .name = "pagerank" }, .{ .name = "pagerank", .direction = .asc } }; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .order_by = &duplicate_orders, + })); + + const range_filters = [_]GraphMetricFilter{ + .{ .name = "pagerank", .op = .gte, .value = 0.1 }, + .{ .name = "pagerank", .op = .lt, .value = 0.9 }, + }; + try validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .where_metric = &range_filters, + }); + + const invalid_filters = [_]GraphMetricFilter{.{ + .name = "pagerank", + .op = .gte, + .value = std.math.nan(f64), + }}; + try std.testing.expectError(error.InvalidQueryRequest, validateGraphMetricQueryShape(.{ + .query_type = .traverse, + .index_name = "g", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .where_metric = &invalid_filters, + })); +} + +test "borrowed graph metric names do not allocate per node" { + var value = GraphMetricValue{ .name = "pagerank", .score = 0.5, .name_owned = false }; + try value.ensureNameOwned(std.testing.allocator); + try std.testing.expect(value.name_owned); + try std.testing.expectEqualStrings("pagerank", value.name); + value.deinit(std.testing.allocator); +} + +test "graph metric column selection retains deterministic bounded top k" { + var scores = [_]?f64{ 0.3, null, 0.9, 0.8, 0.9, 0.1 }; + const columns = [_][]?f64{&scores}; + const orders = [_]GraphMetricOrder{.{ + .name = "rank", + .direction = .desc, + .nulls = .last, + }}; + const metric_indexes = [_]usize{0}; + const context = GraphQueryEngine.MetricColumnSortContext{ + .orders = &orders, + .metric_indexes = &metric_indexes, + .score_columns = &columns, + }; + var candidates = [_]usize{ 0, 1, 2, 3, 4, 5 }; + const selected = GraphQueryEngine.retainOrderedMetricCandidatePrefix(&candidates, 3, context); + try std.testing.expectEqualSlices(usize, &.{ 2, 4, 3 }, selected); + + var ascending_candidates = [_]usize{ 0, 1, 2, 3, 4, 5 }; + const ascending_orders = [_]GraphMetricOrder{.{ + .name = "rank", + .direction = .asc, + .nulls = .first, + }}; + const ascending = GraphQueryEngine.retainOrderedMetricCandidatePrefix( + &ascending_candidates, + 2, + .{ + .orders = &ascending_orders, + .metric_indexes = &metric_indexes, + .score_columns = &columns, + }, + ); + try std.testing.expectEqualSlices(usize, &.{ 1, 5 }, ascending); +} + +test "graph metric shared column application is allocation-failure safe" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var metric_values_slab: []GraphMetricValue = &.{}; + var nodes = try alloc.alloc(GraphResultNode, 3); + var initialized: usize = 0; + defer { + for (nodes[0..initialized]) |*node| node.deinit(alloc); + alloc.free(nodes); + if (metric_values_slab.len > 0) alloc.free(metric_values_slab); + } + for (&[_][]const u8{ "a", "b", "c" }, 0..) |key, i| { + nodes[i] = .{ .key = try alloc.dupe(u8, key), .depth = 1, .distance = 1 }; + initialized += 1; + } + + var scores = [_]?f64{ 0.1, 0.9, 0.5 }; + const columns = [_][]?f64{&scores}; + const names = [_][]const u8{"rank"}; + const reads = [_]GraphMetricRead{.{ .name = "rank" }}; + const orders = [_]GraphMetricOrder{.{ .name = "rank", .direction = .desc }}; + metric_values_slab = try GraphQueryEngine.applyLoadedMetricColumns( + alloc, + &names, + &names, + &columns, + .{ + .query_type = .neighbors, + .index_name = "g", + .start_nodes = .{ .keys = &.{"root"} }, + .params = .{ .max_results = 2 }, + .metrics = &reads, + .order_by = &orders, + }, + true, + &nodes, + ); + initialized = nodes.len; + try std.testing.expectEqual(@as(usize, 2), nodes.len); + try std.testing.expectEqualStrings("b", nodes[0].key); + try std.testing.expectEqual(@as(?f64, 0.9), nodes[0].metrics[0].score); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "graph metric stable row materialization moves nodes once and is allocation-failure safe" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var metric_values_slab: []GraphMetricValue = &.{}; + var nodes = try alloc.alloc(GraphResultNode, 3); + var initialized: usize = 0; + defer { + for (nodes[0..initialized]) |*node| node.deinit(alloc); + alloc.free(nodes); + if (metric_values_slab.len > 0) alloc.free(metric_values_slab); + } + for (&[_][]const u8{ "a", "b", "c" }, 0..) |key, i| { + nodes[i] = .{ .key = try alloc.dupe(u8, key), .depth = 1, .distance = 1 }; + initialized += 1; + } + + var aligned_scores = [_]?f64{ 0.8, 0.2 }; + const columns = [_][]?f64{&aligned_scores}; + const names = [_][]const u8{"rank"}; + metric_values_slab = try GraphQueryEngine.materializeSelectedMetricColumns( + alloc, + &names, + &columns, + &.{ 2, 0 }, + &nodes, + ); + initialized = nodes.len; + try std.testing.expectEqual(@as(usize, 2), nodes.len); + try std.testing.expectEqualStrings("c", nodes[0].key); + try std.testing.expectEqual(@as(?f64, 0.8), nodes[0].metrics[0].score); + try std.testing.expectEqualStrings("a", nodes[1].key); + try std.testing.expectEqual(@as(?f64, 0.2), nodes[1].metrics[0].score); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "graph metric staged reads only load display columns for selected rows" { + const Runner = struct { + const Reader = struct { + keys: usize = 0, + pub fn readColumns(self: *@This(), _: Allocator, names: []const []const u8, keys: []const []const u8, columns: []const []?f64) !void { + self.keys += names.len * keys.len; + for (columns) |column| for (keys, column) |key, *value| { + value.* = @floatFromInt(key[0] - '0'); + }; + } + }; + fn run(alloc: Allocator) !void { + const query = GraphQuery{ + .query_type = .neighbors, + .index_name = "graph", + .start_nodes = .{ .keys = &.{"0"} }, + .params = .{ .max_results = 2 }, + .metrics = &.{ .{ .name = "rank" }, .{ .name = "display" } }, + .order_by = &.{.{ .name = "rank" }}, + .where_metric = &.{.{ .name = "rank", .op = .gte, .value = 2 }}, + }; + const nodes = [_]GraphResultNode{ + .{ .key = "0", .depth = 0, .distance = 0 }, .{ .key = "1", .depth = 0, .distance = 0 }, .{ .key = "2", .depth = 0, .distance = 0 }, + .{ .key = "3", .depth = 0, .distance = 0 }, .{ .key = "4", .depth = 0, .distance = 0 }, .{ .key = "5", .depth = 0, .distance = 0 }, + }; + const plan = try MetricReadPlan.init(query); + var work = try GraphQueryEngine.MetricStageWorkspace.init(alloc, plan, nodes.len); + defer work.deinit(); + var reader = Reader{}; + try work.ensure(&reader, plan.filters.slice(), &nodes); + var filter = query; + filter.metrics = &.{}; + filter.order_by = &.{}; + try work.select(filter, false, plan.filters.slice(), plan.orders.slice(), plan.projections.slice()); + try work.ensure(&reader, plan.orders.slice(), &nodes); + var order = query; + order.metrics = &.{}; + order.where_metric = &.{}; + try work.select(order, true, plan.orders.slice(), plan.projections.slice(), &.{}); + try work.ensure(&reader, plan.projections.slice(), &nodes); + try std.testing.expectEqual(@as(usize, 8), reader.keys); + try std.testing.expectEqualSlices(usize, &.{ 5, 4 }, work.rows); + for (work.columns[0..2]) |column| try std.testing.expectEqualSlices(?f64, &.{ 5, 4 }, column.?); + } + }; + try Runner.run(std.testing.allocator); + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "graph metric staged columns do not alias qualified node identities" { + const alloc = std.testing.allocator; + const query = GraphQuery{ .query_type = .neighbors, .index_name = "graph", .start_nodes = .{ .keys = &.{} }, .metrics = &.{.{ .name = "rank" }} }; + const nodes = [_]GraphResultNode{ + .{ .key = "same", .table = "other", .depth = 0, .distance = 0 }, + .{ .key = "same", .depth = 0, .distance = 0 }, + .{ .key = "same", .table = "other", .depth = 0, .distance = 0 }, + }; + var work = try GraphQueryEngine.MetricStageWorkspace.init(alloc, try MetricReadPlan.init(query), nodes.len); + defer work.deinit(); + const Reader = struct { + pub fn readColumns(_: *@This(), _: Allocator, _: []const []const u8, keys: []const []const u8, columns: []const []?f64) !void { + try std.testing.expectEqual(@as(usize, 1), keys.len); + columns[0][0] = 7; + } + }; + var reader = Reader{}; + try work.ensure(&reader, &.{"rank"}, &nodes); + try std.testing.expectEqualSlices(?f64, &.{ null, 7, null }, work.columns[0].?); +} + +test "graph metric staged query admits scratch and output and pins publication" { + const alloc = std.testing.allocator; + var sb: [256]u8 = undefined; + var rb: [256]u8 = undefined; + const configs = [_]graph_mod.GraphMetricConfig{.{ .name = "degree", .kind = .degree }}; + const ctx = try setupGraphWithOptions(alloc, "gq-staged-budget-s", "gq-staged-budget-r", &sb, &rb, .{ .metric_configs = &configs }); + defer { + ctx.deinit(); + alloc.destroy(ctx); + } + try ctx.graph.addEdge("A", "B", "e", 1, 0, 0, ""); + var published = try ctx.graph.runDegreeMetric("degree"); + published.deinit(alloc); + var session = try ctx.graph.openGraphMetricReadSession(&.{"degree"}, &.{.{ .require_fresh = true }}); + defer session.deinit(); + var scores: [1]?f64 = undefined; + try session.readColumns(alloc, &.{"degree"}, &.{"A"}, &.{&scores}); + try std.testing.expectEqual(@as(?f64, 1), scores[0]); + try ctx.graph.addEdge("A", "C", "e", 1, 0, 0, ""); + published = try ctx.graph.runDegreeMetric("degree"); + published.deinit(alloc); + try session.readColumns(alloc, &.{"degree"}, &.{"A"}, &.{&scores}); + try std.testing.expectEqual(@as(?f64, 1), scores[0]); + + const Runner = struct { + fn run(out_alloc: Allocator, index: *graph_mod.GraphIndex, maximum: usize) !void { + var result = GraphQueryResult{ .nodes = try out_alloc.alloc(GraphResultNode, 1) }; + result.nodes[0] = .{ .key = out_alloc.dupe(u8, "A") catch |err| { + out_alloc.free(result.nodes); + return err; + }, .depth = 0, .distance = 0 }; + defer result.deinit(out_alloc); + var budget = work_budget_mod.WorkBudget.initWithLimits(.{ .max_retained_state_bytes = maximum }); + var engine = GraphQueryEngine{ .alloc = out_alloc, .work_budget = &budget }; + const query = GraphQuery{ .query_type = .neighbors, .index_name = "graph", .start_nodes = .{ .keys = &.{"A"} }, .metrics = &.{.{ .name = "degree" }} }; + engine.applyMetricDependenciesColumnar(index, query, false, &result) catch |err| { + try std.testing.expectEqual(@as(usize, 0), budget.retained_state_bytes); + return err; + }; + try std.testing.expect(budget.retained_state_bytes > 0 and budget.retained_state_bytes <= maximum); + try std.testing.expectEqual(@as(?f64, 2), result.nodes[0].metrics[0].score); + } + }; + try std.testing.expectError(error.GraphWorkBudgetExceeded, Runner.run(alloc, &ctx.graph, 1)); + for ([_]usize{ 64, 256, 512, 1024, 2048, 4096, 8192 }) |maximum| { + Runner.run(alloc, &ctx.graph, maximum) catch |err| { + try std.testing.expectEqual(error.GraphWorkBudgetExceeded, err); + }; + } + try Runner.run(alloc, &ctx.graph, 64 * 1024); + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{ &ctx.graph, @as(usize, 64 * 1024) }); +} + +test "graph metric order and filter dependencies attach status without projection" { + const alloc = std.testing.allocator; + var sb: [256]u8 = undefined; + var rb: [256]u8 = undefined; + const metrics = [_]graph_mod.GraphMetricConfig{.{ + .name = "degree", + .kind = .degree, + }}; + const ctx = try setupGraphWithOptions(alloc, "gq-metric-deps-s", "gq-metric-deps-r", &sb, &rb, .{ .metric_configs = &metrics }); + defer { + ctx.deinit(); + alloc.destroy(ctx); + } + + try ctx.graph.addEdge("A", "B", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("A", "C", "e", 1.0, 0, 0, ""); + var degree_status = try ctx.graph.runDegreeMetric("degree"); + degree_status.deinit(alloc); + + const metric_orders = [_]GraphMetricOrder{.{ + .name = "degree", + .direction = .desc, + .freshness = .published, + }}; + const metric_filters = [_]GraphMetricFilter{.{ + .name = "degree", + .op = .gte, + .value = 1.0, + .freshness = .published, + }}; + + var engine = GraphQueryEngine{ .alloc = alloc }; + const start_keys: []const []const u8 = &.{"A"}; + var result = try engine.execute(&ctx.graph, .{ + .query_type = .neighbors, + .index_name = "test", + .start_nodes = .{ .keys = start_keys }, + .params = .{ .edge_types = &.{"e"}, .direction = .out, .max_results = 8 }, + .order_by = &metric_orders, + .where_metric = &metric_filters, + }, start_keys); + defer result.deinit(alloc); + + try std.testing.expectEqual(@as(usize, 2), result.nodes.len); + try std.testing.expectEqual(@as(usize, 0), result.nodes[0].metrics.len); + try std.testing.expectEqual(@as(usize, 1), result.metric_status.len); + try std.testing.expectEqualStrings("degree", result.metric_status[0].name); + try std.testing.expect(result.metric_status[0].published_generation != 0); +} + +test "graph metric order and filter apply max results after metric processing" { + const alloc = std.testing.allocator; + var sb: [256]u8 = undefined; + var rb: [256]u8 = undefined; + const metrics = [_]graph_mod.GraphMetricConfig{.{ .name = "degree", .kind = .degree }}; + const ctx = try setupGraphWithOptions(alloc, "gq-metric-limit-s", "gq-metric-limit-r", &sb, &rb, .{ .metric_configs = &metrics }); + defer { + ctx.deinit(); + alloc.destroy(ctx); + } + + try ctx.graph.addEdge("A", "B", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("A", "C", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("A", "D", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "X", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "Y", "e", 1.0, 0, 0, ""); + var degree_status = try ctx.graph.runDegreeMetric("degree"); + degree_status.deinit(alloc); + + const metric_orders = [_]GraphMetricOrder{.{ .name = "degree", .direction = .desc, .freshness = .published }}; + const metric_filters = [_]GraphMetricFilter{.{ .name = "degree", .op = .gte, .value = 3.0, .freshness = .published }}; + var engine = GraphQueryEngine{ .alloc = alloc }; + const start_keys: []const []const u8 = &.{"A"}; + var result = try engine.execute(&ctx.graph, .{ + .query_type = .neighbors, + .index_name = "test", + .start_nodes = .{ .keys = start_keys }, + .params = .{ .edge_types = &.{"e"}, .direction = .out, .max_results = 1 }, + .order_by = &metric_orders, + .where_metric = &metric_filters, + }, start_keys); + defer result.deinit(alloc); + + try std.testing.expectEqual(@as(usize, 1), result.nodes.len); + try std.testing.expectEqualStrings("C", result.nodes[0].key); +} + +test "shortest path metric filtering evaluates the complete bounded candidate set" { + const alloc = std.testing.allocator; + var sb: [256]u8 = undefined; + var rb: [256]u8 = undefined; + const metrics = [_]graph_mod.GraphMetricConfig{.{ .name = "degree", .kind = .degree }}; + const ctx = try setupGraphWithOptions(alloc, "gq-metric-shortest-s", "gq-metric-shortest-r", &sb, &rb, .{ .metric_configs = &metrics }); + defer { + ctx.deinit(); + alloc.destroy(ctx); + } + + try ctx.graph.addEdge("A", "B", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("A", "C", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "X", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "Y", "e", 1.0, 0, 0, ""); + var degree_status = try ctx.graph.runDegreeMetric("degree"); + degree_status.deinit(alloc); + + const filters = [_]GraphMetricFilter{.{ .name = "degree", .op = .gte, .value = 3.0, .freshness = .published }}; + var engine = GraphQueryEngine{ .alloc = alloc }; + const starts: []const []const u8 = &.{"A"}; + var result = try engine.execute(&ctx.graph, .{ + .query_type = .shortest_path, + .index_name = "test", + .start_nodes = .{ .keys = starts }, + .target_nodes = .{ .keys = &.{ "B", "C" } }, + .params = .{ .max_depth = 2, .max_results = 1 }, + .where_metric = &filters, + }, starts); + defer result.deinit(alloc); + + try std.testing.expectEqual(@as(usize, 1), result.nodes.len); + try std.testing.expectEqualStrings("C", result.nodes[0].key); +} + +test "pattern metric filtering evaluates matches beyond the response limit" { + const alloc = std.testing.allocator; + var sb: [256]u8 = undefined; + var rb: [256]u8 = undefined; + const metrics = [_]graph_mod.GraphMetricConfig{.{ .name = "degree", .kind = .degree }}; + const ctx = try setupGraphWithOptions(alloc, "gq-metric-pattern-s", "gq-metric-pattern-r", &sb, &rb, .{ .metric_configs = &metrics }); + defer { + ctx.deinit(); + alloc.destroy(ctx); + } + + try ctx.graph.addEdge("A", "B", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("A", "C", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "X", "e", 1.0, 0, 0, ""); + try ctx.graph.addEdge("C", "Y", "e", 1.0, 0, 0, ""); + var degree_status = try ctx.graph.runDegreeMetric("degree"); + degree_status.deinit(alloc); + + const filters = [_]GraphMetricFilter{.{ .name = "degree", .op = .gte, .value = 3.0, .freshness = .published }}; + const steps: []const pattern_mod.PatternStep = &.{ + .{ .alias = "start", .edge = .{ .direction = .out } }, + .{ .alias = "neighbor", .edge = .{ .direction = .out } }, + }; + var engine = GraphQueryEngine{ .alloc = alloc }; + const starts: []const []const u8 = &.{"A"}; + var result = try engine.execute(&ctx.graph, .{ + .query_type = .pattern, + .index_name = "test", + .start_nodes = .{ .keys = starts }, + .pattern = steps, + .params = .{ .max_results = 1 }, + .where_metric = &filters, + }, starts); + defer result.deinit(alloc); + + try std.testing.expectEqual(@as(usize, 1), result.nodes.len); + try std.testing.expectEqualStrings("C", result.nodes[0].key); +} + test "k_shortest_paths via engine" { const alloc = std.testing.allocator; var sb: [256]u8 = undefined; @@ -2920,6 +4243,17 @@ test "k_shortest_paths via engine" { try std.testing.expectEqual(@as(usize, 2), result.nodes.len); // First path should be shorter/lighter try std.testing.expect(result.nodes[0].distance <= result.nodes[1].distance); + + var limited = try engine.execute(&ctx.graph, .{ + .query_type = .k_shortest_paths, + .index_name = "test", + .start_nodes = .{ .keys = start_keys }, + .target_nodes = .{ .keys = &.{"C"} }, + .k = 2, + .params = .{ .weight_mode = .min_weight, .max_results = 1 }, + }, start_keys); + defer limited.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), limited.nodes.len); } test "path result node conversion preserves endpoint semantics and allocation safety" { diff --git a/zig/pkg/antfly/src/graph/score_read.zig b/zig/pkg/antfly/src/graph/score_read.zig new file mode 100644 index 0000000000..d111c40594 --- /dev/null +++ b/zig/pkg/antfly/src/graph/score_read.zig @@ -0,0 +1,224 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Snapshot-local score reads. Physical identity is the complete encoded +//! metric/generation prefix, never just equivalent configuration. Callers +//! resolve freshness policies before allocating results or entering this layer. +const std = @import("std"); +const keys = @import("../storage/internal_keys.zig"); +pub const max_keys_per_read = 4096; +pub const max_key_bytes_per_read = 1024 * 1024; +pub const Stats = struct { keys: usize = 0, batches: usize = 0 }; + +pub fn populate( + alloc: std.mem.Allocator, + txn: anytype, + prefixes: []const ?[]const u8, + nodes: []const []const u8, + columns: []const []?f64, +) !Stats { + std.debug.assert(prefixes.len == columns.len); + for (columns) |column| { + std.debug.assert(column.len == nodes.len); + @memset(column, null); + } + if (nodes.len == 0) return .{}; + const physical = try alloc.alloc(usize, prefixes.len); + defer alloc.free(physical); + var physical_count: usize = 0; + for (prefixes, 0..) |prefix, i| if (prefix != null) { + physical[physical_count] = i; + physical_count += 1; + }; + if (physical_count == 0) return .{}; + const Order = struct { + prefixes: []const ?[]const u8, + fn less(self: @This(), a: usize, b: usize) bool { + const order = std.mem.order(u8, self.prefixes[a].?, self.prefixes[b].?); + return order == .lt or (order == .eq and a < b); + } + }; + std.mem.sort(usize, physical[0..physical_count], Order{ .prefixes = prefixes }, Order.less); + // Retain the sorted logical columns for the final independently-owned + // result fanout; deduplicate physical columns in a separate compact list. + const owners = try alloc.alloc(usize, physical_count); + defer alloc.free(owners); + var owner_count: usize = 0; + for (physical[0..physical_count]) |column| { + if (owner_count != 0 and std.mem.eql(u8, prefixes[owners[owner_count - 1]].?, prefixes[column].?)) continue; + owners[owner_count] = column; + owner_count += 1; + } + const rows = try alloc.alloc(usize, nodes.len); + defer alloc.free(rows); + for (rows, 0..) |*row, i| row.* = i; + const RowOrder = struct { + nodes: []const []const u8, + fn less(self: @This(), a: usize, b: usize) bool { + const order = std.mem.order(u8, self.nodes[a], self.nodes[b]); + return order == .lt or (order == .eq and a < b); + } + }; + std.mem.sort(usize, rows, RowOrder{ .nodes = nodes }, RowOrder.less); + var unique_count: usize = 1; + for (rows[1..], rows[0 .. rows.len - 1]) |row, prior| { + if (!std.mem.eql(u8, nodes[row], nodes[prior])) unique_count += 1; + } + const unique = if (unique_count == rows.len) rows else try alloc.alloc(usize, unique_count); + defer if (unique_count != rows.len) alloc.free(unique); + if (unique_count != rows.len) { + var i: usize = 0; + for (rows) |row| { + if (i != 0 and std.mem.eql(u8, nodes[unique[i - 1]], nodes[row])) continue; + unique[i] = row; + i += 1; + } + } + const total = std.math.mul(usize, owner_count, unique_count) catch return error.GraphMetricQueryBudgetExceeded; + const batch_capacity = @min(total, max_keys_per_read); + const read_keys = try alloc.alloc([]const u8, batch_capacity); + defer alloc.free(read_keys); + const read_values = try alloc.alloc(?[]const u8, batch_capacity); + defer alloc.free(read_values); + var key_bytes = std.ArrayListUnmanaged(u8).empty; + defer key_bytes.deinit(alloc); + var stats = Stats{ .keys = total }; + var offset: usize = 0; + while (offset < total) { + var len: usize = 0; + var bytes: usize = 0; + for (0..@min(batch_capacity, total - offset)) |i| { + const flat = offset + i; + const prefix = prefixes[owners[flat / unique_count]].?; + const node = nodes[unique[flat % unique_count]]; + const key_len = std.math.add(usize, prefix.len, keys.encodedComponentLen(node)) catch return error.GraphMetricQueryBudgetExceeded; + // One oversized key may progress, subject to the caller's live + // allocation budget. Never multiply long IDs by the key-count cap. + if (len != 0 and key_len > max_key_bytes_per_read -| bytes) break; + bytes = std.math.add(usize, bytes, key_len) catch return error.GraphMetricQueryBudgetExceeded; + len += 1; + } + key_bytes.clearRetainingCapacity(); + try key_bytes.ensureTotalCapacity(alloc, bytes); + for (read_keys[0..len], 0..) |*key, i| { + const flat = offset + i; + const start = key_bytes.items.len; + try key_bytes.appendSlice(alloc, prefixes[owners[flat / unique_count]].?); + try keys.appendEncodedComponent(&key_bytes, alloc, nodes[unique[flat % unique_count]]); + key.* = key_bytes.items[start..]; + } + @memset(read_values[0..len], null); + try txn.getManySorted(read_keys[0..len], read_values[0..len]); + for (read_values[0..len], 0..) |maybe_raw, i| { + const raw = maybe_raw orelse continue; + if (raw.len != 8) return error.InvalidGraphMetricScore; + const score: f64 = @bitCast(std.mem.readInt(u64, raw[0..8], .little)); + if (!std.math.isFinite(score)) return error.InvalidGraphMetricScore; + const flat = offset + i; + columns[owners[flat / unique_count]][unique[flat % unique_count]] = score; + } + offset += len; + stats.batches += 1; + } + if (unique_count != rows.len) for (owners[0..owner_count]) |owner| { + var representative = rows[0]; + for (rows[1..]) |row| { + if (std.mem.eql(u8, nodes[row], nodes[representative])) { + columns[owner][row] = columns[owner][representative]; + } else representative = row; + } + }; + var owner = physical[0]; + for (physical[1..physical_count]) |column| { + if (std.mem.eql(u8, prefixes[owner].?, prefixes[column].?)) { + @memcpy(columns[column], columns[owner]); + } else owner = column; + } + return stats; +} + +test "graph metric physical score reads bound encoded bytes as well as key count" { + const alloc = std.testing.allocator; + const buffers = try alloc.alloc([4096]u8, 600); + defer alloc.free(buffers); + const nodes = try alloc.alloc([]const u8, buffers.len); + defer alloc.free(nodes); + for (buffers, nodes, 0..) |*buffer, *node, i| { + @memset(buffer, 'x'); + _ = try std.fmt.bufPrint(buffer[4090..], "{d:0>6}", .{i}); + node.* = buffer; + } + const column = try alloc.alloc(?f64, nodes.len); + defer alloc.free(column); + const Txn = struct { + raw: [8]u8 = @bitCast(@as(f64, 3)), + pub fn getManySorted(self: *@This(), requested: []const []const u8, values: []?[]const u8) !void { + var bytes: usize = 0; + for (requested, values) |key, *value| { + bytes += key.len; + value.* = &self.raw; + } + try std.testing.expect(bytes <= max_key_bytes_per_read or requested.len == 1); + } + }; + var txn = Txn{}; + const result = try populate(alloc, &txn, &.{"prefix/"}, nodes, &.{column}); + try std.testing.expectEqual(@as(usize, 3), result.batches); + try std.testing.expectEqual(nodes.len, result.keys); + for (column) |value| try std.testing.expectEqual(@as(?f64, 3), value); +} + +test "graph metric physical score reads deduplicate keys and retain logical ownership" { + const Runner = struct { + const Txn = struct { + raw: [8]u8 = @bitCast(@as(f64, 1.5)), + seen: usize = 0, + pub fn getManySorted(self: *@This(), requested: []const []const u8, values: []?[]const u8) !void { + for (requested, 0..) |key, i| { + if (i > 0) try std.testing.expect(std.mem.order(u8, requested[i - 1], key) == .lt); + values[i] = if (std.mem.endsWith(u8, key, "missing\x00\x00")) null else &self.raw; + } + self.seen += requested.len; + } + }; + fn run(alloc: std.mem.Allocator) !void { + const nodes = [_][]const u8{ "z", "a\x00b", "z", "missing", "" }; + const prefixes = [_]?[]const u8{ "b\x00\x00", "a\x00\x00", "b\x00\x00", null }; + var columns: [prefixes.len][]?f64 = undefined; + var initialized: usize = 0; + defer for (columns[0..initialized]) |column| alloc.free(column); + for (&columns) |*column| { + column.* = try alloc.alloc(?f64, nodes.len); + initialized += 1; + } + var txn = Txn{}; + const stats = try populate(alloc, &txn, &prefixes, &nodes, &columns); + try std.testing.expectEqual(@as(usize, 8), stats.keys); + try std.testing.expectEqual(stats.keys, txn.seen); + for (columns[0..3]) |column| try std.testing.expectEqualSlices(?f64, &.{ 1.5, 1.5, 1.5, null, 1.5 }, column); + for (columns[3]) |score| try std.testing.expectEqual(@as(?f64, null), score); + columns[0][0] = 9; + try std.testing.expectEqual(@as(?f64, 1.5), columns[2][0]); + txn.raw = @bitCast(std.math.inf(f64)); + _ = populate(alloc, &txn, &prefixes, &nodes, &columns) catch |err| { + if (err == error.OutOfMemory) return err; + try std.testing.expectEqual(error.InvalidGraphMetricScore, err); + return; + }; + return error.TestUnexpectedResult; + } + }; + try Runner.run(std.testing.allocator); + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} diff --git a/zig/pkg/antfly/src/graph/sealed_vector_cache.zig b/zig/pkg/antfly/src/graph/sealed_vector_cache.zig new file mode 100644 index 0000000000..03dcb39b08 --- /dev/null +++ b/zig/pkg/antfly/src/graph/sealed_vector_cache.zig @@ -0,0 +1,292 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Index-owned LRU of sealed numeric vector chunks. Only consumers past the +//! producer barrier may use this cache. Keys hash the complete persisted key +//! (metric, job, lane, iteration, chunk); separate indexes never share entries. +//! Values are owned copies, never transaction/cursor memory. A caller copies +//! a hit under the lock, so eviction cannot invalidate an active checkpoint. +const std = @import("std"); +const vector = @import("vector_chunk.zig"); +const Allocator = std.mem.Allocator; + +/// Shared admission across indexes, including allocation overhead. Exhaustion +/// is a storage-read fallback, never a failed build. Hosts may supply a smaller +/// budget; the default process-wide pool prevents per-index multiplication. +pub const Budget = struct { + /// Configure before sharing with caches; immutable while they are active. + limit: usize = 64 * 1024 * 1024, + used: std.atomic.Value(usize) = .init(0), + + fn reserve(self: *Budget, bytes: usize) bool { + var used = self.used.load(.monotonic); + while (bytes <= self.limit and used <= self.limit - bytes) { + used = self.used.cmpxchgWeak(used, used + bytes, .monotonic, .monotonic) orelse return true; + } + return false; + } + + fn release(self: *Budget, bytes: usize) void { + const prior = self.used.fetchSub(bytes, .monotonic); + std.debug.assert(prior >= bytes); + } +}; + +var process_budget = Budget{}; + +pub const Cache = struct { + pub const default_capacity = 4096; // 8.125 MiB of payload; allocated lazily. + const Entry = struct { + key: [32]u8, + data: vector.Chunk, + scope: [32]u8, + hash_next: ?*Entry = null, + older: ?*Entry = null, + newer: ?*Entry = null, + }; + + mu: std.atomic.Mutex = .unlocked, + // Lazily allocated fixed buckets: no unaccounted hash-table growth. + buckets: ?[]?*Entry = null, + count: usize = 0, + budget: *Budget = &process_budget, + generation: u64 = 0, + oldest: ?*Entry = null, + newest: ?*Entry = null, + capacity: usize = default_capacity, + hits: u64 = 0, + misses: u64 = 0, + + pub fn key(persisted_key: []const u8) [32]u8 { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(persisted_key, &digest, .{}); + return digest; + } + + fn lock(self: *Cache) void { + @import("antfly_platform").sync.lockYielding(&self.mu); + } + + pub fn copy(self: *Cache, id: [32]u8, out: *vector.Chunk) bool { + self.lock(); + defer self.mu.unlock(); + const entry = self.find(id) orelse { + self.misses +|= 1; + return false; + }; + self.hits +|= 1; + self.unlink(entry); + self.append(entry); + out.* = entry.data; + return true; + } + + /// Cache admission is optional: allocation failure must not fail a build. + /// Missing chunks are never cached. Producers can populate partial chunks + /// before the barrier; this API must only see a sealed source lane. + pub fn put(self: *Cache, alloc: Allocator, id: [32]u8, bytes: []const u8) void { + self.putAt(alloc, id, bytes, @splat(0), self.ticket()); + } + + pub fn ticket(self: *Cache) u64 { + self.lock(); + defer self.mu.unlock(); + return self.generation; + } + + /// A retired checkpoint may still finish a read, but cannot readmit data. + pub fn putAt(self: *Cache, alloc: Allocator, id: [32]u8, bytes: []const u8, scope: [32]u8, generation: u64) void { + if (bytes.len != vector.encoded_len or self.capacity == 0) return; + self.lock(); + defer self.mu.unlock(); + if (generation != self.generation or self.find(id) != null) return; + if (self.buckets == null) { + const size = 1024 * @sizeOf(?*Entry); + if (!self.budget.reserve(size)) return; + self.buckets = alloc.alloc(?*Entry, 1024) catch { + self.budget.release(size); + return; + }; + @memset(self.buckets.?, null); + } + const entry = if (self.count == self.capacity) blk: { + const victim = self.oldest.?; + self.removeHash(victim); + self.unlink(victim); + break :blk victim; + } else blk: { + if (!self.budget.reserve(@sizeOf(Entry))) { + // A full shared pool must not freeze an existing index's + // working set. Recycle locally without growing admission. + if (self.oldest) |victim| { + self.removeHash(victim); + self.unlink(victim); + break :blk victim; + } + self.releaseEmptyBuckets(alloc); + return; + } + const created = alloc.create(Entry) catch { + self.budget.release(@sizeOf(Entry)); + self.releaseEmptyBuckets(alloc); + return; + }; + self.count += 1; + break :blk created; + }; + entry.* = .{ .key = id, .scope = scope, .data = bytes[0..vector.encoded_len].* }; + const slot = &self.buckets.?[bucket(id)]; + entry.hash_next = slot.*; + slot.* = entry; + self.append(entry); + } + + fn bucket(id: [32]u8) usize { + return std.mem.readInt(u64, id[0..8], .little) % 1024; + } + + fn find(self: *Cache, id: [32]u8) ?*Entry { + var next = (self.buckets orelse return null)[bucket(id)]; + while (next) |entry| : (next = entry.hash_next) { + if (std.mem.eql(u8, &entry.key, &id)) return entry; + } + return null; + } + + fn removeHash(self: *Cache, entry: *Entry) void { + var link = &self.buckets.?[bucket(entry.key)]; + while (link.*.? != entry) link = &link.*.?.hash_next; + link.* = entry.hash_next; + } + + fn releaseEmptyBuckets(self: *Cache, alloc: Allocator) void { + if (self.count != 0) return; + if (self.buckets) |buckets| { + alloc.free(buckets); + self.budget.release(buckets.len * @sizeOf(?*Entry)); + self.buckets = null; + } + } + + pub fn retire(self: *Cache, alloc: Allocator, scope: [32]u8) void { + self.lock(); + defer self.mu.unlock(); + self.generation +%= 1; + var next = self.oldest; + while (next) |entry| { + next = entry.newer; + if (!std.mem.eql(u8, &entry.scope, &scope)) continue; + self.removeHash(entry); + self.unlink(entry); + alloc.destroy(entry); + self.budget.release(@sizeOf(Entry)); + self.count -= 1; + } + self.releaseEmptyBuckets(alloc); + } + + fn unlink(self: *Cache, entry: *Entry) void { + if (entry.older) |older| older.newer = entry.newer else self.oldest = entry.newer; + if (entry.newer) |newer| newer.older = entry.older else self.newest = entry.older; + } + + fn append(self: *Cache, entry: *Entry) void { + entry.older = self.newest; + entry.newer = null; + if (self.newest) |newest| newest.newer = entry else self.oldest = entry; + self.newest = entry; + } + + pub fn deinit(self: *Cache, alloc: Allocator) void { + var next = self.oldest; + while (next) |entry| { + next = entry.newer; + alloc.destroy(entry); + self.budget.release(@sizeOf(Entry)); + } + self.count = 0; + self.releaseEmptyBuckets(alloc); + self.* = .{ .capacity = self.capacity, .budget = self.budget }; + } +}; + +test "graph metric vector chunks sealed cache owns bytes isolates epochs and evicts least recent" { + const alloc = std.testing.allocator; + var cache = Cache{ .capacity = 2 }; + defer cache.deinit(alloc); + var source: vector.Chunk = @splat(0); + try vector.put(&source, 0, 0.5); + const a = Cache.key("metric/job1/rank/0/1"); + const b = Cache.key("metric/job1/rank/0/2"); + const c = Cache.key("metric/job1/rank/1/1"); + cache.put(alloc, a, &source); + cache.put(alloc, b, &source); + try vector.put(&source, 0, 0.75); + var read: vector.Chunk = undefined; + try std.testing.expect(cache.copy(a, &read)); + try std.testing.expectEqual(0.5, try vector.get(&read, 0, true)); + cache.put(alloc, c, &source); + try std.testing.expect(!cache.copy(b, &read)); + try std.testing.expect(cache.copy(c, &read)); + try std.testing.expectEqual(0.75, try vector.get(&read, 0, true)); + try std.testing.expect(!cache.copy(Cache.key("metric/job2/rank/1/1"), &read)); + try std.testing.expectEqual(@as(usize, 2), cache.count); +} + +test "graph metric vector chunks shared budget retires scopes and fences late admission" { + const alloc = std.testing.allocator; + var budget = Budget{ .limit = 2 * 1024 * @sizeOf(?*Cache.Entry) + 3 * @sizeOf(Cache.Entry) }; + var a = Cache{ .budget = &budget }; + defer a.deinit(alloc); + var b = Cache{ .budget = &budget }; + defer b.deinit(alloc); + const bytes: vector.Chunk = @splat(0); + const scope = Cache.key("metric/job"); + const other = Cache.key("other/job"); + const ticket = a.ticket(); + a.putAt(alloc, Cache.key("a"), &bytes, scope, ticket); + a.putAt(alloc, Cache.key("b"), &bytes, other, ticket); + b.put(alloc, Cache.key("c"), &bytes); + try std.testing.expectEqual(budget.limit, budget.used.load(.monotonic)); + a.putAt(alloc, Cache.key("d"), &bytes, scope, ticket); + try std.testing.expectEqual(@as(usize, 2), a.count); + try std.testing.expectEqual(budget.limit, budget.used.load(.monotonic)); + a.retire(alloc, scope); + try std.testing.expectEqual(@as(usize, 1), a.count); + a.putAt(alloc, Cache.key("late"), &bytes, scope, ticket); + try std.testing.expectEqual(@as(usize, 1), a.count); + var copy: vector.Chunk = undefined; + try std.testing.expect(a.copy(Cache.key("b"), ©)); + a.retire(alloc, other); + try std.testing.expect(a.buckets == null); + b.retire(alloc, @splat(0)); + try std.testing.expectEqual(@as(usize, 0), budget.used.load(.monotonic)); + budget.limit = 1; + a.put(alloc, Cache.key("no room"), &bytes); + try std.testing.expectEqual(@as(usize, 0), budget.used.load(.monotonic)); +} + +test "graph metric vector chunks failed optional allocations release shared admission" { + for (0..2) |fail_index| { + var budget = Budget{}; + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = fail_index }); + var cache = Cache{ .budget = &budget }; + defer cache.deinit(failing.allocator()); + const bytes: vector.Chunk = @splat(0); + cache.put(failing.allocator(), Cache.key("failed"), &bytes); + try std.testing.expectEqual(@as(usize, 0), cache.count); + try std.testing.expect(cache.buckets == null); + try std.testing.expectEqual(@as(usize, 0), budget.used.load(.monotonic)); + } +} diff --git a/zig/pkg/antfly/src/graph/topology_owner.zig b/zig/pkg/antfly/src/graph/topology_owner.zig new file mode 100644 index 0000000000..d466bb0f84 --- /dev/null +++ b/zig/pkg/antfly/src/graph/topology_owner.zig @@ -0,0 +1,151 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Durable immutable topology ownership. Numerical jobs hold explicit pins; +//! neither their page attempt numbers nor their cleanup namespaces own tiles. +const std = @import("std"); +const Allocator = std.mem.Allocator; +pub const epoch: u64 = 1; +pub const catalog_prefix = "meta:metric_topology:owners/"; +pub const gc_cursor_key = "meta:metric_topology:gc-cursor"; +pub const Id = [64]u8; +pub const Digest = [32]u8; +pub const State = enum(u8) { building, sealed, deleting }; + +pub const Record = struct { + // The envelope stays decodable across topology format epochs so obsolete + // owners can be reclaimed without opening their membership or tile data. + format_epoch: u64 = epoch, + state: State = .building, + generation: u64, + filter: Digest, + identity: Digest, + bidirectional: bool, + + pub fn encode(self: @This()) [114]u8 { + var raw: [114]u8 = @splat(0); + std.mem.writeInt(u64, raw[0..8], self.format_epoch, .little); + raw[8] = @intFromEnum(self.state); + raw[9] = @intFromBool(self.bidirectional); + std.mem.writeInt(u64, raw[10..18], self.generation, .little); + @memcpy(raw[18..50], &self.filter); + @memcpy(raw[50..82], &self.identity); + std.crypto.hash.sha2.Sha256.hash(raw[0..82], raw[82..114], .{}); + return raw; + } + + pub fn decode(raw: []const u8) !@This() { + if (raw.len != 114 or std.mem.readInt(u64, raw[0..8], .little) == 0 or raw[9] > 1) + return error.InvalidGraphMetricBuildManifest; + var checksum: Digest = undefined; + std.crypto.hash.sha2.Sha256.hash(raw[0..82], &checksum, .{}); + if (!std.mem.eql(u8, &checksum, raw[82..114])) return error.InvalidGraphMetricBuildManifest; + return .{ + .format_epoch = std.mem.readInt(u64, raw[0..8], .little), + .state = std.enums.fromInt(State, raw[8]) orelse return error.InvalidGraphMetricBuildManifest, + .generation = std.mem.readInt(u64, raw[10..18], .little), + .filter = raw[18..50].*, + .identity = raw[50..82].*, + .bidirectional = raw[9] != 0, + }; + } +}; + +pub const Binding = struct { + id: Id, + adopted: bool, + + pub fn encode(self: @This()) [97]u8 { + var raw: [97]u8 = undefined; + raw[0] = @intFromBool(self.adopted); + @memcpy(raw[1..65], &self.id); + std.crypto.hash.sha2.Sha256.hash(raw[0..65], raw[65..97], .{}); + return raw; + } + + pub fn decode(raw: []const u8) !@This() { + if (raw.len != 97 or raw[0] > 1) return error.InvalidGraphMetricBuildManifest; + var checksum: Digest = undefined; + std.crypto.hash.sha2.Sha256.hash(raw[0..65], &checksum, .{}); + if (!std.mem.eql(u8, &checksum, raw[65..97])) return error.InvalidGraphMetricBuildManifest; + for (raw[1..65]) |c| if (!std.ascii.isHex(c)) return error.InvalidGraphMetricBuildManifest; + return .{ .adopted = raw[0] != 0, .id = raw[1..65].* }; + } +}; + +pub fn filterDigest(alloc: Allocator, filter: anytype) !Digest { + const types = try alloc.dupe([]const u8, filter.types); + defer alloc.free(types); + std.mem.sort([]const u8, types, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update(@tagName(filter.mode)); + for (types) |name| { + var len: [8]u8 = undefined; + std.mem.writeInt(u64, &len, name.len, .little); + hash.update(&len); + hash.update(name); + } + return hash.finalResult(); +} + +pub fn identity(filter: Digest, partition_plan: []const u8) Digest { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + var version: [8]u8 = undefined; + std.mem.writeInt(u64, &version, epoch, .little); + hash.update(&version); + hash.update(&filter); + hash.update(partition_plan); + return hash.finalResult(); +} + +pub fn ownerId(digest: Digest, producer_namespace: []const u8, score_generation: u64) Id { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update(&digest); + hash.update(producer_namespace); + var generation: [8]u8 = undefined; + std.mem.writeInt(u64, &generation, score_generation, .little); + hash.update(&generation); + return std.fmt.bytesToHex(hash.finalResult(), .lower); +} + +pub fn catalogKey(alloc: Allocator, id: Id) ![]u8 { + return std.fmt.allocPrint(alloc, "{s}{s}", .{ catalog_prefix, id }); +} + +pub fn dataPrefix(alloc: Allocator, id: Id) ![]u8 { + return std.fmt.allocPrint(alloc, "meta:metric_topology:data/{s}/", .{id}); +} + +pub fn pinsPrefix(alloc: Allocator, id: Id) ![]u8 { + return std.fmt.allocPrint(alloc, "meta:metric_topology:pins/{s}/", .{id}); +} + +pub fn readyKey(alloc: Allocator, digest: Digest, bidirectional: bool) ![]u8 { + return std.fmt.allocPrint(alloc, "meta:metric_topology:ready/{s}/{d}", .{ std.fmt.bytesToHex(digest, .lower), @intFromBool(bidirectional) }); +} + +test "topology receipts reject corruption and binding traversal" { + const record = Record{ .generation = 7, .filter = @splat(1), .identity = @splat(2), .bidirectional = true }; + var raw = record.encode(); + try std.testing.expectEqualDeep(record, try Record.decode(&raw)); + raw[18] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, Record.decode(&raw)); + var binding = (Binding{ .id = @splat('a'), .adopted = true }).encode(); + binding[12] = '/'; + try std.testing.expectError(error.InvalidGraphMetricBuildManifest, Binding.decode(&binding)); +} diff --git a/zig/pkg/antfly/src/graph/typed_edges.zig b/zig/pkg/antfly/src/graph/typed_edges.zig new file mode 100644 index 0000000000..fda7fa3104 --- /dev/null +++ b/zig/pkg/antfly/src/graph/typed_edges.zig @@ -0,0 +1,414 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Covering relationship-type postings. Values are empty: metric topology +//! needs edge identity, not timestamps, weights or document metadata. Each +//! type range preserves the original reverse-key partition order. +const std = @import("std"); +const backend = @import("../storage/backend_erased.zig"); +const keys = @import("../storage/internal_keys.zig"); +const Allocator = std.mem.Allocator; +pub const prefix = "meta:metric_type_edges:v2/"; +pub const node_prefix = "meta:metric_type_nodes:v2/"; +pub const ready_key = "meta:metric_type_edges_ready:v2"; +pub const cursor_key = "meta:metric_type_edges_cursor:v2"; +pub const active_key = "meta:metric_type_edges_active:v2"; + +pub const MaintenanceState = struct { active: bool, ready: bool }; + +/// First demand activates the covering index. Once activation/backfill starts, +/// keep it current even if the last filter is temporarily removed. Existing +/// partial indexes are detected once before recording an inactive marker; an +/// upgrade must never silently stop maintaining already persisted postings. +pub fn maintenanceState(batch: anytype, demanded: bool) !MaintenanceState { + const marker = batch.get(active_key) catch |err| switch (err) { + error.NotFound => null, + else => return err, + }; + var active = demanded; + var was_active = false; + if (marker) |raw| { + if (!std.mem.eql(u8, raw, "0") and !std.mem.eql(u8, raw, "1")) return error.InvalidGraphMetricBuildManifest; + was_active = raw[0] == '1'; + active = active or was_active; + if (!active) return .{ .active = false, .ready = false }; + } + const ready = if (batch.get(ready_key)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + active = active or ready; + if (!active) active = if (batch.get(cursor_key)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + if (marker == null and !active) { + var cursor = try batch.openCursor(); + defer cursor.close(); + if (try cursor.seekAtOrAfter(prefix)) |entry| active = std.mem.startsWith(u8, entry.key, prefix); + } + if (marker == null or (active and !was_active)) try batch.put(active_key, if (active) "1" else "0"); + return .{ .active = active, .ready = ready }; +} + +pub fn typePrefixAlloc(alloc: Allocator, kind: []const u8) ![]u8 { + return rangePrefixAlloc(alloc, prefix, kind); +} + +fn rangePrefixAlloc(alloc: Allocator, namespace: []const u8, kind: []const u8) ![]u8 { + var out: std.ArrayListUnmanaged(u8) = .empty; + errdefer out.deinit(alloc); + try out.appendSlice(alloc, namespace); + try keys.appendEncodedComponent(&out, alloc, kind); + return out.toOwnedSlice(alloc); +} + +/// One membership ref per (type, endpoint), shared across every metric filter. +/// Posting existence makes maintenance/backfill idempotent in the same batch. +pub fn update(alloc: Allocator, batch: anytype, kind: []const u8, reverse_key: []const u8, source: []const u8, target: []const u8, present: bool) !void { + const key = try keyAlloc(alloc, kind, reverse_key); + defer alloc.free(key); + const exists = if (batch.get(key)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + if (exists == present) return; + const start = try rangePrefixAlloc(alloc, node_prefix, kind); + defer alloc.free(start); + for ([_][]const u8{ source, target }) |node| { + const node_key = try std.mem.concat(alloc, u8, &.{ start, node }); + defer alloc.free(node_key); + const current = if (batch.get(node_key)) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else |err| switch (err) { + error.NotFound => @as(u64, 0), + else => return err, + }; + const next = if (present) std.math.add(u64, current, 1) catch return error.InvalidGraphMetricBuildManifest else std.math.sub(u64, current, 1) catch return error.InvalidGraphMetricBuildManifest; + if (next == 0) try batch.delete(node_key) else { + var raw: [8]u8 = undefined; + std.mem.writeInt(u64, &raw, next, .little); + try batch.put(node_key, &raw); + } + } + if (present) try batch.put(key, "") else try batch.delete(key); +} + +/// Transaction-scoped incidence deltas. Postings are staged immediately so +/// duplicate/reversed operations remain idempotent; endpoint counts are read +/// and written only once per distinct (type, node). The caller must hold the +/// store's write transaction across staging and flush, and abort on any error. +pub const Updates = struct { + const Prefixes = struct { edge: []u8, node: []u8 }; + alloc: Allocator, + deltas: std.StringHashMapUnmanaged(i64) = .empty, + prefixes: std.StringHashMapUnmanaged(Prefixes) = .empty, + scratch: std.ArrayListUnmanaged(u8) = .empty, + + pub fn init(alloc: Allocator) Updates { + return .{ .alloc = alloc }; + } + + pub fn deinit(self: *Updates) void { + var entries = self.deltas.keyIterator(); + while (entries.next()) |key| self.alloc.free(key.*); + self.deltas.deinit(self.alloc); + var kinds = self.prefixes.iterator(); + while (kinds.next()) |entry| { + self.alloc.free(entry.key_ptr.*); + self.alloc.free(entry.value_ptr.edge); + self.alloc.free(entry.value_ptr.node); + } + self.prefixes.deinit(self.alloc); + self.scratch.deinit(self.alloc); + } + + pub fn stage(self: *Updates, batch: anytype, kind: []const u8, reverse_key: []const u8, source: []const u8, target: []const u8, present: bool) !void { + return self.stageKnown(batch, kind, reverse_key, source, target, present, null); + } + + /// Net foreground mutations already know reverse-edge existence. Reuse it + /// only after checking covering-index readiness in this same transaction. + /// Backfill and arbitrary operation streams must pass null. + pub fn stageKnown(self: *Updates, batch: anytype, kind: []const u8, reverse_key: []const u8, source: []const u8, target: []const u8, present: bool, known_exists: ?bool) !void { + if (known_exists) |exists| if (exists == present) return; + const start = self.prefixes.get(kind) orelse blk: { + const owned_kind = try self.alloc.dupe(u8, kind); + errdefer self.alloc.free(owned_kind); + const node = try rangePrefixAlloc(self.alloc, node_prefix, kind); + errdefer self.alloc.free(node); + const edge = try typePrefixAlloc(self.alloc, kind); + errdefer self.alloc.free(edge); + const value = Prefixes{ .node = node, .edge = edge }; + try self.prefixes.put(self.alloc, owned_kind, value); + break :blk value; + }; + self.scratch.clearRetainingCapacity(); + try self.scratch.appendSlice(self.alloc, start.edge); + try self.scratch.appendSlice(self.alloc, reverse_key); + const exists = known_exists orelse if (batch.get(self.scratch.items)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + if (exists == present) return; + if (present) try batch.put(self.scratch.items, "") else try batch.delete(self.scratch.items); + for ([_][]const u8{ source, target }) |node| { + self.scratch.clearRetainingCapacity(); + try self.scratch.appendSlice(self.alloc, start.node); + try self.scratch.appendSlice(self.alloc, node); + const entry = try self.deltas.getOrPut(self.alloc, self.scratch.items); + if (!entry.found_existing) { + // Remove the borrowed scratch key if ownership allocation fails. + errdefer _ = self.deltas.remove(self.scratch.items); + entry.key_ptr.* = try self.alloc.dupe(u8, self.scratch.items); + entry.value_ptr.* = 0; + } + entry.value_ptr.* = std.math.add(i64, entry.value_ptr.*, if (present) 1 else -1) catch return error.InvalidGraphMetricBuildManifest; + } + } + + pub fn flush(self: *Updates, batch: anytype) !void { + const ordered = try self.alloc.alloc([]const u8, self.deltas.count()); + defer self.alloc.free(ordered); + var count: usize = 0; + var entries = self.deltas.iterator(); + while (entries.next()) |entry| { + if (entry.value_ptr.* == 0) continue; + ordered[count] = entry.key_ptr.*; + count += 1; + } + const sorted = ordered[0..count]; + std.mem.sort([]const u8, sorted, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + // Bound bulk-read scratch independently of ingestion batch size. + var values: [256]?[]const u8 = undefined; + var next_counts: [256]u64 = undefined; + var offset: usize = 0; + while (offset < sorted.len) { + const page = sorted[offset..@min(sorted.len, offset + values.len)]; + try batch.getManySorted(page, values[0..page.len]); + // Values may borrow the mutable batch. Decode all before any put. + for (page, values[0..page.len], 0..) |key, value, i| { + const current = if (value) |raw| blk: { + if (raw.len != 8) return error.InvalidGraphMetricBuildManifest; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else 0; + const next = @as(i128, current) + self.deltas.get(key).?; + next_counts[i] = std.math.cast(u64, next) orelse return error.InvalidGraphMetricBuildManifest; + } + for (page, next_counts[0..page.len]) |key, next| { + if (next == 0) try batch.delete(key) else { + var raw: [8]u8 = undefined; + std.mem.writeInt(u64, &raw, next, .little); + try batch.put(key, &raw); + } + self.deltas.getPtr(key).?.* = 0; + } + offset += page.len; + } + } +}; + +/// Merge selected type ranges by their raw suffix, deduplicating endpoints. +/// Memory is bounded by filter fanout, not graph cardinality. Each stream's +/// key borrows its own cursor until popped; the returned key lasts until next(). +pub const MergedCursor = struct { + const Stream = struct { raw: backend.Cursor, start: []u8, head: ?[]const u8 = null }; + const Heap = std.PriorityQueue(usize, []Stream, order); + alloc: Allocator, + streams: []Stream, + heap: Heap, + last: std.ArrayListUnmanaged(u8) = .empty, + + fn order(streams: []Stream, a: usize, b: usize) std.math.Order { + return std.mem.order(u8, streams[a].head.?, streams[b].head.?); + } + + pub fn init(alloc: Allocator, txn: anytype, types: []const []const u8, nodes: bool, after: []const u8) !MergedCursor { + _ = try txn.get(ready_key); + const streams = try alloc.alloc(Stream, types.len); + var result = MergedCursor{ .alloc = alloc, .streams = streams, .heap = Heap.initContext(streams) }; + var initialized: usize = 0; + errdefer { + result.streams = streams[0..initialized]; + for (result.streams) |*stream| { + stream.raw.close(); + alloc.free(stream.start); + } + result.heap.deinit(alloc); + alloc.free(streams); + } + try result.heap.ensureTotalCapacity(alloc, types.len); + for (types, streams, 0..) |kind, *stream, i| { + const start = try rangePrefixAlloc(alloc, if (nodes) node_prefix else prefix, kind); + const raw = txn.openCursor() catch |err| { + alloc.free(start); + return err; + }; + stream.* = .{ .raw = raw, .start = start }; + initialized += 1; + const seek = try std.mem.concat(alloc, u8, &.{ start, after }); + defer alloc.free(seek); + var item = try stream.raw.seekAtOrAfter(seek); + if (after.len > 0) if (item) |entry| if (std.mem.eql(u8, entry.key, seek)) { + item = try stream.raw.next(); + }; + if (item) |entry| if (std.mem.startsWith(u8, entry.key, start)) { + stream.head = entry.key[start.len..]; + try result.heap.push(alloc, i); + }; + } + return result; + } + + pub fn deinit(self: *@This()) void { + for (self.streams) |*stream| { + stream.raw.close(); + self.alloc.free(stream.start); + } + self.alloc.free(self.streams); + self.heap.deinit(self.alloc); + self.last.deinit(self.alloc); + } + + pub fn next(self: *@This()) !?[]const u8 { + const first = self.heap.peek() orelse return null; + self.last.clearRetainingCapacity(); + try self.last.appendSlice(self.alloc, self.streams[first].head.?); + while (self.heap.peek()) |i| { + if (!std.mem.eql(u8, self.streams[i].head.?, self.last.items)) break; + _ = self.heap.pop(); + const stream = &self.streams[i]; + stream.head = null; + if (try stream.raw.next()) |entry| if (std.mem.startsWith(u8, entry.key, stream.start)) { + stream.head = entry.key[stream.start.len..]; + try self.heap.push(self.alloc, i); + }; + } + return self.last.items; + } +}; + +pub fn keyAlloc(alloc: Allocator, kind: []const u8, reverse_key: []const u8) ![]u8 { + const start = try typePrefixAlloc(alloc, kind); + defer alloc.free(start); + return std.mem.concat(alloc, u8, &.{ start, reverse_key }); +} + +pub const Entry = struct { key: []const u8, cursor: []const u8 }; + +pub const Cursor = struct { + alloc: Allocator, + raw: backend.Cursor, + types: []const []const u8, + lower: []const u8, + upper: []const u8, + resume_key: []const u8, + type_index: usize = 0, + started: bool = false, + type_prefix: []u8 = &.{}, + exhausted: bool = false, + + pub fn init(alloc: Allocator, txn: anytype, filter: anytype, lower: []const u8, upper: []const u8, resume_key: []const u8) !Cursor { + const types = try alloc.dupe([]const u8, if (filter.mode == .all) &.{} else filter.types); + errdefer alloc.free(types); + std.mem.sort([]const u8, types, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + // Index preparation is a durable prerequisite, not a silent scan + // fallback that can change cursor interpretation across checkpoints. + if (types.len > 0) _ = txn.get(ready_key) catch |err| switch (err) { + error.NotFound => return error.InvalidGraphMetricBuildManifest, + else => return err, + }; + if (resume_key.len > 0) { + var raw_resume: ?[]const u8 = if (types.len == 0) resume_key else null; + for (types) |kind| { + const start = try typePrefixAlloc(alloc, kind); + defer alloc.free(start); + if (std.mem.startsWith(u8, resume_key, start)) { + raw_resume = resume_key[start.len..]; + break; + } + } + const key = raw_resume orelse return error.InvalidGraphMetricBuildManifest; + if (key.len == 0 or std.mem.startsWith(u8, key, "meta:") or + (lower.len > 0 and std.mem.order(u8, key, lower) == .lt) or + (upper.len > 0 and std.mem.order(u8, key, upper) != .lt)) + return error.InvalidGraphMetricBuildManifest; + } + return .{ .alloc = alloc, .raw = try txn.openCursor(), .types = types, .lower = lower, .upper = upper, .resume_key = resume_key }; + } + + pub fn deinit(self: *@This()) void { + self.raw.close(); + self.alloc.free(self.types); + self.alloc.free(self.type_prefix); + } + + pub fn next(self: *@This()) !?Entry { + if (self.exhausted) return null; + if (self.types.len == 0) { + var found = if (self.started) try self.raw.next() else blk: { + self.started = true; + const start = if (self.resume_key.len > 0) self.resume_key else self.lower; + var item = if (start.len > 0) try self.raw.seekAtOrAfter(start) else try self.raw.first(); + if (item) |entry| if (self.resume_key.len > 0 and std.mem.eql(u8, entry.key, self.resume_key)) { + item = try self.raw.next(); + }; + break :blk item; + }; + if (found) |entry| if (std.mem.startsWith(u8, entry.key, "meta:")) { + found = try self.raw.seekAtOrAfter("meta;"); + }; + if (found) |entry| { + if (self.upper.len == 0 or std.mem.order(u8, entry.key, self.upper) == .lt) + return .{ .key = entry.key, .cursor = entry.key }; + } + self.exhausted = true; + return null; + } + while (self.type_index < self.types.len) { + const found = if (self.started) try self.raw.next() else blk: { + self.alloc.free(self.type_prefix); + self.type_prefix = &.{}; + self.type_prefix = try typePrefixAlloc(self.alloc, self.types[self.type_index]); + const lower = try std.mem.concat(self.alloc, u8, &.{ self.type_prefix, self.lower }); + defer self.alloc.free(lower); + const start = if (self.resume_key.len > 0 and std.mem.order(u8, self.resume_key, lower) == .gt) self.resume_key else lower; + self.started = true; + var item = try self.raw.seekAtOrAfter(start); + if (item) |entry| if (self.resume_key.len > 0 and std.mem.eql(u8, entry.key, self.resume_key)) { + item = try self.raw.next(); + }; + break :blk item; + }; + if (found) |entry| if (std.mem.startsWith(u8, entry.key, self.type_prefix)) { + const key = entry.key[self.type_prefix.len..]; + if (self.upper.len == 0 or std.mem.order(u8, key, self.upper) == .lt) + return .{ .key = key, .cursor = entry.key }; + }; + self.type_index += 1; + self.started = false; + } + self.exhausted = true; + return null; + } +}; diff --git a/zig/pkg/antfly/src/graph/vector_chunk.zig b/zig/pkg/antfly/src/graph/vector_chunk.zig new file mode 100644 index 0000000000..5d7d347a50 --- /dev/null +++ b/zig/pkg/antfly/src/graph/vector_chunk.zig @@ -0,0 +1,70 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Fixed ordinal vector blocks. Presence is separate from value: a missing +//! producer output must never silently become a valid zero score. +const std = @import("std"); + +pub const entries = 256; +const bitmap_bytes = entries / 8; +pub const encoded_len = bitmap_bytes + entries * @sizeOf(f64); +pub const Chunk = [encoded_len]u8; + +// Integer lanes (for example immutable out-degrees) share the framing and +// presence bitmap, but never round-trip through floating point. +pub fn putU64(chunk: *Chunk, slot: usize, value: u64) !void { + if (slot >= entries) return error.InvalidGraphMetricScore; + chunk[slot / 8] |= @as(u8, 1) << @intCast(slot % 8); + std.mem.writeInt(u64, chunk[bitmap_bytes + slot * 8 ..][0..8], value, .little); +} + +pub fn getU64(chunk: []const u8, slot: usize, required: bool) !u64 { + if (chunk.len != encoded_len or slot >= entries) return error.InvalidGraphMetricScore; + if (chunk[slot / 8] & (@as(u8, 1) << @intCast(slot % 8)) == 0) { + if (required) return error.InvalidGraphMetricScore; + return 0; + } + return std.mem.readInt(u64, chunk[bitmap_bytes + slot * 8 ..][0..8], .little); +} + +pub fn put(chunk: *Chunk, slot: usize, value: f64) !void { + if (slot >= entries or !std.math.isFinite(value) or value < 0) return error.InvalidGraphMetricScore; + chunk[slot / 8] |= @as(u8, 1) << @intCast(slot % 8); + std.mem.writeInt(u64, chunk[bitmap_bytes + slot * 8 ..][0..8], @bitCast(value), .little); +} + +pub fn get(chunk: []const u8, slot: usize, required: bool) !f64 { + if (chunk.len != encoded_len or slot >= entries) return error.InvalidGraphMetricScore; + if (chunk[slot / 8] & (@as(u8, 1) << @intCast(slot % 8)) == 0) { + if (required) return error.InvalidGraphMetricScore; + return 0; + } + const value: f64 = @bitCast(std.mem.readInt(u64, chunk[bitmap_bytes + slot * 8 ..][0..8], .little)); + if (!std.math.isFinite(value) or value < 0) return error.InvalidGraphMetricScore; + return value; +} + +test "graph metric vector chunks distinguish missing from zero and reject malformed values" { + var chunk: Chunk = @splat(0); + try std.testing.expectError(error.InvalidGraphMetricScore, get(&chunk, 0, true)); + try put(&chunk, 0, 0); + try put(&chunk, entries - 1, 0.5); + try std.testing.expectEqual(@as(f64, 0), try get(&chunk, 0, true)); + try std.testing.expectEqual(@as(f64, 0.5), try get(&chunk, entries - 1, true)); + try std.testing.expectError(error.InvalidGraphMetricScore, put(&chunk, 2, std.math.nan(f64))); + try std.testing.expectError(error.InvalidGraphMetricScore, get(chunk[0..10], 0, true)); + const exact_degree = std.math.maxInt(u64) - 1; + try putU64(&chunk, 0, exact_degree); + try std.testing.expectEqual(exact_degree, try getU64(&chunk, 0, true)); +} diff --git a/zig/pkg/antfly/src/graph/warm_start.zig b/zig/pkg/antfly/src/graph/warm_start.zig new file mode 100644 index 0000000000..6a21d808d9 --- /dev/null +++ b/zig/pkg/antfly/src/graph/warm_start.zig @@ -0,0 +1,50 @@ +// Copyright 2026 Antfly, Inc. +// SPDX-License-Identifier: Elastic-2.0 + +//! Shared warm-start contract. Only PageRank has a unique damped fixed point +//! independent of support in the seed. Spectral metrics cold-start: even a +//! normalized old vector can be orthogonal to a newly dominant component. +const std = @import("std"); + +pub fn supported(kind: anytype) bool { + return kind == .pagerank; +} + +pub const Mass = struct { + sum: f64 = 0, + correction: f64 = 0, + + pub fn add(self: *Mass, value: f64) !void { + if (!std.math.isFinite(value) or value < 0) return error.InvalidGraphMetricWarmStart; + const next = self.sum + value; + self.correction += if (@abs(self.sum) >= value) (self.sum - next) + value else (value - next) + self.sum; + self.sum = next; + } + + pub fn total(self: Mass) !f64 { + const result = self.sum + self.correction; + if (!std.math.isFinite(result) or result < 0) return error.InvalidGraphMetricWarmStart; + return result; + } +}; + +pub fn normalized(value: f64, mass: f64, node_count: usize) !f64 { + if (node_count == 0 or !std.math.isFinite(value) or value < 0 or !std.math.isFinite(mass) or mass < 0) + return error.InvalidGraphMetricWarmStart; + // A deleted/zero prior component is a cold start, never a zero vector. + // Divide rather than multiplying by 1/mass, which can overflow for a + // perfectly valid subnormal seed mass. + return if (mass == 0) 1.0 / @as(f64, @floatFromInt(node_count)) else value / mass; +} + +test "graph metric warm start mass handles removed nodes and subnormal seeds" { + var mass = Mass{}; + try mass.add(0.5); + try mass.add(0.5); + try mass.add(0.25); + try mass.add(0.25); + try std.testing.expectEqual(@as(f64, 1.5), try mass.total()); + try std.testing.expectEqual(@as(f64, 0.25), try normalized(0, 0, 4)); + try std.testing.expectEqual(@as(f64, 1), try normalized(5e-324, 5e-324, 1)); + try std.testing.expectError(error.InvalidGraphMetricWarmStart, mass.add(-1)); +} diff --git a/zig/pkg/antfly/src/graph/work_budget.zig b/zig/pkg/antfly/src/graph/work_budget.zig index 06e486c2dd..f9c1c7cfab 100644 --- a/zig/pkg/antfly/src/graph/work_budget.zig +++ b/zig/pkg/antfly/src/graph/work_budget.zig @@ -268,6 +268,77 @@ pub const RetainedLease = struct { } }; +/// Scoped allocator for request-owned metric work. Scratch frees return their +/// reservation immediately; escaping output is explicitly detached and retains +/// a consumptive request charge. Only free allocations made by this wrapper. +pub const RetainedAllocator = struct { + backing: std.mem.Allocator, + budget: ?*WorkBudget, + live_bytes: usize = 0, + denied: bool = false, + + pub fn allocator(self: *@This()) std.mem.Allocator { + return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + + pub fn detach(self: *@This()) void { + self.live_bytes = 0; + } + + fn reserve(self: *@This(), bytes: usize) bool { + if (self.budget) |budget| budget.retainStateBytes(bytes) catch { + self.denied = true; + return false; + }; + self.live_bytes += bytes; + return true; + } + + fn release(self: *@This(), bytes: usize) void { + self.live_bytes -= bytes; + if (self.budget) |budget| budget.releaseStateBytes(bytes); + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + if (!self.reserve(len)) return null; + return self.backing.rawAlloc(len, alignment, ret_addr) orelse { + self.release(len); + return null; + }; + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + const self: *@This() = @ptrCast(@alignCast(ctx)); + const growth = new_len -| memory.len; + if (!self.reserve(growth)) return false; + if (!self.backing.rawResize(memory, alignment, new_len, ret_addr)) { + self.release(growth); + return false; + } + self.release(memory.len -| new_len); + return true; + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + const growth = new_len -| memory.len; + if (!self.reserve(growth)) return null; + const result = self.backing.rawRemap(memory, alignment, new_len, ret_addr) orelse { + self.release(growth); + return null; + }; + self.release(memory.len -| new_len); + return result; + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *@This() = @ptrCast(@alignCast(ctx)); + self.backing.rawFree(memory, alignment, ret_addr); + self.release(memory.len); + } +}; + pub const AllocationReplacement = struct { lease: *RetainedLease, prior_bytes: usize, diff --git a/zig/pkg/antfly/src/inference/managed_embedder.zig b/zig/pkg/antfly/src/inference/managed_embedder.zig index 277940d795..69c7e0272b 100644 --- a/zig/pkg/antfly/src/inference/managed_embedder.zig +++ b/zig/pkg/antfly/src/inference/managed_embedder.zig @@ -1088,7 +1088,20 @@ pub const ManagedEmbedder = struct { return try initFromIndexValueObjectWithOptions(alloc, root, .{}); } - fn initFromIndexValueObjectWithOptions(alloc: std.mem.Allocator, root: std.json.Value, supplied_options: InitOptions) !ManagedEmbedder { + pub fn initFromIndexValueObjectWithOptions(alloc: std.mem.Allocator, root: std.json.Value, options: InitOptions) !ManagedEmbedder { + return try initFromIndexValueObjectWithOptionsAndKind(alloc, root, options, null); + } + + pub fn initDenseFromIndexValueObjectWithOptions(alloc: std.mem.Allocator, root: std.json.Value, options: InitOptions) !ManagedEmbedder { + return try initFromIndexValueObjectWithOptionsAndKind(alloc, root, options, false); + } + + fn initFromIndexValueObjectWithOptionsAndKind( + alloc: std.mem.Allocator, + root: std.json.Value, + supplied_options: InitOptions, + sparse_kind: ?bool, + ) !ManagedEmbedder { const object = switch (root) { .object => |object| object, else => return error.InvalidManagedEmbeddingIndex, @@ -1106,14 +1119,14 @@ pub const ManagedEmbedder = struct { var it = object.iterator(); while (it.next()) |entry| { - var managed = try parseManagedEmbeddingEntry(alloc, entry.key_ptr.*, entry.value_ptr.*, options) orelse continue; + var managed = try parseManagedEmbeddingEntry(alloc, entry.key_ptr.*, entry.value_ptr.*, options, sparse_kind) orelse continue; entries.append(alloc, managed) catch |err| { managed.deinit(alloc); return err; }; } - try validateAllEmbeddingEnrichmentProducers(alloc, root, options, entries.items); - try addArtifactBackedManagedEmbeddingEntries(alloc, root, options, &entries); + try validateAllEmbeddingEnrichmentProducers(alloc, root, options, entries.items, sparse_kind); + try addArtifactBackedManagedEmbeddingEntries(alloc, root, options, &entries, sparse_kind); var vector_spaces = VectorSpaceMap.empty; defer vector_spaces.deinit(alloc); try collectEmbeddingVectorSpaces(root, &vector_spaces, alloc); @@ -1270,10 +1283,20 @@ pub const ManagedEmbedder = struct { alloc: std.mem.Allocator, indexes_json: []const u8, options: InitOptions, + ) !?db_embedder.DenseEmbedder { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, indexes_json, .{}); + defer parsed.deinit(); + return try createDenseEmbedderFromIndexValueWithOptions(alloc, parsed.value, options); + } + + pub fn createDenseEmbedderFromIndexValueWithOptions( + alloc: std.mem.Allocator, + root: std.json.Value, + options: InitOptions, ) !?db_embedder.DenseEmbedder { const owned = try alloc.create(ManagedEmbedder); errdefer alloc.destroy(owned); - owned.* = try initFromIndexesJsonWithOptions(alloc, indexes_json, options); + owned.* = try initFromIndexValueObjectWithOptionsAndKind(alloc, root, options, false); if (!owned.hasDenseEntries()) { owned.deinit(); alloc.destroy(owned); @@ -1298,10 +1321,20 @@ pub const ManagedEmbedder = struct { alloc: std.mem.Allocator, indexes_json: []const u8, options: InitOptions, + ) !?db_embedder.SparseEmbedder { + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, indexes_json, .{}); + defer parsed.deinit(); + return try createSparseEmbedderFromIndexValueWithOptions(alloc, parsed.value, options); + } + + pub fn createSparseEmbedderFromIndexValueWithOptions( + alloc: std.mem.Allocator, + root: std.json.Value, + options: InitOptions, ) !?db_embedder.SparseEmbedder { const owned = try alloc.create(ManagedEmbedder); errdefer alloc.destroy(owned); - owned.* = try initFromIndexesJsonWithOptions(alloc, indexes_json, options); + owned.* = try initFromIndexValueObjectWithOptionsAndKind(alloc, root, options, true); if (!owned.hasSparseEntries()) { owned.deinit(); alloc.destroy(owned); @@ -3475,9 +3508,14 @@ fn validateEmbeddingEnrichmentProducerValue( producer: std.json.Value, options: InitOptions, executable_entries: ?[]const ManagedEmbeddingEntry, + sparse_kind: ?bool, ) !void { const producer_sparse = try semanticProducerV2Sparse(producer); if (producer_sparse) |sparse| { + // Stage factories own only one executable registry. Do not mistake + // the other stage's deliberately omitted owner for an orphan. + // Unfiltered catalog admission still validates every producer. + if (sparse_kind) |selected| if (sparse != selected) return; if (sparse and enrichment_dims != null) return error.ConflictingEmbeddingArtifactDimensions; if (!sparse and enrichment_dims == null) return error.EmbeddingArtifactDimensionRequired; } @@ -3521,6 +3559,7 @@ fn validateEmbeddingEnrichmentProducer( enrichment: std.json.Value, options: InitOptions, executable_entries: ?[]const ManagedEmbeddingEntry, + sparse_kind: ?bool, ) !void { const object = switch (enrichment) { .object => |object| object, @@ -3564,6 +3603,7 @@ fn validateEmbeddingEnrichmentProducer( producer.value, options, executable_entries, + sparse_kind, ); }, .object => try validateEmbeddingEnrichmentProducerValue( @@ -3573,6 +3613,7 @@ fn validateEmbeddingEnrichmentProducer( producer_json, options, executable_entries, + sparse_kind, ), else => return error.InvalidEmbeddingArtifactProducer, } @@ -3583,23 +3624,24 @@ fn validateAllEmbeddingEnrichmentProducers( value: std.json.Value, options: InitOptions, executable_entries: []const ManagedEmbeddingEntry, + sparse_kind: ?bool, ) !void { switch (value) { .object => |object| { if (object.get("enrichments")) |enrichments| { if (enrichments != .array) return error.InvalidManagedEmbeddingIndex; for (enrichments.array.items) |enrichment| { - try validateEmbeddingEnrichmentProducer(alloc, enrichment, options, executable_entries); + try validateEmbeddingEnrichmentProducer(alloc, enrichment, options, executable_entries, sparse_kind); } } var it = object.iterator(); while (it.next()) |entry| { if (std.mem.eql(u8, entry.key_ptr.*, "enrichments")) continue; - try validateAllEmbeddingEnrichmentProducers(alloc, entry.value_ptr.*, options, executable_entries); + try validateAllEmbeddingEnrichmentProducers(alloc, entry.value_ptr.*, options, executable_entries, sparse_kind); } }, .array => |array| for (array.items) |item| - try validateAllEmbeddingEnrichmentProducers(alloc, item, options, executable_entries), + try validateAllEmbeddingEnrichmentProducers(alloc, item, options, executable_entries, sparse_kind), else => {}, } } @@ -3614,7 +3656,7 @@ pub fn validateEmbeddingEnrichmentProducerJsonWithOptions( ) !void { var parsed = try std.json.parseFromSlice(std.json.Value, alloc, enrichment_json, .{}); defer parsed.deinit(); - try validateEmbeddingEnrichmentProducer(alloc, parsed.value, options, null); + try validateEmbeddingEnrichmentProducer(alloc, parsed.value, options, null, null); } const CatalogProducerOwner = struct { @@ -4014,6 +4056,7 @@ fn validateCatalogEmbeddingProducerOwnership( producer, .{}, null, + null, ); } } @@ -4249,6 +4292,7 @@ fn addArtifactBackedManagedEmbeddingEntries( root: std.json.Value, options: InitOptions, entries: *std.ArrayListUnmanaged(ManagedEmbeddingEntry), + sparse_kind: ?bool, ) !void { const object = root.object; var it = object.iterator(); @@ -4265,6 +4309,7 @@ fn addArtifactBackedManagedEmbeddingEntries( defer parsed_cfg.deinit(); const cfg = parsed_cfg.value; if (cfg.external orelse false) continue; + if (sparse_kind) |selected| if ((cfg.sparse orelse false) != selected) continue; if (cfg.embedding_name) |artifact_name| { try registerArtifactManagedEmbeddingLookup( @@ -4298,6 +4343,7 @@ fn parseManagedEmbeddingEntry( index_name: []const u8, value: std.json.Value, options: InitOptions, + sparse_kind: ?bool, ) !?ManagedEmbeddingEntry { const root = switch (value) { .object => |object| object, @@ -4315,6 +4361,7 @@ fn parseManagedEmbeddingEntry( if (external) return null; const sparse = cfg.sparse orelse false; + if (sparse_kind) |expected_sparse| if (sparse != expected_sparse) return null; const embedder = root.get("embedder") orelse return null; const declared_dims = if (sparse) null else try resolveDeclaredEmbeddingDimensions(cfg); @@ -7698,6 +7745,67 @@ test "managed embedder interface deinit uses owner allocator" { dense.deinit(std.heap.page_allocator); } +test "serverless managed embedder stage factories ignore unrelated provider construction" { + var local = TestLocalDenseProvider{ .dimensions = 3 }; + + var sparse_parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, + \\{ + \\ "sparse_idx":{"type":"embeddings","field":"body","sparse":true,"embedder":{"provider":"antfly","model":"antflydb/splade"}}, + \\ "broken_dense":{"type":"embeddings","field":"body","dimension":3,"embedder":{"provider":"openai","model":""}} + \\} + , .{}); + defer sparse_parsed.deinit(); + const sparse = (try ManagedEmbedder.createSparseEmbedderFromIndexValueWithOptions( + std.testing.allocator, + sparse_parsed.value, + .{ .antfly_provider = local.provider() }, + )) orelse return error.TestUnexpectedResult; + sparse.deinit(std.testing.allocator); + + var dense_parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, + \\{ + \\ "dense_idx":{"type":"embeddings","field":"body","dimension":3,"embedder":{"provider":"antfly","model":"antflydb/clipclap"}}, + \\ "broken_sparse":{"type":"embeddings","field":"body","sparse":true,"embedder":{"provider":"openai","model":""}} + \\} + , .{}); + defer dense_parsed.deinit(); + const dense = (try ManagedEmbedder.createDenseEmbedderFromIndexValueWithOptions( + std.testing.allocator, + dense_parsed.value, + .{ .antfly_provider = local.provider() }, + )) orelse return error.TestUnexpectedResult; + dense.deinit(std.testing.allocator); +} + +test "serverless managed embedder stage selection includes producer validation and artifact consumers" { + const alloc = std.testing.allocator; + var local = TestLocalDenseProvider{ .dimensions = 3 }; + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, + \\{ + \\ "dense_owner":{"type":"embeddings","field":"body","dimension":3,"embedding_name":"dense_v1","embedder":{"provider":"antfly","model":"test-model"}}, + \\ "sparse_owner":{"type":"embeddings","field":"body","sparse":true,"embedding_name":"sparse_v1","embedder":{"provider":"antfly","model":"test-model"}}, + \\ "dense_consumer":{"type":"embeddings","dimension":3,"sources":[{"artifact":"dense_v1"}]}, + \\ "sparse_consumer":{"type":"embeddings","sparse":true,"sources":[{"artifact":"sparse_v1"}]}, + \\ "enrichments":[ + \\ {"name":"dense_v1","kind":"embedding","field":"body","expected_dims":3,"producer_json":{"version":2,"provider":"antfly","model":"test-model","endpoint":"antfly:embedded","sparse":false}}, + \\ {"name":"sparse_v1","kind":"embedding","field":"body","producer_json":{"version":2,"provider":"antfly","model":"test-model","endpoint":"antfly:embedded","sparse":true}} + \\ ] + \\} + , .{}); + defer parsed.deinit(); + for ([_]?bool{ null, false, true }) |kind| { + var managed = try ManagedEmbedder.initFromIndexValueObjectWithOptionsAndKind(alloc, parsed.value, .{ .antfly_provider = local.provider() }, kind); + defer managed.deinit(); + try std.testing.expectEqual(@as(usize, if (kind == null) 2 else 1), managed.entries.len); + try std.testing.expectEqual(kind == null or !kind.?, managed.hasDenseEntries()); + try std.testing.expectEqual(kind == null or kind.?, managed.hasSparseEntries()); + } + // In-stage validation must still reject an orphan; selecting a factory + // must not disable ownership validation altogether. + _ = parsed.value.object.swapRemove("dense_owner"); + try std.testing.expectError(error.InvalidEmbeddingArtifactProducer, ManagedEmbedder.initDenseFromIndexValueObjectWithOptions(alloc, parsed.value, .{ .antfly_provider = local.provider() })); +} + test "managed embedder uses embedder dimensions metadata at runtime" { var local = TestLocalDenseProvider{ .dimensions = 3 }; var managed = try ManagedEmbedder.initFromIndexesJsonWithAntflyProvider(std.testing.allocator, diff --git a/zig/pkg/antfly/src/main.zig b/zig/pkg/antfly/src/main.zig index 351eff315c..07141fca20 100644 --- a/zig/pkg/antfly/src/main.zig +++ b/zig/pkg/antfly/src/main.zig @@ -73,6 +73,7 @@ fn mainImpl(init: std.process.Init) !void { switch (command.route) { .cli => return runRuntimeUnit(.cli, subcommand, init, &args), .data => return runRuntimeUnit(.data, subcommand, init, &args), + .graph_metric_maintenance => return runRuntimeUnit(.graph_metric_maintenance, subcommand, init, &args), .ha => return runRuntimeUnit(.ha, subcommand, init, &args), .inference => { var worker_lifetime = inference_process_supervisor.WorkerLifetime{}; @@ -93,10 +94,11 @@ fn mainImpl(init: std.process.Init) !void { } } -const RuntimeRole = enum { cli, data, ha, inference, metadata, serverless, standalone }; +const RuntimeRole = enum { cli, data, graph_metric_maintenance, ha, inference, metadata, serverless, standalone }; extern fn antfly_runtime_cli(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_data(context: *const runtime_bridge.Context) callconv(.c) c_int; +extern fn antfly_runtime_graph_metric_maintenance(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_ha(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_inference(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_metadata(context: *const runtime_bridge.Context) callconv(.c) c_int; @@ -132,6 +134,7 @@ pub fn runRuntimeUnit( const code = switch (role) { .cli => antfly_runtime_cli(&context), .data => antfly_runtime_data(&context), + .graph_metric_maintenance => antfly_runtime_graph_metric_maintenance(&context), .ha => antfly_runtime_ha(&context), .inference => antfly_runtime_inference(&context), .metadata => antfly_runtime_metadata(&context), diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig index ef7a15a55f..188882feb7 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig @@ -1307,6 +1307,23 @@ pub const Client = struct { return ApiResponse(std.json.Value).fromResponse(self.allocator, &resp); } + /// Execute a graph metric operational action + /// POST /db/v1/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} + pub fn executeGraphMetricAction(self: *@This(), table_name: []const u8, index_name: []const u8, metric_name: []const u8, action: []const u8) !ApiResponse(types.GraphMetricActionResponse) { + const encoded_table_name = try httpx.PercentEncoding.encode(self.allocator, table_name); + defer self.allocator.free(encoded_table_name); + const encoded_index_name = try httpx.PercentEncoding.encode(self.allocator, index_name); + defer self.allocator.free(encoded_index_name); + const encoded_metric_name = try httpx.PercentEncoding.encode(self.allocator, metric_name); + defer self.allocator.free(encoded_metric_name); + const encoded_action = try httpx.PercentEncoding.encode(self.allocator, action); + defer self.allocator.free(encoded_action); + const url = try std.fmt.allocPrint(self.allocator, "{s}/db/v1/tables/{s}/indexes/{s}/graph-metrics/{s}:{s}", .{ self.base_url, encoded_table_name, encoded_index_name, encoded_metric_name, encoded_action }); + defer self.allocator.free(url); + var resp = try self.http.post(url, .{ .headers = self.authHeaders() }); + return ApiResponse(types.GraphMetricActionResponse).fromResponse(self.allocator, &resp); + } + /// Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge /// POST /db/v1/tables/{tableName}/merge pub fn linearMerge(self: *@This(), table_name: []const u8, body: types.LinearMergeRequest) !ApiResponse(types.LinearMergeResult) { diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/root.zig b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/root.zig index b354025d37..8e27862ade 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/root.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/root.zig @@ -46,6 +46,7 @@ pub const BackupListResponse = types.BackupListResponse; pub const BackupMetadataUnavailableError = types.BackupMetadataUnavailableError; pub const BackupOutcomeAmbiguousConflict = types.BackupOutcomeAmbiguousConflict; pub const BackupRequest = types.BackupRequest; +pub const BatchCommittedFailure = types.BatchCommittedFailure; pub const BatchRequest = types.BatchRequest; pub const BatchResponse = types.BatchResponse; pub const BedrockEmbedderConfig = types.BedrockEmbedderConfig; @@ -284,6 +285,22 @@ pub const GraphMatchEdge = types.GraphMatchEdge; pub const GraphMatchNode = types.GraphMatchNode; pub const GraphMatchOperationLimitExceededError = types.GraphMatchOperationLimitExceededError; pub const GraphMatchQuery = types.GraphMatchQuery; +pub const GraphMetricActionResponse = types.GraphMetricActionResponse; +pub const GraphMetricBuildPageStatus = types.GraphMetricBuildPageStatus; +pub const GraphMetricConfig = types.GraphMetricConfig; +pub const GraphMetricEdgeFilter = types.GraphMetricEdgeFilter; +pub const GraphMetricEdgeFilterStatus = types.GraphMetricEdgeFilterStatus; +pub const GraphMetricEvent = types.GraphMetricEvent; +pub const GraphMetricFilter = types.GraphMetricFilter; +pub const GraphMetricOrder = types.GraphMetricOrder; +pub const GraphMetricProfile = types.GraphMetricProfile; +pub const GraphMetricQuery = types.GraphMetricQuery; +pub const GraphMetricRerank = types.GraphMetricRerank; +pub const GraphMetricRerankScoreDetails = types.GraphMetricRerankScoreDetails; +pub const GraphMetricResult = types.GraphMetricResult; +pub const GraphMetricRuntimeStats = types.GraphMetricRuntimeStats; +pub const GraphMetricScore = types.GraphMetricScore; +pub const GraphMetricStatus = types.GraphMetricStatus; pub const GraphNodeSelector = types.GraphNodeSelector; pub const GraphNodesResult = types.GraphNodesResult; pub const GraphNotEqualPredicate = types.GraphNotEqualPredicate; @@ -540,6 +557,7 @@ pub const QueryRequest = types.QueryRequest; pub const QueryResponses = types.QueryResponses; pub const QueryResult = types.QueryResult; pub const QueryResultBase = types.QueryResultBase; +pub const QueryScoreDetails = types.QueryScoreDetails; pub const QueryStrategy = types.QueryStrategy; pub const QueryStringQuery = types.QueryStringQuery; pub const QueryTemporarilyUnavailableError = types.QueryTemporarilyUnavailableError; diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/types.zig b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/types.zig index b2b28eeed8..90e6645115 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/types.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/types.zig @@ -2455,6 +2455,49 @@ pub const BackupRequest = struct { } }; +/// Additive details for a committed batch that needs operator action. The open string code is forward-compatible with older SDKs; clients should treat unknown codes as non-retryable when `retryable` is false. +pub const BatchCommittedFailure = struct { + /// Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + code: []const u8, + /// Actionable operator guidance. + message: []const u8, + /// Optional stable reason within the failure category, such as `build_budget_exceeded`. + reason: ?[]const u8 = null, + /// Whether replaying the document mutation is safe. Committed repair outcomes are false. + retryable: bool, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "code", "code", false }, + .{ "message", "message", false }, + .{ "reason", "reason", true }, + .{ "retryable", "retryable", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("code"); + try jw.write(self.code); + try jw.objectField("message"); + try jw.write(self.message); + if (self.reason) |value| { + try jw.objectField("reason"); + try jw.write(value); + } + try jw.objectField("retryable"); + try jw.write(self.retryable); + try jw.endObject(); + } +}; + /// Batch insert, delete, and transform operations in a single request. **Atomicity**: - **Single shard**: Operations are atomic within shard boundaries - **Multiple shards**: Uses distributed 2-phase commit (2PC) for atomic cross-shard writes **How distributed transactions work**: 1. Metadata server allocates HLC timestamp and selects coordinator shard 2. Coordinator writes transaction record, participants write intents 3. After all intents succeed, coordinator commits transaction 4. Participants are notified asynchronously to resolve intents 5. Recovery loop ensures notifications complete even after coordinator failure **Performance**: - Single-shard batches: < 5ms latency - Cross-shard transactions: ~20ms latency - Intent resolution: < 30 seconds worst-case (via recovery loop) **Guarantees**: - All writes succeed or all fail (atomicity across all shards) - Coordinator failure is recoverable (new leader resumes notifications) - Idempotent resolution (duplicate notifications are safe) **Benefits**: - Reduces network overhead compared to individual requests - More efficient indexing (updates are batched) - Automatic distributed transactions when operations span shards The inserts are upserts - existing keys are overwritten, new keys are created. pub const BatchRequest = struct { /// Map of document IDs to document objects. Each key is the unique identifier for the document. Best practices: - Use consistent key naming schemes (e.g., "user:123", "article:456") - Key length affects storage and performance - keep them reasonably short - Keys are sorted lexicographically, so choose prefixes that support range scans @@ -2504,7 +2547,7 @@ pub const BatchRequest = struct { }; pub const BatchResponse = struct { - /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal enrichment failure needs operator repair and will not be retried indefinitely. + /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal background materialization failure needs operator repair and will not be retried indefinitely. Inspect `failure` when present; retrying the document write is unnecessary. status: ?[]const u8 = null, /// Number of documents successfully inserted inserted: ?i64 = null, @@ -2512,6 +2555,7 @@ pub const BatchResponse = struct { deleted: ?i64 = null, /// Number of documents successfully transformed transformed: ?i64 = null, + failure: ?BatchCommittedFailure = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -2519,6 +2563,7 @@ pub const BatchResponse = struct { .{ "inserted", "inserted", true }, .{ "deleted", "deleted", true }, .{ "transformed", "transformed", true }, + .{ "failure", "failure", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -2547,6 +2592,10 @@ pub const BatchResponse = struct { try jw.objectField("transformed"); try jw.write(value); } + if (self.failure) |value| { + try jw.objectField("failure"); + try jw.write(value); + } try jw.endObject(); } }; @@ -5225,6 +5274,8 @@ pub const CreateGraphIndexRequest = struct { version: ?i64 = null, /// Inline managed enrichment definitions required by this index. enrichments: ?[]const EnrichmentConfig = null, + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires index_capabilities.artifact_sources=true and is rejected by serverless deployments. sources: ?[]const GraphArtifactSourceConfig = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) @@ -5248,6 +5299,7 @@ pub const CreateGraphIndexRequest = struct { .{ "description", "description", true }, .{ "version", "version", true }, .{ "enrichments", "enrichments", true }, + .{ "metrics", "metrics", true }, .{ "sources", "sources", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, @@ -5282,6 +5334,10 @@ pub const CreateGraphIndexRequest = struct { try jw.objectField("enrichments"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.sources) |value| { try jw.objectField("sources"); try jw.write(value); @@ -6237,6 +6293,7 @@ pub const CreatedGraphIndex = struct { version: ?i64 = null, /// Normalized inline managed enrichment definitions required by this index. enrichments: ?[]const CreatedEnrichmentConfig = null, + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, summarizer: ?CreatedProviderConfig = null, template: ?[]const u8 = null, edge_types: ?[]const EdgeTypeConfig = null, @@ -6254,6 +6311,7 @@ pub const CreatedGraphIndex = struct { .{ "description", "description", true }, .{ "version", "version", true }, .{ "enrichments", "enrichments", true }, + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, .{ "edge_types", "edge_types", true }, @@ -6289,6 +6347,10 @@ pub const CreatedGraphIndex = struct { try jw.objectField("enrichments"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -6329,6 +6391,7 @@ pub const CreatedGraphIndex = struct { /// Credential-free normalized graph configuration returned after creation. pub const CreatedGraphIndexConfig = struct { + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, summarizer: ?CreatedProviderConfig = null, template: ?[]const u8 = null, edge_types: ?[]const EdgeTypeConfig = null, @@ -6341,6 +6404,7 @@ pub const CreatedGraphIndexConfig = struct { /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, .{ "edge_types", "edge_types", true }, @@ -6361,6 +6425,10 @@ pub const CreatedGraphIndexConfig = struct { pub fn jsonStringify(self: @This(), jw: anytype) !void { try jw.beginObject(); + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -12895,6 +12963,10 @@ pub const GlobalStatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?GraphQueries = null, @@ -12940,6 +13012,8 @@ pub const GlobalStatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", true }, + .{ "graph_metric", "graph_metric", true }, + .{ "graph_metric_rerank", "graph_metric_rerank", true }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", true }, .{ "document_renderer", "document_renderer", true }, @@ -13066,6 +13140,14 @@ pub const GlobalStatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(value); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -14272,6 +14354,8 @@ pub const GraphIdentityNodeSelector = struct { /// Configuration for graph index type pub const GraphIndexConfig = struct { + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires index_capabilities.artifact_sources=true and is rejected by serverless deployments. sources: ?[]const GraphArtifactSourceConfig = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) @@ -14291,6 +14375,7 @@ pub const GraphIndexConfig = struct { /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ + .{ "metrics", "metrics", true }, .{ "sources", "sources", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, @@ -14312,6 +14397,10 @@ pub const GraphIndexConfig = struct { pub fn jsonStringify(self: @This(), jw: anytype) !void { try jw.beginObject(); + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.sources) |value| { try jw.objectField("sources"); try jw.write(value); @@ -14454,6 +14543,7 @@ pub const GraphIndexStats = struct { promotion: ?std.json.ArrayHashMap(std.json.Value) = null, /// Algebraic graph execution health for bounded semiring traversal. algebraic_graph: ?std.json.Value = null, + graph_metric_runtime: ?GraphMetricRuntimeStats = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -14509,6 +14599,7 @@ pub const GraphIndexStats = struct { .{ "resolution", "resolution", true }, .{ "promotion", "promotion", true }, .{ "algebraic_graph", "algebraic_graph", true }, + .{ "graph_metric_runtime", "graph_metric_runtime", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -14727,6 +14818,10 @@ pub const GraphIndexStats = struct { try jw.objectField("algebraic_graph"); try jw.write(value); } + if (self.graph_metric_runtime) |value| { + try jw.objectField("graph_metric_runtime"); + try jw.write(value); + } try jw.endObject(); } }; @@ -14866,86 +14961,1007 @@ pub const GraphMatch = struct { try jw.objectField("where"); try jw.write(value); } - if (self.optional) |value| { - try jw.objectField("optional"); + if (self.optional) |value| { + try jw.objectField("optional"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Structural edge expansion from the `from` alias to the `to` alias. Direction defaults to `out`; use `in` to reverse the stored edge or `both` to match an undirected relationship without duplicating stored edges. A fixed single-hop relationship preserves physical self-loops and may bind two distinct aliases to the same node identity. Variable-length expansion uses node-simple paths: a (table, key) identity is visited at most once within one expanded edge path, except when closing onto an already bound target alias for an explicit cycle. Exact distributed and serverless execution rejects planner-required reverse variable expansion when the source tables of unnamed intermediate nodes cannot be proven. Express cross-table multi-hop patterns as explicit single-hop edges with a table-qualified alias at each table boundary. +pub const GraphMatchEdge = struct { + from: GraphIdentifier, + to: GraphIdentifier, + /// Stored-edge direction relative to `from`; defaults to `out`. + direction: ?EdgeDirection = null, + /// Empty or omitted matches every edge type; otherwise at most 64 unique types totaling at most 64 KiB. + types: ?[]const GraphEdgeType = null, + min_hops: ?i64 = null, + max_hops: ?i64 = null, + edge_weight: ?GraphEdgeWeightRange = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "from", "from", false }, + .{ "to", "to", false }, + .{ "direction", "direction", true }, + .{ "types", "types", true }, + .{ "min_hops", "min_hops", true }, + .{ "max_hops", "max_hops", true }, + .{ "edge_weight", "edge_weight", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("from"); + try jw.write(self.from); + try jw.objectField("to"); + try jw.write(self.to); + if (self.direction) |value| { + try jw.objectField("direction"); + try jw.write(value); + } + if (self.types) |value| { + try jw.objectField("types"); + try jw.write(value); + } + if (self.min_hops) |value| { + try jw.objectField("min_hops"); + try jw.write(value); + } + if (self.max_hops) |value| { + try jw.objectField("max_hops"); + try jw.write(value); + } + if (self.edge_weight) |value| { + try jw.objectField("edge_weight"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Declared under an alias of at most 128 Unicode code points. Omit table for the queried table. Declare it for a cross-table alias that may be used as the source of a relationship, including planner-selected reverse expansion of a branched pattern. +pub const GraphMatchNode = struct { + /// Owning table for this alias. Omit for the queried table. + table: ?[]const u8 = null, + /// Non-scoring structured stored-document predicate evaluated for this alias. Serverless execution rejects document filters on aliases qualified with a different table because its published snapshot contains only the queried table. Explicitly qualifying an alias with the queried table is equivalent to omitting `table`. + filter: ?GraphDocumentFilter = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "table", "table", true }, + .{ "filter", "filter", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.table) |value| { + try jw.objectField("table"); + try jw.write(value); + } + if (self.filter) |value| { + try jw.objectField("filter"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMatchOperationLimitExceededError = struct { + status: i32, + @"error": []const u8, + message: []const u8, + retryable: bool, + /// Maximum named MATCH operations accepted in one request. + maximum: i64, + /// Named MATCH operations supplied by the request. + actual: i64, +}; + +/// Conjunctive graph match over the complete authorized source universe. Top-level retrieval queries and filters do not scope that universe; put source constraints on the node named by match.anchor. Results are exact or the request fails; execution never labels a partial aggregate exact. Source anchors are streamed in stable snapshot-pinned pages and charged to the request-wide `scanned_anchors` work dimension; transient expansion state remains bounded, and execution observes request deadlines, cancellation, and server resource admission. Exact distinct identity sets are also bounded and fail closed when their request-scoped memory budget is exhausted. +pub const GraphMatchQuery = struct { + index: []const u8, + match: GraphMatch, + @"return": GraphReturn, +}; + +pub const GraphMetricActionResponse = struct { + status: GraphMetricStatus, +}; + +pub const GraphMetricBuildPageStatus = struct { + phase: []const u8, + iteration: i64, + page_id: i64, + state: []const u8, + range_kind: []const u8, + /// Worker id that owns or last failed this page. + worker_id: ?[]const u8 = null, + /// Unix epoch milliseconds when the page lease expires, or 0 when not leased. + lease_expires_at_ms: ?i64 = null, + /// Current attempt number for this page. + attempt: ?i64 = null, + /// Opaque resumable cursor for this page. + cursor: ?[]const u8 = null, + /// Completed work units for this page. + completed_units: ?i64 = null, + /// Estimated total work units for this page. + total_units: ?i64 = null, + /// Last page-level error. + last_error: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "phase", "phase", false }, + .{ "iteration", "iteration", false }, + .{ "page_id", "page_id", false }, + .{ "state", "state", false }, + .{ "range_kind", "range_kind", false }, + .{ "worker_id", "worker_id", true }, + .{ "lease_expires_at_ms", "lease_expires_at_ms", true }, + .{ "attempt", "attempt", true }, + .{ "cursor", "cursor", true }, + .{ "completed_units", "completed_units", true }, + .{ "total_units", "total_units", true }, + .{ "last_error", "last_error", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("phase"); + try jw.write(self.phase); + try jw.objectField("iteration"); + try jw.write(self.iteration); + try jw.objectField("page_id"); + try jw.write(self.page_id); + try jw.objectField("state"); + try jw.write(self.state); + try jw.objectField("range_kind"); + try jw.write(self.range_kind); + if (self.worker_id) |value| { + try jw.objectField("worker_id"); + try jw.write(value); + } + if (self.lease_expires_at_ms) |value| { + try jw.objectField("lease_expires_at_ms"); + try jw.write(value); + } + if (self.attempt) |value| { + try jw.objectField("attempt"); + try jw.write(value); + } + if (self.cursor) |value| { + try jw.objectField("cursor"); + try jw.write(value); + } + if (self.completed_units) |value| { + try jw.objectField("completed_units"); + try jw.write(value); + } + if (self.total_units) |value| { + try jw.objectField("total_units"); + try jw.write(value); + } + if (self.last_error) |value| { + try jw.objectField("last_error"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Published metric configuration. If kind is omitted, the metric name must be a supported kind. +pub const GraphMetricConfig = struct { + enabled: ?bool = null, + kind: ?[]const u8 = null, + /// Serverless accepts background only. + refresh: ?[]const u8 = null, + damping: ?f64 = null, + tolerance: ?f64 = null, + max_iterations: ?i32 = null, + edge_filter: ?GraphMetricEdgeFilter = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "enabled", "enabled", true }, + .{ "kind", "kind", true }, + .{ "refresh", "refresh", true }, + .{ "damping", "damping", true }, + .{ "tolerance", "tolerance", true }, + .{ "max_iterations", "max_iterations", true }, + .{ "edge_filter", "edge_filter", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.enabled) |value| { + try jw.objectField("enabled"); + try jw.write(value); + } + if (self.kind) |value| { + try jw.objectField("kind"); + try jw.write(value); + } + if (self.refresh) |value| { + try jw.objectField("refresh"); + try jw.write(value); + } + if (self.damping) |value| { + try jw.objectField("damping"); + try jw.write(value); + } + if (self.tolerance) |value| { + try jw.objectField("tolerance"); + try jw.write(value); + } + if (self.max_iterations) |value| { + try jw.objectField("max_iterations"); + try jw.write(value); + } + if (self.edge_filter) |value| { + try jw.objectField("edge_filter"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied. +pub const GraphMetricEdgeFilter = struct { + mode: ?[]const u8 = null, + types: ?[]const GraphEdgeType = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "mode", "mode", true }, + .{ "types", "types", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.mode) |value| { + try jw.objectField("mode"); + try jw.write(value); + } + if (self.types) |value| { + try jw.objectField("types"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricEdgeFilterStatus = struct { + mode: []const u8, + types: ?[]const []const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "mode", "mode", false }, + .{ "types", "types", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("mode"); + try jw.write(self.mode); + if (self.types) |value| { + try jw.objectField("types"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricEvent = struct { + sequence: i64, + kind: []const u8, + at_ms: i64, + target_edge_generation: i64, + published_generation: i64, + score_count: i64, +}; + +pub const GraphMetricFilter = struct { + metric: []const u8, + /// Semantic comparison operator. Named values keep generated SDK enums portable and readable. + op: []const u8, + value: f64, +}; + +pub const GraphMetricOrder = struct { + metric: []const u8, + direction: ?[]const u8 = null, + nulls: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "metric", "metric", false }, + .{ "direction", "direction", true }, + .{ "nulls", "nulls", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.direction) |value| { + try jw.objectField("direction"); + try jw.write(value); + } + if (self.nulls) |value| { + try jw.objectField("nulls"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricProfile = struct { + /// Name of the graph query or graph metric query that used the metric. + query_name: []const u8, + /// Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + source: []const u8, + /// Graph index that owns the metric. + index_name: []const u8, + /// Graph metric name within the index. + metric_name: []const u8, + /// Effective freshness mode requested for this metric use. + freshness: []const u8, + /// Published generation and freshness status observed by the query. + status: GraphMetricStatus, +}; + +/// Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. +pub const GraphMetricQuery = struct { + /// Optional result key. Defaults to the metric name. + name: ?[]const u8 = null, + /// Graph index that owns the published metric. + index: []const u8, + /// Graph metric to read. + metric: []const u8, + /// Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated metric snapshot. + top_k: ?i32 = null, + /// Whether the latest published generation may be stale or must match the graph edge generation. + metric_freshness: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "name", "name", true }, + .{ "index", "index", false }, + .{ "metric", "metric", false }, + .{ "top_k", "top_k", true }, + .{ "metric_freshness", "metric_freshness", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.name) |value| { + try jw.objectField("name"); + try jw.write(value); + } + try jw.objectField("index"); + try jw.write(self.index); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.top_k) |value| { + try jw.objectField("top_k"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. +pub const GraphMetricRerank = struct { + /// Graph index that owns the published metric. + index: []const u8, + /// Graph metric name to blend into the search hit score. + metric: []const u8, + /// Bounded retrieval window scored by the graph metric before offset and limit are applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. An explicit value must cover offset plus limit. Larger windows improve promotion recall at predictable linear score-read cost. + candidate_count: ?i32 = null, + /// Multiplier applied to the existing hit score before adding the graph metric feature. + base_weight: ?f64 = null, + /// Multiplier applied to the graph metric score before it is added to the existing hit score. + weight: ?f64 = null, + /// Metric feature value to use for hits that do not have a score in the published metric generation. + missing_score: ?f64 = null, + /// Whether stale published generations are acceptable or the metric must be fresh. + metric_freshness: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "index", "index", false }, + .{ "metric", "metric", false }, + .{ "candidate_count", "candidate_count", true }, + .{ "base_weight", "base_weight", true }, + .{ "weight", "weight", true }, + .{ "missing_score", "missing_score", true }, + .{ "metric_freshness", "metric_freshness", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("index"); + try jw.write(self.index); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.candidate_count) |value| { + try jw.objectField("candidate_count"); + try jw.write(value); + } + if (self.base_weight) |value| { + try jw.objectField("base_weight"); + try jw.write(value); + } + if (self.weight) |value| { + try jw.objectField("weight"); + try jw.write(value); + } + if (self.missing_score) |value| { + try jw.objectField("missing_score"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricRerankScoreDetails = struct { + /// Graph index that provided the metric score. + index_name: []const u8, + /// Graph metric used as a score feature. + metric_name: []const u8, + /// Hit score before graph metric rerank composition. + base_score: f64, + /// Weight applied to the base score. + base_weight: f64, + /// Published metric score for this hit, or null when the hit was missing from the metric generation. + metric_score: OpenApiOptionalNullable(f64) = .absent, + /// Metric feature value used in the formula after applying missing_score fallback if needed. + metric_score_used: f64, + /// Weight applied to the metric score feature. + metric_weight: f64, + /// True when metric_score was missing and the request's missing_score fallback was used. + missing_score_used: bool, + /// Final hit score after graph metric rerank composition. + final_score: f64, + /// Published graph metric score generation used for this hit. + published_generation: i64, + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("index_name"); + try jw.write(self.index_name); + try jw.objectField("metric_name"); + try jw.write(self.metric_name); + try jw.objectField("base_score"); + try jw.write(self.base_score); + try jw.objectField("base_weight"); + try jw.write(self.base_weight); + switch (self.metric_score) { + .absent => {}, + .null_value => { + try jw.objectField("metric_score"); + try jw.write(@as(?u8, null)); + }, + .value => |value| { + try jw.objectField("metric_score"); + try jw.write(value); + }, + } + try jw.objectField("metric_score_used"); + try jw.write(self.metric_score_used); + try jw.objectField("metric_weight"); + try jw.write(self.metric_weight); + try jw.objectField("missing_score_used"); + try jw.write(self.missing_score_used); + try jw.objectField("final_score"); + try jw.write(self.final_score); + try jw.objectField("published_generation"); + try jw.write(self.published_generation); + try jw.endObject(); + } +}; + +pub const GraphMetricResult = struct { + index_name: []const u8, + metric: []const u8, + scores: []const GraphMetricScore, + status: GraphMetricStatus, +}; + +/// Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers. +pub const GraphMetricRuntimeStats = struct { + enabled: ?bool = null, + role: ?[]const u8 = null, + runtime_id_hash: ?i64 = null, + owner_id_hash: ?i64 = null, + lease_key_hash: ?i64 = null, + worker_id_hash: ?i64 = null, + worker_count: ?i64 = null, + lease_owned: ?bool = null, + has_lease: ?bool = null, + acquisition_count: ?i64 = null, + takeover_count: ?i64 = null, + lease_acquire_failures: ?i64 = null, + lost_leases: ?i64 = null, + last_acquired_ms: ?i64 = null, + /// Cached expiry of the currently held maintenance lease, or zero when no lease is held. + lease_expires_at_ms: ?i64 = null, + /// Earliest time the runtime will renew its maintenance lease, or zero when no lease is held. + lease_renew_after_ms: ?i64 = null, + /// Number of durable maintenance lease renewals completed by this runtime. + renewal_count: ?i64 = null, + started: ?bool = null, + shutdown: ?bool = null, + notified: ?bool = null, + ticks_started: ?i64 = null, + ticks_completed: ?i64 = null, + durable_progress_ticks: ?i64 = null, + idle_ticks: ?i64 = null, + error_ticks: ?i64 = null, + last_error_name: ?[]const u8 = null, + total_metrics_scanned: ?i64 = null, + total_active_builds: ?i64 = null, + total_builds_started: ?i64 = null, + total_worker_steps: ?i64 = null, + total_coordinator_steps: ?i64 = null, + /// Consumed intermediate records retired at completed reduction barriers. + total_retired_input_records: ?i64 = null, + total_pages_claimed: ?i64 = null, + total_pages_completed: ?i64 = null, + total_phases_advanced: ?i64 = null, + total_published: ?i64 = null, + total_failed_builds: ?i64 = null, + last_metrics_scanned: ?i64 = null, + last_active_builds: ?i64 = null, + last_builds_started: ?i64 = null, + last_worker_steps: ?i64 = null, + last_coordinator_steps: ?i64 = null, + /// Consumed intermediate records retired in the latest maintenance tick. + last_retired_input_records: ?i64 = null, + last_pages_claimed: ?i64 = null, + last_pages_completed: ?i64 = null, + last_phases_advanced: ?i64 = null, + last_published: ?i64 = null, + last_failed_builds: ?i64 = null, + last_budget_exhausted: ?bool = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "enabled", "enabled", true }, + .{ "role", "role", true }, + .{ "runtime_id_hash", "runtime_id_hash", true }, + .{ "owner_id_hash", "owner_id_hash", true }, + .{ "lease_key_hash", "lease_key_hash", true }, + .{ "worker_id_hash", "worker_id_hash", true }, + .{ "worker_count", "worker_count", true }, + .{ "lease_owned", "lease_owned", true }, + .{ "has_lease", "has_lease", true }, + .{ "acquisition_count", "acquisition_count", true }, + .{ "takeover_count", "takeover_count", true }, + .{ "lease_acquire_failures", "lease_acquire_failures", true }, + .{ "lost_leases", "lost_leases", true }, + .{ "last_acquired_ms", "last_acquired_ms", true }, + .{ "lease_expires_at_ms", "lease_expires_at_ms", true }, + .{ "lease_renew_after_ms", "lease_renew_after_ms", true }, + .{ "renewal_count", "renewal_count", true }, + .{ "started", "started", true }, + .{ "shutdown", "shutdown", true }, + .{ "notified", "notified", true }, + .{ "ticks_started", "ticks_started", true }, + .{ "ticks_completed", "ticks_completed", true }, + .{ "durable_progress_ticks", "durable_progress_ticks", true }, + .{ "idle_ticks", "idle_ticks", true }, + .{ "error_ticks", "error_ticks", true }, + .{ "last_error_name", "last_error_name", true }, + .{ "total_metrics_scanned", "total_metrics_scanned", true }, + .{ "total_active_builds", "total_active_builds", true }, + .{ "total_builds_started", "total_builds_started", true }, + .{ "total_worker_steps", "total_worker_steps", true }, + .{ "total_coordinator_steps", "total_coordinator_steps", true }, + .{ "total_retired_input_records", "total_retired_input_records", true }, + .{ "total_pages_claimed", "total_pages_claimed", true }, + .{ "total_pages_completed", "total_pages_completed", true }, + .{ "total_phases_advanced", "total_phases_advanced", true }, + .{ "total_published", "total_published", true }, + .{ "total_failed_builds", "total_failed_builds", true }, + .{ "last_metrics_scanned", "last_metrics_scanned", true }, + .{ "last_active_builds", "last_active_builds", true }, + .{ "last_builds_started", "last_builds_started", true }, + .{ "last_worker_steps", "last_worker_steps", true }, + .{ "last_coordinator_steps", "last_coordinator_steps", true }, + .{ "last_retired_input_records", "last_retired_input_records", true }, + .{ "last_pages_claimed", "last_pages_claimed", true }, + .{ "last_pages_completed", "last_pages_completed", true }, + .{ "last_phases_advanced", "last_phases_advanced", true }, + .{ "last_published", "last_published", true }, + .{ "last_failed_builds", "last_failed_builds", true }, + .{ "last_budget_exhausted", "last_budget_exhausted", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.enabled) |value| { + try jw.objectField("enabled"); + try jw.write(value); + } + if (self.role) |value| { + try jw.objectField("role"); + try jw.write(value); + } + if (self.runtime_id_hash) |value| { + try jw.objectField("runtime_id_hash"); + try jw.write(value); + } + if (self.owner_id_hash) |value| { + try jw.objectField("owner_id_hash"); + try jw.write(value); + } + if (self.lease_key_hash) |value| { + try jw.objectField("lease_key_hash"); + try jw.write(value); + } + if (self.worker_id_hash) |value| { + try jw.objectField("worker_id_hash"); + try jw.write(value); + } + if (self.worker_count) |value| { + try jw.objectField("worker_count"); + try jw.write(value); + } + if (self.lease_owned) |value| { + try jw.objectField("lease_owned"); + try jw.write(value); + } + if (self.has_lease) |value| { + try jw.objectField("has_lease"); + try jw.write(value); + } + if (self.acquisition_count) |value| { + try jw.objectField("acquisition_count"); + try jw.write(value); + } + if (self.takeover_count) |value| { + try jw.objectField("takeover_count"); + try jw.write(value); + } + if (self.lease_acquire_failures) |value| { + try jw.objectField("lease_acquire_failures"); + try jw.write(value); + } + if (self.lost_leases) |value| { + try jw.objectField("lost_leases"); + try jw.write(value); + } + if (self.last_acquired_ms) |value| { + try jw.objectField("last_acquired_ms"); + try jw.write(value); + } + if (self.lease_expires_at_ms) |value| { + try jw.objectField("lease_expires_at_ms"); + try jw.write(value); + } + if (self.lease_renew_after_ms) |value| { + try jw.objectField("lease_renew_after_ms"); + try jw.write(value); + } + if (self.renewal_count) |value| { + try jw.objectField("renewal_count"); + try jw.write(value); + } + if (self.started) |value| { + try jw.objectField("started"); + try jw.write(value); + } + if (self.shutdown) |value| { + try jw.objectField("shutdown"); + try jw.write(value); + } + if (self.notified) |value| { + try jw.objectField("notified"); + try jw.write(value); + } + if (self.ticks_started) |value| { + try jw.objectField("ticks_started"); + try jw.write(value); + } + if (self.ticks_completed) |value| { + try jw.objectField("ticks_completed"); + try jw.write(value); + } + if (self.durable_progress_ticks) |value| { + try jw.objectField("durable_progress_ticks"); + try jw.write(value); + } + if (self.idle_ticks) |value| { + try jw.objectField("idle_ticks"); + try jw.write(value); + } + if (self.error_ticks) |value| { + try jw.objectField("error_ticks"); + try jw.write(value); + } + if (self.last_error_name) |value| { + try jw.objectField("last_error_name"); + try jw.write(value); + } + if (self.total_metrics_scanned) |value| { + try jw.objectField("total_metrics_scanned"); + try jw.write(value); + } + if (self.total_active_builds) |value| { + try jw.objectField("total_active_builds"); + try jw.write(value); + } + if (self.total_builds_started) |value| { + try jw.objectField("total_builds_started"); + try jw.write(value); + } + if (self.total_worker_steps) |value| { + try jw.objectField("total_worker_steps"); + try jw.write(value); + } + if (self.total_coordinator_steps) |value| { + try jw.objectField("total_coordinator_steps"); + try jw.write(value); + } + if (self.total_retired_input_records) |value| { + try jw.objectField("total_retired_input_records"); + try jw.write(value); + } + if (self.total_pages_claimed) |value| { + try jw.objectField("total_pages_claimed"); + try jw.write(value); + } + if (self.total_pages_completed) |value| { + try jw.objectField("total_pages_completed"); + try jw.write(value); + } + if (self.total_phases_advanced) |value| { + try jw.objectField("total_phases_advanced"); + try jw.write(value); + } + if (self.total_published) |value| { + try jw.objectField("total_published"); + try jw.write(value); + } + if (self.total_failed_builds) |value| { + try jw.objectField("total_failed_builds"); + try jw.write(value); + } + if (self.last_metrics_scanned) |value| { + try jw.objectField("last_metrics_scanned"); + try jw.write(value); + } + if (self.last_active_builds) |value| { + try jw.objectField("last_active_builds"); + try jw.write(value); + } + if (self.last_builds_started) |value| { + try jw.objectField("last_builds_started"); + try jw.write(value); + } + if (self.last_worker_steps) |value| { + try jw.objectField("last_worker_steps"); try jw.write(value); } - try jw.endObject(); - } -}; - -/// Structural edge expansion from the `from` alias to the `to` alias. Direction defaults to `out`; use `in` to reverse the stored edge or `both` to match an undirected relationship without duplicating stored edges. A fixed single-hop relationship preserves physical self-loops and may bind two distinct aliases to the same node identity. Variable-length expansion uses node-simple paths: a (table, key) identity is visited at most once within one expanded edge path, except when closing onto an already bound target alias for an explicit cycle. Exact distributed and serverless execution rejects planner-required reverse variable expansion when the source tables of unnamed intermediate nodes cannot be proven. Express cross-table multi-hop patterns as explicit single-hop edges with a table-qualified alias at each table boundary. -pub const GraphMatchEdge = struct { - from: GraphIdentifier, - to: GraphIdentifier, - /// Stored-edge direction relative to `from`; defaults to `out`. - direction: ?EdgeDirection = null, - /// Empty or omitted matches every edge type; otherwise at most 64 unique types totaling at most 64 KiB. - types: ?[]const GraphEdgeType = null, - min_hops: ?i64 = null, - max_hops: ?i64 = null, - edge_weight: ?GraphEdgeWeightRange = null, - - /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. - pub const openApiFieldMetadata = .{ - .{ "from", "from", false }, - .{ "to", "to", false }, - .{ "direction", "direction", true }, - .{ "types", "types", true }, - .{ "min_hops", "min_hops", true }, - .{ "max_hops", "max_hops", true }, - .{ "edge_weight", "edge_weight", true }, - }; - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { - return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { - return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); - } - - pub fn jsonStringify(self: @This(), jw: anytype) !void { - try jw.beginObject(); - try jw.objectField("from"); - try jw.write(self.from); - try jw.objectField("to"); - try jw.write(self.to); - if (self.direction) |value| { - try jw.objectField("direction"); + if (self.last_coordinator_steps) |value| { + try jw.objectField("last_coordinator_steps"); try jw.write(value); } - if (self.types) |value| { - try jw.objectField("types"); + if (self.last_retired_input_records) |value| { + try jw.objectField("last_retired_input_records"); try jw.write(value); } - if (self.min_hops) |value| { - try jw.objectField("min_hops"); + if (self.last_pages_claimed) |value| { + try jw.objectField("last_pages_claimed"); try jw.write(value); } - if (self.max_hops) |value| { - try jw.objectField("max_hops"); + if (self.last_pages_completed) |value| { + try jw.objectField("last_pages_completed"); try jw.write(value); } - if (self.edge_weight) |value| { - try jw.objectField("edge_weight"); + if (self.last_phases_advanced) |value| { + try jw.objectField("last_phases_advanced"); + try jw.write(value); + } + if (self.last_published) |value| { + try jw.objectField("last_published"); + try jw.write(value); + } + if (self.last_failed_builds) |value| { + try jw.objectField("last_failed_builds"); + try jw.write(value); + } + if (self.last_budget_exhausted) |value| { + try jw.objectField("last_budget_exhausted"); try jw.write(value); } try jw.endObject(); } }; -/// Declared under an alias of at most 128 Unicode code points. Omit table for the queried table. Declare it for a cross-table alias that may be used as the source of a relationship, including planner-selected reverse expansion of a branched pattern. -pub const GraphMatchNode = struct { - /// Owning table for this alias. Omit for the queried table. - table: ?[]const u8 = null, - /// Non-scoring structured stored-document predicate evaluated for this alias. Serverless execution rejects document filters on aliases qualified with a different table because its published snapshot contains only the queried table. Explicitly qualifying an alias with the queried table is equivalent to omitting `table`. - filter: ?GraphDocumentFilter = null, +pub const GraphMetricScore = struct { + node: []const u8, + score: f64, +}; + +pub const GraphMetricStatus = struct { + state: []const u8, + phase: []const u8, + edge_filter: ?GraphMetricEdgeFilterStatus = null, + /// Version of the published graph metric metadata schema. + metadata_version: ?i64 = null, + /// Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every SDK preserves all 64 bits. + config_fingerprint: ?[]const u8 = null, + maintenance_paused: ?bool = null, + /// Whether a local or distributed build is queued after the currently published or building generation. + build_queued: bool, + published_generation: i64, + edge_generation: i64, + target_edge_generation: i64, + /// Pending edge generation waiting to build, or 0 when no build is queued. + queued_generation: ?i64 = null, + /// Edge generation currently held by an active build lease, or 0 when idle. + building_generation: ?i64 = null, + /// Durable identifier for the active graph metric build job, or 0 when idle. + build_job_id: ?i64 = null, + /// Unix epoch milliseconds when the active graph metric build started, or 0 when idle. + build_started_at_ms: ?i64 = null, + /// Iteration number reported by the active build lease, or 0 when idle or not iterative. + build_iteration: ?i64 = null, + /// Unix epoch milliseconds when the active build lease expires, or 0 when idle. + build_lease_expires_at_ms: ?i64 = null, + /// Worker id that owns the active build lease. Local builds use `local`. + build_worker_id: ?[]const u8 = null, + /// Opaque resumable cursor for the active build phase. Empty or omitted when idle or when the phase has no cursor. + build_cursor: ?[]const u8 = null, + /// Completed work units for the active graph metric build, or 0 when idle or unknown. + build_completed_units: ?i64 = null, + /// Estimated total work units for the active graph metric build, or 0 when idle or unknown. + build_total_units: ?i64 = null, + /// Active leased or failed build pages for the current build phase, capped and ordered by durable page key. + build_pages: ?[]const GraphMetricBuildPageStatus = null, + /// Whether build_pages was capped before every active page could be included. + build_pages_truncated: ?bool = null, + /// Number of consecutive failed build attempts for the current target generation, or 0 when no failure applies. + retry_count: ?i64 = null, + /// Last build error for the current failed target generation. + last_error: ?[]const u8 = null, + /// Build progress for the target edge generation, from 0.0 to 1.0 + progress: f64, + converged: bool, + iterations_completed: i64, + delta: f64, + computed_at_ms: i64, + last_event: ?GraphMetricEvent = null, + /// Recent graph metric events, newest first. + recent_events: ?[]const GraphMetricEvent = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ - .{ "table", "table", true }, - .{ "filter", "filter", true }, + .{ "state", "state", false }, + .{ "phase", "phase", false }, + .{ "edge_filter", "edge_filter", true }, + .{ "metadata_version", "metadata_version", true }, + .{ "config_fingerprint", "config_fingerprint", true }, + .{ "maintenance_paused", "maintenance_paused", true }, + .{ "build_queued", "build_queued", false }, + .{ "published_generation", "published_generation", false }, + .{ "edge_generation", "edge_generation", false }, + .{ "target_edge_generation", "target_edge_generation", false }, + .{ "queued_generation", "queued_generation", true }, + .{ "building_generation", "building_generation", true }, + .{ "build_job_id", "build_job_id", true }, + .{ "build_started_at_ms", "build_started_at_ms", true }, + .{ "build_iteration", "build_iteration", true }, + .{ "build_lease_expires_at_ms", "build_lease_expires_at_ms", true }, + .{ "build_worker_id", "build_worker_id", true }, + .{ "build_cursor", "build_cursor", true }, + .{ "build_completed_units", "build_completed_units", true }, + .{ "build_total_units", "build_total_units", true }, + .{ "build_pages", "build_pages", true }, + .{ "build_pages_truncated", "build_pages_truncated", true }, + .{ "retry_count", "retry_count", true }, + .{ "last_error", "last_error", true }, + .{ "progress", "progress", false }, + .{ "converged", "converged", false }, + .{ "iterations_completed", "iterations_completed", false }, + .{ "delta", "delta", false }, + .{ "computed_at_ms", "computed_at_ms", false }, + .{ "last_event", "last_event", true }, + .{ "recent_events", "recent_events", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -14958,36 +15974,112 @@ pub const GraphMatchNode = struct { pub fn jsonStringify(self: @This(), jw: anytype) !void { try jw.beginObject(); - if (self.table) |value| { - try jw.objectField("table"); + try jw.objectField("state"); + try jw.write(self.state); + try jw.objectField("phase"); + try jw.write(self.phase); + if (self.edge_filter) |value| { + try jw.objectField("edge_filter"); try jw.write(value); } - if (self.filter) |value| { - try jw.objectField("filter"); + if (self.metadata_version) |value| { + try jw.objectField("metadata_version"); + try jw.write(value); + } + if (self.config_fingerprint) |value| { + try jw.objectField("config_fingerprint"); + try jw.write(value); + } + if (self.maintenance_paused) |value| { + try jw.objectField("maintenance_paused"); + try jw.write(value); + } + try jw.objectField("build_queued"); + try jw.write(self.build_queued); + try jw.objectField("published_generation"); + try jw.write(self.published_generation); + try jw.objectField("edge_generation"); + try jw.write(self.edge_generation); + try jw.objectField("target_edge_generation"); + try jw.write(self.target_edge_generation); + if (self.queued_generation) |value| { + try jw.objectField("queued_generation"); + try jw.write(value); + } + if (self.building_generation) |value| { + try jw.objectField("building_generation"); + try jw.write(value); + } + if (self.build_job_id) |value| { + try jw.objectField("build_job_id"); + try jw.write(value); + } + if (self.build_started_at_ms) |value| { + try jw.objectField("build_started_at_ms"); + try jw.write(value); + } + if (self.build_iteration) |value| { + try jw.objectField("build_iteration"); + try jw.write(value); + } + if (self.build_lease_expires_at_ms) |value| { + try jw.objectField("build_lease_expires_at_ms"); + try jw.write(value); + } + if (self.build_worker_id) |value| { + try jw.objectField("build_worker_id"); + try jw.write(value); + } + if (self.build_cursor) |value| { + try jw.objectField("build_cursor"); + try jw.write(value); + } + if (self.build_completed_units) |value| { + try jw.objectField("build_completed_units"); + try jw.write(value); + } + if (self.build_total_units) |value| { + try jw.objectField("build_total_units"); + try jw.write(value); + } + if (self.build_pages) |value| { + try jw.objectField("build_pages"); + try jw.write(value); + } + if (self.build_pages_truncated) |value| { + try jw.objectField("build_pages_truncated"); + try jw.write(value); + } + if (self.retry_count) |value| { + try jw.objectField("retry_count"); + try jw.write(value); + } + if (self.last_error) |value| { + try jw.objectField("last_error"); + try jw.write(value); + } + try jw.objectField("progress"); + try jw.write(self.progress); + try jw.objectField("converged"); + try jw.write(self.converged); + try jw.objectField("iterations_completed"); + try jw.write(self.iterations_completed); + try jw.objectField("delta"); + try jw.write(self.delta); + try jw.objectField("computed_at_ms"); + try jw.write(self.computed_at_ms); + if (self.last_event) |value| { + try jw.objectField("last_event"); + try jw.write(value); + } + if (self.recent_events) |value| { + try jw.objectField("recent_events"); try jw.write(value); } try jw.endObject(); } }; -pub const GraphMatchOperationLimitExceededError = struct { - status: i32, - @"error": []const u8, - message: []const u8, - retryable: bool, - /// Maximum named MATCH operations accepted in one request. - maximum: i64, - /// Named MATCH operations supplied by the request. - actual: i64, -}; - -/// Conjunctive graph match over the complete authorized source universe. Top-level retrieval queries and filters do not scope that universe; put source constraints on the node named by match.anchor. Results are exact or the request fails; execution never labels a partial aggregate exact. Source anchors are streamed in stable snapshot-pinned pages and charged to the request-wide `scanned_anchors` work dimension; transient expansion state remains bounded, and execution observes request deadlines, cancellation, and server resource admission. Exact distinct identity sets are also bounded and fail closed when their request-scoped memory budget is exhausted. -pub const GraphMatchQuery = struct { - index: []const u8, - match: GraphMatch, - @"return": GraphReturn, -}; - /// Select graph nodes using exactly one explicit, exact selector form. pub const GraphNodeSelector = union(enum) { graph_result_ref_node_selector: *GraphResultRefNodeSelector, @@ -15053,7 +16145,40 @@ pub const GraphNodesResult = struct { kind: []const u8, /// Traversal result nodes; requested paths are stored on each node. nodes: []const GraphResultNode, + /// Graph metric status metadata keyed by metric name when requested. + metric_status: ?std.json.ArrayHashMap(GraphMetricStatus) = null, stats: GraphResultStats, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "kind", "kind", false }, + .{ "nodes", "nodes", false }, + .{ "metric_status", "metric_status", true }, + .{ "stats", "stats", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("kind"); + try jw.write(self.kind); + try jw.objectField("nodes"); + try jw.write(self.nodes); + if (self.metric_status) |value| { + try jw.objectField("metric_status"); + try jw.write(value); + } + try jw.objectField("stats"); + try jw.write(self.stats); + try jw.endObject(); + } }; pub const GraphNotEqualPredicate = struct { @@ -15839,6 +16964,8 @@ pub const GraphResultNode = struct { path_edges: ?[]const GraphPathEdge = null, /// Algebraic provenance labels folded into this result, when requested by an algebraic graph executor provenance: ?[]const []const u8 = null, + /// Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score for the node. + metrics: ?std.json.ArrayHashMap(std.json.Value) = null, /// Parsed evidence envelope for provenance labels and edge metadata evidence: ?std.json.ArrayHashMap(std.json.Value) = null, @@ -15851,6 +16978,7 @@ pub const GraphResultNode = struct { .{ "path", "path", true }, .{ "path_edges", "path_edges", true }, .{ "provenance", "provenance", true }, + .{ "metrics", "metrics", true }, .{ "evidence", "evidence", true }, }; @@ -15888,6 +17016,10 @@ pub const GraphResultNode = struct { try jw.objectField("provenance"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.evidence) |value| { try jw.objectField("evidence"); try jw.write(value); @@ -16131,6 +17263,16 @@ pub const GraphTraversal = struct { include_documents: ?bool = null, /// Requires include_documents=true. Omit to include all document fields. fields: ?[]const []const u8 = null, + /// Graph metric names to project onto returned traversal nodes. + metrics: ?[]const []const u8 = null, + /// Sort traversal candidates by graph metric score before applying limit. + order_by: ?[]const GraphMetricOrder = null, + /// Filter traversal candidates by graph metric score before applying limit. + where_metric: ?[]const GraphMetricFilter = null, + /// Freshness required for projected, ordered, and filtered graph metrics. + metric_freshness: ?[]const u8 = null, + /// Include graph metric status metadata in the traversal profile. + include_metric_status: ?bool = null, /// Non-scoring structured stored-document predicate for reached nodes. filter: ?GraphDocumentFilter = null, @@ -16145,6 +17287,11 @@ pub const GraphTraversal = struct { .{ "include_paths", "include_paths", true }, .{ "include_documents", "include_documents", true }, .{ "fields", "fields", true }, + .{ "metrics", "metrics", true }, + .{ "order_by", "order_by", true }, + .{ "where_metric", "where_metric", true }, + .{ "metric_freshness", "metric_freshness", true }, + .{ "include_metric_status", "include_metric_status", true }, .{ "filter", "filter", true }, }; @@ -16192,6 +17339,26 @@ pub const GraphTraversal = struct { try jw.objectField("fields"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } + if (self.order_by) |value| { + try jw.objectField("order_by"); + try jw.write(value); + } + if (self.where_metric) |value| { + try jw.objectField("where_metric"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + if (self.include_metric_status) |value| { + try jw.objectField("include_metric_status"); + try jw.write(value); + } if (self.filter) |value| { try jw.objectField("filter"); try jw.write(value); @@ -16891,6 +18058,8 @@ pub const IndexConfig = struct { chunk_size: ?i64 = null, /// Non-semantic execution policy for shorthand-created chunking or embedding producers. execution: ?IndexExecutionConfig = null, + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) summarizer: ?GeneratorConfig = null, /// List of edge types with their configurations @@ -16932,6 +18101,7 @@ pub const IndexConfig = struct { .{ "min_weight", "min_weight", true }, .{ "chunk_size", "chunk_size", true }, .{ "execution", "execution", true }, + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", true }, .{ "edge_types", "edge_types", true }, .{ "max_edges_per_document", "max_edges_per_document", true }, @@ -17044,6 +18214,10 @@ pub const IndexConfig = struct { try jw.objectField("execution"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -21802,6 +22976,16 @@ pub const LegacyGraphQuery = struct { include_documents: ?bool = null, include_edges: ?bool = null, fields: ?[]const []const u8 = null, + /// Graph metric names to project onto legacy graph_searches result nodes. + metrics: ?[]const []const u8 = null, + /// Sort legacy graph_searches result nodes by graph metric score. + order_by: ?[]const GraphMetricOrder = null, + /// Filter legacy graph_searches result nodes by graph metric score. + where_metric: ?[]const GraphMetricFilter = null, + /// Freshness required for projected, ordered, and filtered graph metrics. + metric_freshness: ?[]const u8 = null, + /// Include graph metric status metadata in the legacy graph_searches result. + include_metric_status: ?bool = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -21815,6 +22999,11 @@ pub const LegacyGraphQuery = struct { .{ "include_documents", "include_documents", true }, .{ "include_edges", "include_edges", true }, .{ "fields", "fields", true }, + .{ "metrics", "metrics", true }, + .{ "order_by", "order_by", true }, + .{ "where_metric", "where_metric", true }, + .{ "metric_freshness", "metric_freshness", true }, + .{ "include_metric_status", "include_metric_status", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -21863,6 +23052,26 @@ pub const LegacyGraphQuery = struct { try jw.objectField("fields"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } + if (self.order_by) |value| { + try jw.objectField("order_by"); + try jw.write(value); + } + if (self.where_metric) |value| { + try jw.objectField("where_metric"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + if (self.include_metric_status) |value| { + try jw.objectField("include_metric_status"); + try jw.write(value); + } try jw.endObject(); } }; @@ -21971,6 +23180,8 @@ pub const LegacyGraphSearchResult = struct { total: i64, /// Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field. took: ?i64 = null, + /// Graph metric status metadata keyed by metric name. + metric_status: ?std.json.ArrayHashMap(GraphMetricStatus) = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -21981,6 +23192,7 @@ pub const LegacyGraphSearchResult = struct { .{ "matches", "matches", true }, .{ "total", "total", false }, .{ "took", "took", true }, + .{ "metric_status", "metric_status", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -22017,6 +23229,10 @@ pub const LegacyGraphSearchResult = struct { try jw.objectField("took"); try jw.write(value); } + if (self.metric_status) |value| { + try jw.objectField("metric_status"); + try jw.write(value); + } try jw.endObject(); } }; @@ -25225,6 +26441,8 @@ pub const QueryHit = struct { _distance: ?f32 = null, /// Scores partitioned by index when using RRF search. _index_scores: ?std.json.ArrayHashMap(f64) = null, + /// Optional score provenance for ranking features applied to this hit. + _score_details: ?QueryScoreDetails = null, _source: ?std.json.ArrayHashMap(std.json.Value) = null, /// Stable ancestry envelope for derived document hierarchy hits. Present when the hit is a derived unit/chunk/embedding artifact or when a source-level group includes nested matches. Standard fields include `level`, `parent_doc_key`, optional `parent_unit_id`, `artifact` or `matched_artifact`, `matches`, and `ancestors` with response-local or requested DB-backed source/unit context when available. V0.2-compatible implicit rollup requests continue to use the deprecated `chunks` field instead of `matches`. hierarchy: ?QueryHitHierarchy = null, @@ -25237,6 +26455,7 @@ pub const QueryHit = struct { .{ "_score", "_score", false }, .{ "_distance", "_distance", true }, .{ "_index_scores", "_index_scores", true }, + .{ "_score_details", "_score_details", true }, .{ "_source", "_source", true }, .{ "hierarchy", "hierarchy", true }, .{ "_sort", "_sort", true }, @@ -25264,6 +26483,10 @@ pub const QueryHit = struct { try jw.objectField("_index_scores"); try jw.write(value); } + if (self._score_details) |value| { + try jw.objectField("_score_details"); + try jw.write(value); + } if (self._source) |value| { try jw.objectField("_source"); try jw.write(value); @@ -25500,6 +26723,8 @@ pub const QueryProfile = struct { reranker: ?RerankerProfile = null, /// Result merge statistics (present for hybrid search). merge: ?MergeProfile = null, + /// Graph metric freshness and generation details for metric-aware query work. + graph_metrics: ?[]const GraphMetricProfile = null, /// Sort execution statistics (present when the query used ordered page options and profiling was enabled). sort: ?SortProfile = null, @@ -25509,6 +26734,7 @@ pub const QueryProfile = struct { .{ "join", "join", true }, .{ "reranker", "reranker", true }, .{ "merge", "merge", true }, + .{ "graph_metrics", "graph_metrics", true }, .{ "sort", "sort", true }, }; @@ -25538,6 +26764,10 @@ pub const QueryProfile = struct { try jw.objectField("merge"); try jw.write(value); } + if (self.graph_metrics) |value| { + try jw.objectField("graph_metrics"); + try jw.write(value); + } if (self.sort) |value| { try jw.objectField("sort"); try jw.write(value); @@ -25600,6 +26830,10 @@ pub const QueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?GraphQueries = null, @@ -25641,6 +26875,8 @@ pub const QueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", true }, + .{ "graph_metric", "graph_metric", true }, + .{ "graph_metric_rerank", "graph_metric_rerank", true }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", true }, .{ "document_renderer", "document_renderer", true }, @@ -25767,6 +27003,14 @@ pub const QueryRequest = struct { try jw.objectField("reranker"); try jw.write(value); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -25829,6 +27073,8 @@ pub const QueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -25846,6 +27092,7 @@ pub const QueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -25876,6 +27123,10 @@ pub const QueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -25907,6 +27158,8 @@ pub const QueryResultBase = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -25923,6 +27176,7 @@ pub const QueryResultBase = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -25952,6 +27206,10 @@ pub const QueryResultBase = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -25972,6 +27230,34 @@ pub const QueryResultBase = struct { } }; +/// Optional score provenance for ranking features that changed the final hit score. +pub const QueryScoreDetails = struct { + /// Score contribution from an explicit graph_metric_rerank request. + graph_metric_rerank: ?GraphMetricRerankScoreDetails = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "graph_metric_rerank", "graph_metric_rerank", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } + try jw.endObject(); + } +}; + /// Strategy for query transformation and retrieval: - simple: Direct query with multi-phrase expansion. Best for straightforward factual queries. - decompose: Break complex queries into sub-questions, retrieve for each. Best for multi-part questions. - step_back: Generate broader background query first, then specific query. Best for questions needing context. - hyde: Generate hypothetical answer document, embed that for retrieval. Best for abstract/conceptual questions. pub const QueryStrategy = enum { simple, @@ -27793,6 +29079,10 @@ pub const RetrievalQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?GraphQueries = null, @@ -27836,6 +29126,8 @@ pub const RetrievalQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", true }, + .{ "graph_metric", "graph_metric", true }, + .{ "graph_metric_rerank", "graph_metric_rerank", true }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", true }, .{ "document_renderer", "document_renderer", true }, @@ -27963,6 +29255,14 @@ pub const RetrievalQueryRequest = struct { try jw.objectField("reranker"); try jw.write(value); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -29132,9 +30432,9 @@ pub const StatefulGraphQueryResults = std.json.ArrayHashMap(StatefulGraphResult) /// Graph result emitted by the stateful compatibility transport. Canonical graph_queries produce GraphResult; deprecated graph_searches may produce LegacyGraphSearchResult during the compatibility window. pub const StatefulGraphResult = union(enum) { + graph_nodes_result: *GraphNodesResult, graph_aggregates_result: *GraphAggregatesResult, graph_bindings_result: *GraphBindingsResult, - graph_nodes_result: *GraphNodesResult, graph_paths_result: *GraphPathsResult, legacy_graph_search_result: *LegacyGraphSearchResult, @@ -29176,9 +30476,9 @@ pub const StatefulGraphResult = union(enum) { const probe = try std.json.parseFromSliceLeaky(Probe, allocator, input, probe_options); switch (probe.kind) { .value => |disc_str| { + if (std.mem.eql(u8, disc_str, "nodes")) return .{ .graph_nodes_result = try parseStructuralVariantFromSlice(GraphNodesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "aggregates")) return .{ .graph_aggregates_result = try parseStructuralVariantFromSlice(GraphAggregatesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "bindings")) return .{ .graph_bindings_result = try parseStructuralVariantFromSlice(GraphBindingsResult, allocator, input, options) }; - if (std.mem.eql(u8, disc_str, "nodes")) return .{ .graph_nodes_result = try parseStructuralVariantFromSlice(GraphNodesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "paths")) return .{ .graph_paths_result = try parseStructuralVariantFromSlice(GraphPathsResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "legacy")) return .{ .legacy_graph_search_result = try parseStructuralVariantFromSlice(LegacyGraphSearchResult, allocator, input, options) }; return error.UnexpectedToken; @@ -29204,6 +30504,10 @@ pub const StatefulGraphResult = union(enum) { .string => |value| value, else => return error.UnexpectedToken, }; + if (std.mem.eql(u8, disc_str, "nodes")) { + const parsed = try parseStructuralVariant(GraphNodesResult, allocator, source, options) orelse return error.UnexpectedToken; + return .{ .graph_nodes_result = parsed }; + } if (std.mem.eql(u8, disc_str, "aggregates")) { const parsed = try parseStructuralVariant(GraphAggregatesResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_aggregates_result = parsed }; @@ -29212,10 +30516,6 @@ pub const StatefulGraphResult = union(enum) { const parsed = try parseStructuralVariant(GraphBindingsResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_bindings_result = parsed }; } - if (std.mem.eql(u8, disc_str, "nodes")) { - const parsed = try parseStructuralVariant(GraphNodesResult, allocator, source, options) orelse return error.UnexpectedToken; - return .{ .graph_nodes_result = parsed }; - } if (std.mem.eql(u8, disc_str, "paths")) { const parsed = try parseStructuralVariant(GraphPathsResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_paths_result = parsed }; @@ -29229,9 +30529,9 @@ pub const StatefulGraphResult = union(enum) { pub fn jsonStringify(self: @This(), jw: anytype) !void { switch (self) { + .graph_nodes_result => |v| try jw.write(v.*), .graph_aggregates_result => |v| try jw.write(v.*), .graph_bindings_result => |v| try jw.write(v.*), - .graph_nodes_result => |v| try jw.write(v.*), .graph_paths_result => |v| try jw.write(v.*), .legacy_graph_search_result => |v| try jw.write(v.*), } @@ -29293,6 +30593,10 @@ pub const StatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?GraphQueries = null, @@ -29338,6 +30642,8 @@ pub const StatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", true }, + .{ "graph_metric", "graph_metric", true }, + .{ "graph_metric_rerank", "graph_metric_rerank", true }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", true }, .{ "document_renderer", "document_renderer", true }, @@ -29466,6 +30772,14 @@ pub const StatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(value); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -29536,6 +30850,8 @@ pub const StatefulQueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -29553,6 +30869,7 @@ pub const StatefulQueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -29583,6 +30900,10 @@ pub const StatefulQueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/root.zig b/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/root.zig index 55fcca0996..a91d84bbd6 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/root.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/root.zig @@ -98,6 +98,19 @@ pub const GraphMatch = types.GraphMatch; pub const GraphMatchEdge = types.GraphMatchEdge; pub const GraphMatchNode = types.GraphMatchNode; pub const GraphMatchQuery = types.GraphMatchQuery; +pub const GraphMetricBuildPageStatus = types.GraphMetricBuildPageStatus; +pub const GraphMetricConfig = types.GraphMetricConfig; +pub const GraphMetricEdgeFilter = types.GraphMetricEdgeFilter; +pub const GraphMetricEdgeFilterStatus = types.GraphMetricEdgeFilterStatus; +pub const GraphMetricEvent = types.GraphMetricEvent; +pub const GraphMetricFilter = types.GraphMetricFilter; +pub const GraphMetricOrder = types.GraphMetricOrder; +pub const GraphMetricQuery = types.GraphMetricQuery; +pub const GraphMetricRerank = types.GraphMetricRerank; +pub const GraphMetricResult = types.GraphMetricResult; +pub const GraphMetricRuntimeStats = types.GraphMetricRuntimeStats; +pub const GraphMetricScore = types.GraphMetricScore; +pub const GraphMetricStatus = types.GraphMetricStatus; pub const GraphNodeSelector = types.GraphNodeSelector; pub const GraphNodesResult = types.GraphNodesResult; pub const GraphNotEqualPredicate = types.GraphNotEqualPredicate; diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/types.zig b/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/types.zig index 33f8da09c2..4a478d94ea 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/types.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_indexes_openapi/types.zig @@ -915,6 +915,8 @@ pub const CreateGraphIndexRequest = struct { version: ?i64 = null, /// Inline managed enrichment definitions required by this index. enrichments: ?[]const EnrichmentConfig = null, + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires index_capabilities.artifact_sources=true and is rejected by serverless deployments. sources: ?[]const GraphArtifactSourceConfig = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) @@ -938,6 +940,7 @@ pub const CreateGraphIndexRequest = struct { .{ "description", "description", true }, .{ "version", "version", true }, .{ "enrichments", "enrichments", true }, + .{ "metrics", "metrics", true }, .{ "sources", "sources", true }, .{ "summarizer", "summarizer", false }, .{ "template", "template", true }, @@ -972,6 +975,10 @@ pub const CreateGraphIndexRequest = struct { try jw.objectField("enrichments"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.sources) |value| { try jw.objectField("sources"); try jw.write(value); @@ -1816,6 +1823,7 @@ pub const CreatedGraphIndex = struct { version: ?i64 = null, /// Normalized inline managed enrichment definitions required by this index. enrichments: ?[]const CreatedEnrichmentConfig = null, + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, summarizer: ?CreatedProviderConfig = null, template: ?[]const u8 = null, edge_types: ?[]const EdgeTypeConfig = null, @@ -1833,6 +1841,7 @@ pub const CreatedGraphIndex = struct { .{ "description", "description", true }, .{ "version", "version", true }, .{ "enrichments", "enrichments", true }, + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, .{ "edge_types", "edge_types", true }, @@ -1868,6 +1877,10 @@ pub const CreatedGraphIndex = struct { try jw.objectField("enrichments"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -1908,6 +1921,7 @@ pub const CreatedGraphIndex = struct { /// Credential-free normalized graph configuration returned after creation. pub const CreatedGraphIndexConfig = struct { + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, summarizer: ?CreatedProviderConfig = null, template: ?[]const u8 = null, edge_types: ?[]const EdgeTypeConfig = null, @@ -1920,6 +1934,7 @@ pub const CreatedGraphIndexConfig = struct { /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", true }, .{ "template", "template", true }, .{ "edge_types", "edge_types", true }, @@ -1940,6 +1955,10 @@ pub const CreatedGraphIndexConfig = struct { pub fn jsonStringify(self: @This(), jw: anytype) !void { try jw.beginObject(); + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -5174,6 +5193,8 @@ pub const GraphIdentityNodeSelector = struct { /// Configuration for graph index type pub const GraphIndexConfig = struct { + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Ordered chunk or JSON asset streams whose edge-like values are unioned into this graph index. Artifact names must be unique within the array because the artifact name is the source identity. Earlier sources win when multiple sources materialize the same edge identity. Requires index_capabilities.artifact_sources=true and is rejected by serverless deployments. sources: ?[]const GraphArtifactSourceConfig = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) @@ -5193,6 +5214,7 @@ pub const GraphIndexConfig = struct { /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ + .{ "metrics", "metrics", true }, .{ "sources", "sources", true }, .{ "summarizer", "summarizer", false }, .{ "template", "template", true }, @@ -5214,6 +5236,10 @@ pub const GraphIndexConfig = struct { pub fn jsonStringify(self: @This(), jw: anytype) !void { try jw.beginObject(); + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.sources) |value| { try jw.objectField("sources"); try jw.write(value); @@ -5359,6 +5385,7 @@ pub const GraphIndexStats = struct { promotion: ?std.json.ArrayHashMap(std.json.Value) = null, /// Algebraic graph execution health for bounded semiring traversal. algebraic_graph: ?std.json.Value = null, + graph_metric_runtime: ?GraphMetricRuntimeStats = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -5414,6 +5441,7 @@ pub const GraphIndexStats = struct { .{ "resolution", "resolution", true }, .{ "promotion", "promotion", true }, .{ "algebraic_graph", "algebraic_graph", true }, + .{ "graph_metric_runtime", "graph_metric_runtime", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -5632,6 +5660,10 @@ pub const GraphIndexStats = struct { try jw.objectField("algebraic_graph"); try jw.write(value); } + if (self.graph_metric_runtime) |value| { + try jw.objectField("graph_metric_runtime"); + try jw.write(value); + } try jw.endObject(); } }; @@ -5882,6 +5914,927 @@ pub const GraphMatchQuery = struct { @"return": GraphReturn, }; +pub const GraphMetricBuildPageStatus = struct { + phase: []const u8, + iteration: i64, + page_id: i64, + state: []const u8, + range_kind: []const u8, + /// Worker id that owns or last failed this page. + worker_id: ?[]const u8 = null, + /// Unix epoch milliseconds when the page lease expires, or 0 when not leased. + lease_expires_at_ms: ?i64 = null, + /// Current attempt number for this page. + attempt: ?i64 = null, + /// Opaque resumable cursor for this page. + cursor: ?[]const u8 = null, + /// Completed work units for this page. + completed_units: ?i64 = null, + /// Estimated total work units for this page. + total_units: ?i64 = null, + /// Last page-level error. + last_error: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "phase", "phase", false }, + .{ "iteration", "iteration", false }, + .{ "page_id", "page_id", false }, + .{ "state", "state", false }, + .{ "range_kind", "range_kind", false }, + .{ "worker_id", "worker_id", true }, + .{ "lease_expires_at_ms", "lease_expires_at_ms", true }, + .{ "attempt", "attempt", true }, + .{ "cursor", "cursor", true }, + .{ "completed_units", "completed_units", true }, + .{ "total_units", "total_units", true }, + .{ "last_error", "last_error", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("phase"); + try jw.write(self.phase); + try jw.objectField("iteration"); + try jw.write(self.iteration); + try jw.objectField("page_id"); + try jw.write(self.page_id); + try jw.objectField("state"); + try jw.write(self.state); + try jw.objectField("range_kind"); + try jw.write(self.range_kind); + if (self.worker_id) |value| { + try jw.objectField("worker_id"); + try jw.write(value); + } + if (self.lease_expires_at_ms) |value| { + try jw.objectField("lease_expires_at_ms"); + try jw.write(value); + } + if (self.attempt) |value| { + try jw.objectField("attempt"); + try jw.write(value); + } + if (self.cursor) |value| { + try jw.objectField("cursor"); + try jw.write(value); + } + if (self.completed_units) |value| { + try jw.objectField("completed_units"); + try jw.write(value); + } + if (self.total_units) |value| { + try jw.objectField("total_units"); + try jw.write(value); + } + if (self.last_error) |value| { + try jw.objectField("last_error"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Published metric configuration. If kind is omitted, the metric name must be a supported kind. +pub const GraphMetricConfig = struct { + enabled: ?bool = null, + kind: ?[]const u8 = null, + /// Serverless accepts background only. + refresh: ?[]const u8 = null, + damping: ?f64 = null, + tolerance: ?f64 = null, + max_iterations: ?i32 = null, + edge_filter: ?GraphMetricEdgeFilter = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "enabled", "enabled", true }, + .{ "kind", "kind", true }, + .{ "refresh", "refresh", true }, + .{ "damping", "damping", true }, + .{ "tolerance", "tolerance", true }, + .{ "max_iterations", "max_iterations", true }, + .{ "edge_filter", "edge_filter", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.enabled) |value| { + try jw.objectField("enabled"); + try jw.write(value); + } + if (self.kind) |value| { + try jw.objectField("kind"); + try jw.write(value); + } + if (self.refresh) |value| { + try jw.objectField("refresh"); + try jw.write(value); + } + if (self.damping) |value| { + try jw.objectField("damping"); + try jw.write(value); + } + if (self.tolerance) |value| { + try jw.objectField("tolerance"); + try jw.write(value); + } + if (self.max_iterations) |value| { + try jw.objectField("max_iterations"); + try jw.write(value); + } + if (self.edge_filter) |value| { + try jw.objectField("edge_filter"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Omitting this object selects all edge types. A types list selects only those types; mode and types cannot both be supplied. +pub const GraphMetricEdgeFilter = struct { + mode: ?[]const u8 = null, + types: ?[]const GraphEdgeType = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "mode", "mode", true }, + .{ "types", "types", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.mode) |value| { + try jw.objectField("mode"); + try jw.write(value); + } + if (self.types) |value| { + try jw.objectField("types"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricEdgeFilterStatus = struct { + mode: []const u8, + types: ?[]const []const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "mode", "mode", false }, + .{ "types", "types", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("mode"); + try jw.write(self.mode); + if (self.types) |value| { + try jw.objectField("types"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricEvent = struct { + sequence: i64, + kind: []const u8, + at_ms: i64, + target_edge_generation: i64, + published_generation: i64, + score_count: i64, +}; + +pub const GraphMetricFilter = struct { + metric: []const u8, + /// Semantic comparison operator. Named values keep generated SDK enums portable and readable. + op: []const u8, + value: f64, +}; + +pub const GraphMetricOrder = struct { + metric: []const u8, + direction: ?[]const u8 = null, + nulls: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "metric", "metric", false }, + .{ "direction", "direction", true }, + .{ "nulls", "nulls", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.direction) |value| { + try jw.objectField("direction"); + try jw.write(value); + } + if (self.nulls) |value| { + try jw.objectField("nulls"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Reads a published graph metric. Score-bearing graph metric queries on multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required instead of merging mathematically incompatible shard-local scores. +pub const GraphMetricQuery = struct { + /// Optional result key. Defaults to the metric name. + name: ?[]const u8 = null, + /// Graph index that owns the published metric. + index: []const u8, + /// Graph metric to read. + metric: []const u8, + /// Maximum ranked metric scores to return. Multi-shard tables require a globally coordinated metric snapshot. + top_k: ?i32 = null, + /// Whether the latest published generation may be stale or must match the graph edge generation. + metric_freshness: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "name", "name", true }, + .{ "index", "index", false }, + .{ "metric", "metric", false }, + .{ "top_k", "top_k", true }, + .{ "metric_freshness", "metric_freshness", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.name) |value| { + try jw.objectField("name"); + try jw.write(value); + } + try jw.objectField("index"); + try jw.write(self.index); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.top_k) |value| { + try jw.objectField("top_k"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +/// Blends a published graph metric into hit scores. Multi-shard tables require a globally coordinated metric snapshot and otherwise return graph_metric_global_materialization_required. +pub const GraphMetricRerank = struct { + /// Graph index that owns the published metric. + index: []const u8, + /// Graph metric name to blend into the search hit score. + metric: []const u8, + /// Bounded retrieval window scored by the graph metric before offset and limit are applied. When omitted, Antfly uses an adaptive four-times page window, capped at 10,000 candidates. An explicit value must cover offset plus limit. Larger windows improve promotion recall at predictable linear score-read cost. + candidate_count: ?i32 = null, + /// Multiplier applied to the existing hit score before adding the graph metric feature. + base_weight: ?f64 = null, + /// Multiplier applied to the graph metric score before it is added to the existing hit score. + weight: ?f64 = null, + /// Metric feature value to use for hits that do not have a score in the published metric generation. + missing_score: ?f64 = null, + /// Whether stale published generations are acceptable or the metric must be fresh. + metric_freshness: ?[]const u8 = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "index", "index", false }, + .{ "metric", "metric", false }, + .{ "candidate_count", "candidate_count", true }, + .{ "base_weight", "base_weight", true }, + .{ "weight", "weight", true }, + .{ "missing_score", "missing_score", true }, + .{ "metric_freshness", "metric_freshness", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("index"); + try jw.write(self.index); + try jw.objectField("metric"); + try jw.write(self.metric); + if (self.candidate_count) |value| { + try jw.objectField("candidate_count"); + try jw.write(value); + } + if (self.base_weight) |value| { + try jw.objectField("base_weight"); + try jw.write(value); + } + if (self.weight) |value| { + try jw.objectField("weight"); + try jw.write(value); + } + if (self.missing_score) |value| { + try jw.objectField("missing_score"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricResult = struct { + index_name: []const u8, + metric: []const u8, + scores: []const GraphMetricScore, + status: GraphMetricStatus, +}; + +/// Summarized graph metric maintenance runtime state. Identity fields are stable hashes, not raw process or owner identifiers. +pub const GraphMetricRuntimeStats = struct { + enabled: ?bool = null, + role: ?[]const u8 = null, + runtime_id_hash: ?i64 = null, + owner_id_hash: ?i64 = null, + lease_key_hash: ?i64 = null, + worker_id_hash: ?i64 = null, + worker_count: ?i64 = null, + lease_owned: ?bool = null, + has_lease: ?bool = null, + acquisition_count: ?i64 = null, + takeover_count: ?i64 = null, + lease_acquire_failures: ?i64 = null, + lost_leases: ?i64 = null, + last_acquired_ms: ?i64 = null, + /// Cached expiry of the currently held maintenance lease, or zero when no lease is held. + lease_expires_at_ms: ?i64 = null, + /// Earliest time the runtime will renew its maintenance lease, or zero when no lease is held. + lease_renew_after_ms: ?i64 = null, + /// Number of durable maintenance lease renewals completed by this runtime. + renewal_count: ?i64 = null, + started: ?bool = null, + shutdown: ?bool = null, + notified: ?bool = null, + ticks_started: ?i64 = null, + ticks_completed: ?i64 = null, + durable_progress_ticks: ?i64 = null, + idle_ticks: ?i64 = null, + error_ticks: ?i64 = null, + last_error_name: ?[]const u8 = null, + total_metrics_scanned: ?i64 = null, + total_active_builds: ?i64 = null, + total_builds_started: ?i64 = null, + total_worker_steps: ?i64 = null, + total_coordinator_steps: ?i64 = null, + /// Consumed intermediate records retired at completed reduction barriers. + total_retired_input_records: ?i64 = null, + total_pages_claimed: ?i64 = null, + total_pages_completed: ?i64 = null, + total_phases_advanced: ?i64 = null, + total_published: ?i64 = null, + total_failed_builds: ?i64 = null, + last_metrics_scanned: ?i64 = null, + last_active_builds: ?i64 = null, + last_builds_started: ?i64 = null, + last_worker_steps: ?i64 = null, + last_coordinator_steps: ?i64 = null, + /// Consumed intermediate records retired in the latest maintenance tick. + last_retired_input_records: ?i64 = null, + last_pages_claimed: ?i64 = null, + last_pages_completed: ?i64 = null, + last_phases_advanced: ?i64 = null, + last_published: ?i64 = null, + last_failed_builds: ?i64 = null, + last_budget_exhausted: ?bool = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "enabled", "enabled", true }, + .{ "role", "role", true }, + .{ "runtime_id_hash", "runtime_id_hash", true }, + .{ "owner_id_hash", "owner_id_hash", true }, + .{ "lease_key_hash", "lease_key_hash", true }, + .{ "worker_id_hash", "worker_id_hash", true }, + .{ "worker_count", "worker_count", true }, + .{ "lease_owned", "lease_owned", true }, + .{ "has_lease", "has_lease", true }, + .{ "acquisition_count", "acquisition_count", true }, + .{ "takeover_count", "takeover_count", true }, + .{ "lease_acquire_failures", "lease_acquire_failures", true }, + .{ "lost_leases", "lost_leases", true }, + .{ "last_acquired_ms", "last_acquired_ms", true }, + .{ "lease_expires_at_ms", "lease_expires_at_ms", true }, + .{ "lease_renew_after_ms", "lease_renew_after_ms", true }, + .{ "renewal_count", "renewal_count", true }, + .{ "started", "started", true }, + .{ "shutdown", "shutdown", true }, + .{ "notified", "notified", true }, + .{ "ticks_started", "ticks_started", true }, + .{ "ticks_completed", "ticks_completed", true }, + .{ "durable_progress_ticks", "durable_progress_ticks", true }, + .{ "idle_ticks", "idle_ticks", true }, + .{ "error_ticks", "error_ticks", true }, + .{ "last_error_name", "last_error_name", true }, + .{ "total_metrics_scanned", "total_metrics_scanned", true }, + .{ "total_active_builds", "total_active_builds", true }, + .{ "total_builds_started", "total_builds_started", true }, + .{ "total_worker_steps", "total_worker_steps", true }, + .{ "total_coordinator_steps", "total_coordinator_steps", true }, + .{ "total_retired_input_records", "total_retired_input_records", true }, + .{ "total_pages_claimed", "total_pages_claimed", true }, + .{ "total_pages_completed", "total_pages_completed", true }, + .{ "total_phases_advanced", "total_phases_advanced", true }, + .{ "total_published", "total_published", true }, + .{ "total_failed_builds", "total_failed_builds", true }, + .{ "last_metrics_scanned", "last_metrics_scanned", true }, + .{ "last_active_builds", "last_active_builds", true }, + .{ "last_builds_started", "last_builds_started", true }, + .{ "last_worker_steps", "last_worker_steps", true }, + .{ "last_coordinator_steps", "last_coordinator_steps", true }, + .{ "last_retired_input_records", "last_retired_input_records", true }, + .{ "last_pages_claimed", "last_pages_claimed", true }, + .{ "last_pages_completed", "last_pages_completed", true }, + .{ "last_phases_advanced", "last_phases_advanced", true }, + .{ "last_published", "last_published", true }, + .{ "last_failed_builds", "last_failed_builds", true }, + .{ "last_budget_exhausted", "last_budget_exhausted", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.enabled) |value| { + try jw.objectField("enabled"); + try jw.write(value); + } + if (self.role) |value| { + try jw.objectField("role"); + try jw.write(value); + } + if (self.runtime_id_hash) |value| { + try jw.objectField("runtime_id_hash"); + try jw.write(value); + } + if (self.owner_id_hash) |value| { + try jw.objectField("owner_id_hash"); + try jw.write(value); + } + if (self.lease_key_hash) |value| { + try jw.objectField("lease_key_hash"); + try jw.write(value); + } + if (self.worker_id_hash) |value| { + try jw.objectField("worker_id_hash"); + try jw.write(value); + } + if (self.worker_count) |value| { + try jw.objectField("worker_count"); + try jw.write(value); + } + if (self.lease_owned) |value| { + try jw.objectField("lease_owned"); + try jw.write(value); + } + if (self.has_lease) |value| { + try jw.objectField("has_lease"); + try jw.write(value); + } + if (self.acquisition_count) |value| { + try jw.objectField("acquisition_count"); + try jw.write(value); + } + if (self.takeover_count) |value| { + try jw.objectField("takeover_count"); + try jw.write(value); + } + if (self.lease_acquire_failures) |value| { + try jw.objectField("lease_acquire_failures"); + try jw.write(value); + } + if (self.lost_leases) |value| { + try jw.objectField("lost_leases"); + try jw.write(value); + } + if (self.last_acquired_ms) |value| { + try jw.objectField("last_acquired_ms"); + try jw.write(value); + } + if (self.lease_expires_at_ms) |value| { + try jw.objectField("lease_expires_at_ms"); + try jw.write(value); + } + if (self.lease_renew_after_ms) |value| { + try jw.objectField("lease_renew_after_ms"); + try jw.write(value); + } + if (self.renewal_count) |value| { + try jw.objectField("renewal_count"); + try jw.write(value); + } + if (self.started) |value| { + try jw.objectField("started"); + try jw.write(value); + } + if (self.shutdown) |value| { + try jw.objectField("shutdown"); + try jw.write(value); + } + if (self.notified) |value| { + try jw.objectField("notified"); + try jw.write(value); + } + if (self.ticks_started) |value| { + try jw.objectField("ticks_started"); + try jw.write(value); + } + if (self.ticks_completed) |value| { + try jw.objectField("ticks_completed"); + try jw.write(value); + } + if (self.durable_progress_ticks) |value| { + try jw.objectField("durable_progress_ticks"); + try jw.write(value); + } + if (self.idle_ticks) |value| { + try jw.objectField("idle_ticks"); + try jw.write(value); + } + if (self.error_ticks) |value| { + try jw.objectField("error_ticks"); + try jw.write(value); + } + if (self.last_error_name) |value| { + try jw.objectField("last_error_name"); + try jw.write(value); + } + if (self.total_metrics_scanned) |value| { + try jw.objectField("total_metrics_scanned"); + try jw.write(value); + } + if (self.total_active_builds) |value| { + try jw.objectField("total_active_builds"); + try jw.write(value); + } + if (self.total_builds_started) |value| { + try jw.objectField("total_builds_started"); + try jw.write(value); + } + if (self.total_worker_steps) |value| { + try jw.objectField("total_worker_steps"); + try jw.write(value); + } + if (self.total_coordinator_steps) |value| { + try jw.objectField("total_coordinator_steps"); + try jw.write(value); + } + if (self.total_retired_input_records) |value| { + try jw.objectField("total_retired_input_records"); + try jw.write(value); + } + if (self.total_pages_claimed) |value| { + try jw.objectField("total_pages_claimed"); + try jw.write(value); + } + if (self.total_pages_completed) |value| { + try jw.objectField("total_pages_completed"); + try jw.write(value); + } + if (self.total_phases_advanced) |value| { + try jw.objectField("total_phases_advanced"); + try jw.write(value); + } + if (self.total_published) |value| { + try jw.objectField("total_published"); + try jw.write(value); + } + if (self.total_failed_builds) |value| { + try jw.objectField("total_failed_builds"); + try jw.write(value); + } + if (self.last_metrics_scanned) |value| { + try jw.objectField("last_metrics_scanned"); + try jw.write(value); + } + if (self.last_active_builds) |value| { + try jw.objectField("last_active_builds"); + try jw.write(value); + } + if (self.last_builds_started) |value| { + try jw.objectField("last_builds_started"); + try jw.write(value); + } + if (self.last_worker_steps) |value| { + try jw.objectField("last_worker_steps"); + try jw.write(value); + } + if (self.last_coordinator_steps) |value| { + try jw.objectField("last_coordinator_steps"); + try jw.write(value); + } + if (self.last_retired_input_records) |value| { + try jw.objectField("last_retired_input_records"); + try jw.write(value); + } + if (self.last_pages_claimed) |value| { + try jw.objectField("last_pages_claimed"); + try jw.write(value); + } + if (self.last_pages_completed) |value| { + try jw.objectField("last_pages_completed"); + try jw.write(value); + } + if (self.last_phases_advanced) |value| { + try jw.objectField("last_phases_advanced"); + try jw.write(value); + } + if (self.last_published) |value| { + try jw.objectField("last_published"); + try jw.write(value); + } + if (self.last_failed_builds) |value| { + try jw.objectField("last_failed_builds"); + try jw.write(value); + } + if (self.last_budget_exhausted) |value| { + try jw.objectField("last_budget_exhausted"); + try jw.write(value); + } + try jw.endObject(); + } +}; + +pub const GraphMetricScore = struct { + node: []const u8, + score: f64, +}; + +pub const GraphMetricStatus = struct { + state: []const u8, + phase: []const u8, + edge_filter: ?GraphMetricEdgeFilterStatus = null, + /// Version of the published graph metric metadata schema. + metadata_version: ?i64 = null, + /// Deterministic configuration fingerprint encoded as fixed-width hexadecimal so every SDK preserves all 64 bits. + config_fingerprint: ?[]const u8 = null, + maintenance_paused: ?bool = null, + /// Whether a local or distributed build is queued after the currently published or building generation. + build_queued: bool, + published_generation: i64, + edge_generation: i64, + target_edge_generation: i64, + /// Pending edge generation waiting to build, or 0 when no build is queued. + queued_generation: ?i64 = null, + /// Edge generation currently held by an active build lease, or 0 when idle. + building_generation: ?i64 = null, + /// Durable identifier for the active graph metric build job, or 0 when idle. + build_job_id: ?i64 = null, + /// Unix epoch milliseconds when the active graph metric build started, or 0 when idle. + build_started_at_ms: ?i64 = null, + /// Iteration number reported by the active build lease, or 0 when idle or not iterative. + build_iteration: ?i64 = null, + /// Unix epoch milliseconds when the active build lease expires, or 0 when idle. + build_lease_expires_at_ms: ?i64 = null, + /// Worker id that owns the active build lease. Local builds use `local`. + build_worker_id: ?[]const u8 = null, + /// Opaque resumable cursor for the active build phase. Empty or omitted when idle or when the phase has no cursor. + build_cursor: ?[]const u8 = null, + /// Completed work units for the active graph metric build, or 0 when idle or unknown. + build_completed_units: ?i64 = null, + /// Estimated total work units for the active graph metric build, or 0 when idle or unknown. + build_total_units: ?i64 = null, + /// Active leased or failed build pages for the current build phase, capped and ordered by durable page key. + build_pages: ?[]const GraphMetricBuildPageStatus = null, + /// Whether build_pages was capped before every active page could be included. + build_pages_truncated: ?bool = null, + /// Number of consecutive failed build attempts for the current target generation, or 0 when no failure applies. + retry_count: ?i64 = null, + /// Last build error for the current failed target generation. + last_error: ?[]const u8 = null, + /// Build progress for the target edge generation, from 0.0 to 1.0 + progress: f64, + converged: bool, + iterations_completed: i64, + delta: f64, + computed_at_ms: i64, + last_event: ?GraphMetricEvent = null, + /// Recent graph metric events, newest first. + recent_events: ?[]const GraphMetricEvent = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "state", "state", false }, + .{ "phase", "phase", false }, + .{ "edge_filter", "edge_filter", true }, + .{ "metadata_version", "metadata_version", true }, + .{ "config_fingerprint", "config_fingerprint", true }, + .{ "maintenance_paused", "maintenance_paused", true }, + .{ "build_queued", "build_queued", false }, + .{ "published_generation", "published_generation", false }, + .{ "edge_generation", "edge_generation", false }, + .{ "target_edge_generation", "target_edge_generation", false }, + .{ "queued_generation", "queued_generation", true }, + .{ "building_generation", "building_generation", true }, + .{ "build_job_id", "build_job_id", true }, + .{ "build_started_at_ms", "build_started_at_ms", true }, + .{ "build_iteration", "build_iteration", true }, + .{ "build_lease_expires_at_ms", "build_lease_expires_at_ms", true }, + .{ "build_worker_id", "build_worker_id", true }, + .{ "build_cursor", "build_cursor", true }, + .{ "build_completed_units", "build_completed_units", true }, + .{ "build_total_units", "build_total_units", true }, + .{ "build_pages", "build_pages", true }, + .{ "build_pages_truncated", "build_pages_truncated", true }, + .{ "retry_count", "retry_count", true }, + .{ "last_error", "last_error", true }, + .{ "progress", "progress", false }, + .{ "converged", "converged", false }, + .{ "iterations_completed", "iterations_completed", false }, + .{ "delta", "delta", false }, + .{ "computed_at_ms", "computed_at_ms", false }, + .{ "last_event", "last_event", true }, + .{ "recent_events", "recent_events", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("state"); + try jw.write(self.state); + try jw.objectField("phase"); + try jw.write(self.phase); + if (self.edge_filter) |value| { + try jw.objectField("edge_filter"); + try jw.write(value); + } + if (self.metadata_version) |value| { + try jw.objectField("metadata_version"); + try jw.write(value); + } + if (self.config_fingerprint) |value| { + try jw.objectField("config_fingerprint"); + try jw.write(value); + } + if (self.maintenance_paused) |value| { + try jw.objectField("maintenance_paused"); + try jw.write(value); + } + try jw.objectField("build_queued"); + try jw.write(self.build_queued); + try jw.objectField("published_generation"); + try jw.write(self.published_generation); + try jw.objectField("edge_generation"); + try jw.write(self.edge_generation); + try jw.objectField("target_edge_generation"); + try jw.write(self.target_edge_generation); + if (self.queued_generation) |value| { + try jw.objectField("queued_generation"); + try jw.write(value); + } + if (self.building_generation) |value| { + try jw.objectField("building_generation"); + try jw.write(value); + } + if (self.build_job_id) |value| { + try jw.objectField("build_job_id"); + try jw.write(value); + } + if (self.build_started_at_ms) |value| { + try jw.objectField("build_started_at_ms"); + try jw.write(value); + } + if (self.build_iteration) |value| { + try jw.objectField("build_iteration"); + try jw.write(value); + } + if (self.build_lease_expires_at_ms) |value| { + try jw.objectField("build_lease_expires_at_ms"); + try jw.write(value); + } + if (self.build_worker_id) |value| { + try jw.objectField("build_worker_id"); + try jw.write(value); + } + if (self.build_cursor) |value| { + try jw.objectField("build_cursor"); + try jw.write(value); + } + if (self.build_completed_units) |value| { + try jw.objectField("build_completed_units"); + try jw.write(value); + } + if (self.build_total_units) |value| { + try jw.objectField("build_total_units"); + try jw.write(value); + } + if (self.build_pages) |value| { + try jw.objectField("build_pages"); + try jw.write(value); + } + if (self.build_pages_truncated) |value| { + try jw.objectField("build_pages_truncated"); + try jw.write(value); + } + if (self.retry_count) |value| { + try jw.objectField("retry_count"); + try jw.write(value); + } + if (self.last_error) |value| { + try jw.objectField("last_error"); + try jw.write(value); + } + try jw.objectField("progress"); + try jw.write(self.progress); + try jw.objectField("converged"); + try jw.write(self.converged); + try jw.objectField("iterations_completed"); + try jw.write(self.iterations_completed); + try jw.objectField("delta"); + try jw.write(self.delta); + try jw.objectField("computed_at_ms"); + try jw.write(self.computed_at_ms); + if (self.last_event) |value| { + try jw.objectField("last_event"); + try jw.write(value); + } + if (self.recent_events) |value| { + try jw.objectField("recent_events"); + try jw.write(value); + } + try jw.endObject(); + } +}; + /// Select graph nodes using exactly one explicit, exact selector form. pub const GraphNodeSelector = union(enum) { graph_result_ref_node_selector: *GraphResultRefNodeSelector, @@ -5947,7 +6900,40 @@ pub const GraphNodesResult = struct { kind: []const u8, /// Traversal result nodes; requested paths are stored on each node. nodes: []const GraphResultNode, + /// Graph metric status metadata keyed by metric name when requested. + metric_status: ?std.json.ArrayHashMap(GraphMetricStatus) = null, stats: GraphResultStats, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "kind", "kind", false }, + .{ "nodes", "nodes", false }, + .{ "metric_status", "metric_status", true }, + .{ "stats", "stats", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("kind"); + try jw.write(self.kind); + try jw.objectField("nodes"); + try jw.write(self.nodes); + if (self.metric_status) |value| { + try jw.objectField("metric_status"); + try jw.write(value); + } + try jw.objectField("stats"); + try jw.write(self.stats); + try jw.endObject(); + } }; pub const GraphNotEqualPredicate = struct { @@ -6611,6 +7597,8 @@ pub const GraphResultNode = struct { path_edges: ?[]const GraphPathEdge = null, /// Algebraic provenance labels folded into this result, when requested by an algebraic graph executor provenance: ?[]const []const u8 = null, + /// Projected graph metric scores keyed by metric name. Values are numbers or null when a requested metric has no score for the node. + metrics: ?std.json.ArrayHashMap(std.json.Value) = null, /// Parsed evidence envelope for provenance labels and edge metadata evidence: ?std.json.ArrayHashMap(std.json.Value) = null, @@ -6623,6 +7611,7 @@ pub const GraphResultNode = struct { .{ "path", "path", true }, .{ "path_edges", "path_edges", true }, .{ "provenance", "provenance", true }, + .{ "metrics", "metrics", true }, .{ "evidence", "evidence", true }, }; @@ -6660,6 +7649,10 @@ pub const GraphResultNode = struct { try jw.objectField("provenance"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.evidence) |value| { try jw.objectField("evidence"); try jw.write(value); @@ -6906,6 +7899,16 @@ pub const GraphTraversal = struct { include_documents: ?bool = null, /// Requires include_documents=true. Omit to include all document fields. fields: ?[]const []const u8 = null, + /// Graph metric names to project onto returned traversal nodes. + metrics: ?[]const []const u8 = null, + /// Sort traversal candidates by graph metric score before applying limit. + order_by: ?[]const GraphMetricOrder = null, + /// Filter traversal candidates by graph metric score before applying limit. + where_metric: ?[]const GraphMetricFilter = null, + /// Freshness required for projected, ordered, and filtered graph metrics. + metric_freshness: ?[]const u8 = null, + /// Include graph metric status metadata in the traversal profile. + include_metric_status: ?bool = null, /// Non-scoring structured stored-document predicate for reached nodes. filter: ?GraphDocumentFilter = null, @@ -6920,6 +7923,11 @@ pub const GraphTraversal = struct { .{ "include_paths", "include_paths", true }, .{ "include_documents", "include_documents", true }, .{ "fields", "fields", true }, + .{ "metrics", "metrics", true }, + .{ "order_by", "order_by", true }, + .{ "where_metric", "where_metric", true }, + .{ "metric_freshness", "metric_freshness", true }, + .{ "include_metric_status", "include_metric_status", true }, .{ "filter", "filter", true }, }; @@ -6967,6 +7975,26 @@ pub const GraphTraversal = struct { try jw.objectField("fields"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } + if (self.order_by) |value| { + try jw.objectField("order_by"); + try jw.write(value); + } + if (self.where_metric) |value| { + try jw.objectField("where_metric"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + if (self.include_metric_status) |value| { + try jw.objectField("include_metric_status"); + try jw.write(value); + } if (self.filter) |value| { try jw.objectField("filter"); try jw.write(value); @@ -7095,6 +8123,8 @@ pub const IndexConfig = struct { chunk_size: ?i64 = null, /// Non-semantic execution policy for shorthand-created chunking or embedding producers. execution: ?IndexExecutionConfig = null, + /// Named published graph metrics. Serverless supports background refresh only and limits configurations to 16 metrics per graph, 64 total per publication, 64 types per filter, and 128 UTF-8 bytes per metric name. + metrics: ?std.json.ArrayHashMap(GraphMetricConfig) = null, /// Configuration for generating node summaries (enables tree navigation in Retrieval Agent) summarizer: ?antfly_generating_openapi.GeneratorConfig = null, /// List of edge types with their configurations @@ -7136,6 +8166,7 @@ pub const IndexConfig = struct { .{ "min_weight", "min_weight", true }, .{ "chunk_size", "chunk_size", true }, .{ "execution", "execution", true }, + .{ "metrics", "metrics", true }, .{ "summarizer", "summarizer", false }, .{ "edge_types", "edge_types", true }, .{ "max_edges_per_document", "max_edges_per_document", true }, @@ -7248,6 +8279,10 @@ pub const IndexConfig = struct { try jw.objectField("execution"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } if (self.summarizer) |value| { try jw.objectField("summarizer"); try jw.write(value); @@ -7800,6 +8835,16 @@ pub const LegacyGraphQuery = struct { include_documents: ?bool = null, include_edges: ?bool = null, fields: ?[]const []const u8 = null, + /// Graph metric names to project onto legacy graph_searches result nodes. + metrics: ?[]const []const u8 = null, + /// Sort legacy graph_searches result nodes by graph metric score. + order_by: ?[]const GraphMetricOrder = null, + /// Filter legacy graph_searches result nodes by graph metric score. + where_metric: ?[]const GraphMetricFilter = null, + /// Freshness required for projected, ordered, and filtered graph metrics. + metric_freshness: ?[]const u8 = null, + /// Include graph metric status metadata in the legacy graph_searches result. + include_metric_status: ?bool = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -7813,6 +8858,11 @@ pub const LegacyGraphQuery = struct { .{ "include_documents", "include_documents", true }, .{ "include_edges", "include_edges", true }, .{ "fields", "fields", true }, + .{ "metrics", "metrics", true }, + .{ "order_by", "order_by", true }, + .{ "where_metric", "where_metric", true }, + .{ "metric_freshness", "metric_freshness", true }, + .{ "include_metric_status", "include_metric_status", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -7861,6 +8911,26 @@ pub const LegacyGraphQuery = struct { try jw.objectField("fields"); try jw.write(value); } + if (self.metrics) |value| { + try jw.objectField("metrics"); + try jw.write(value); + } + if (self.order_by) |value| { + try jw.objectField("order_by"); + try jw.write(value); + } + if (self.where_metric) |value| { + try jw.objectField("where_metric"); + try jw.write(value); + } + if (self.metric_freshness) |value| { + try jw.objectField("metric_freshness"); + try jw.write(value); + } + if (self.include_metric_status) |value| { + try jw.objectField("include_metric_status"); + try jw.write(value); + } try jw.endObject(); } }; @@ -7969,6 +9039,8 @@ pub const LegacyGraphSearchResult = struct { total: i64, /// Whole-query execution time in milliseconds; optional for compatibility with v0.2 responses. Use the parent query result's took field. took: ?i64 = null, + /// Graph metric status metadata keyed by metric name. + metric_status: ?std.json.ArrayHashMap(GraphMetricStatus) = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -7979,6 +9051,7 @@ pub const LegacyGraphSearchResult = struct { .{ "matches", "matches", true }, .{ "total", "total", false }, .{ "took", "took", true }, + .{ "metric_status", "metric_status", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -8015,6 +9088,10 @@ pub const LegacyGraphSearchResult = struct { try jw.objectField("took"); try jw.write(value); } + if (self.metric_status) |value| { + try jw.objectField("metric_status"); + try jw.write(value); + } try jw.endObject(); } }; @@ -8606,9 +9683,9 @@ pub const StatefulGraphQueryResults = std.json.ArrayHashMap(StatefulGraphResult) /// Graph result emitted by the stateful compatibility transport. Canonical graph_queries produce GraphResult; deprecated graph_searches may produce LegacyGraphSearchResult during the compatibility window. pub const StatefulGraphResult = union(enum) { + graph_nodes_result: *GraphNodesResult, graph_aggregates_result: *GraphAggregatesResult, graph_bindings_result: *GraphBindingsResult, - graph_nodes_result: *GraphNodesResult, graph_paths_result: *GraphPathsResult, legacy_graph_search_result: *LegacyGraphSearchResult, @@ -8650,9 +9727,9 @@ pub const StatefulGraphResult = union(enum) { const probe = try std.json.parseFromSliceLeaky(Probe, allocator, input, probe_options); switch (probe.kind) { .value => |disc_str| { + if (std.mem.eql(u8, disc_str, "nodes")) return .{ .graph_nodes_result = try parseStructuralVariantFromSlice(GraphNodesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "aggregates")) return .{ .graph_aggregates_result = try parseStructuralVariantFromSlice(GraphAggregatesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "bindings")) return .{ .graph_bindings_result = try parseStructuralVariantFromSlice(GraphBindingsResult, allocator, input, options) }; - if (std.mem.eql(u8, disc_str, "nodes")) return .{ .graph_nodes_result = try parseStructuralVariantFromSlice(GraphNodesResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "paths")) return .{ .graph_paths_result = try parseStructuralVariantFromSlice(GraphPathsResult, allocator, input, options) }; if (std.mem.eql(u8, disc_str, "legacy")) return .{ .legacy_graph_search_result = try parseStructuralVariantFromSlice(LegacyGraphSearchResult, allocator, input, options) }; return error.UnexpectedToken; @@ -8678,6 +9755,10 @@ pub const StatefulGraphResult = union(enum) { .string => |value| value, else => return error.UnexpectedToken, }; + if (std.mem.eql(u8, disc_str, "nodes")) { + const parsed = try parseStructuralVariant(GraphNodesResult, allocator, source, options) orelse return error.UnexpectedToken; + return .{ .graph_nodes_result = parsed }; + } if (std.mem.eql(u8, disc_str, "aggregates")) { const parsed = try parseStructuralVariant(GraphAggregatesResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_aggregates_result = parsed }; @@ -8686,10 +9767,6 @@ pub const StatefulGraphResult = union(enum) { const parsed = try parseStructuralVariant(GraphBindingsResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_bindings_result = parsed }; } - if (std.mem.eql(u8, disc_str, "nodes")) { - const parsed = try parseStructuralVariant(GraphNodesResult, allocator, source, options) orelse return error.UnexpectedToken; - return .{ .graph_nodes_result = parsed }; - } if (std.mem.eql(u8, disc_str, "paths")) { const parsed = try parseStructuralVariant(GraphPathsResult, allocator, source, options) orelse return error.UnexpectedToken; return .{ .graph_paths_result = parsed }; @@ -8703,9 +9780,9 @@ pub const StatefulGraphResult = union(enum) { pub fn jsonStringify(self: @This(), jw: anytype) !void { switch (self) { + .graph_nodes_result => |v| try jw.write(v.*), .graph_aggregates_result => |v| try jw.write(v.*), .graph_bindings_result => |v| try jw.write(v.*), - .graph_nodes_result => |v| try jw.write(v.*), .graph_paths_result => |v| try jw.write(v.*), .legacy_graph_search_result => |v| try jw.write(v.*), } diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/root.zig b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/root.zig index 9494fb0675..0919317ab5 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/root.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/root.zig @@ -34,6 +34,7 @@ pub const BackupListResponse = types.BackupListResponse; pub const BackupMetadataUnavailableError = types.BackupMetadataUnavailableError; pub const BackupOutcomeAmbiguousConflict = types.BackupOutcomeAmbiguousConflict; pub const BackupRequest = types.BackupRequest; +pub const BatchCommittedFailure = types.BatchCommittedFailure; pub const BatchRequest = types.BatchRequest; pub const BatchResponse = types.BatchResponse; pub const ByteRange = types.ByteRange; @@ -91,6 +92,9 @@ pub const GlobalStatefulQueryRequest = types.GlobalStatefulQueryRequest; pub const GraphAnchorFilterRequiresIndexError = types.GraphAnchorFilterRequiresIndexError; pub const GraphDistinctBudgetExceededError = types.GraphDistinctBudgetExceededError; pub const GraphMatchOperationLimitExceededError = types.GraphMatchOperationLimitExceededError; +pub const GraphMetricActionResponse = types.GraphMetricActionResponse; +pub const GraphMetricProfile = types.GraphMetricProfile; +pub const GraphMetricRerankScoreDetails = types.GraphMetricRerankScoreDetails; pub const GraphPathWeightDomainError = types.GraphPathWeightDomainError; pub const GraphQueryUnprocessableError = types.GraphQueryUnprocessableError; pub const GraphQueryUnsupportedError = types.GraphQueryUnsupportedError; @@ -159,6 +163,7 @@ pub const QueryRequest = types.QueryRequest; pub const QueryResponses = types.QueryResponses; pub const QueryResult = types.QueryResult; pub const QueryResultBase = types.QueryResultBase; +pub const QueryScoreDetails = types.QueryScoreDetails; pub const QueryTemporarilyUnavailableError = types.QueryTemporarilyUnavailableError; pub const QueryUnprocessableError = types.QueryUnprocessableError; pub const RawQuery = types.RawQuery; diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig index 854998e126..12acfae7e6 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig @@ -351,6 +351,18 @@ pub const DropIndexPathParams = struct { index_name: []const u8, }; +/// Execute a graph metric operational action +pub const ExecuteGraphMetricActionPathParams = struct { + /// Name of the table + table_name: []const u8, + /// Name of the graph index + index_name: []const u8, + /// Name of the configured graph metric + metric_name: []const u8, + /// Operational action to apply to the graph metric materialization + action: []const u8, +}; + /// Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge pub const LinearMergePathParams = struct { /// Name of the table @@ -602,6 +614,7 @@ pub const routes = [_]Route{ .{ .method = "GET", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "getIndex", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "createIndex", .request_body = .buffered, .streaming_response = false }, .{ .method = "DELETE", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "dropIndex", .request_body = .none, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action}", .operation_id = "executeGraphMetricAction", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/merge", .operation_id = "linearMerge", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/query", .operation_id = "queryTable", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/repair/issues", .operation_id = "listTableRepairIssues", .request_body = .buffered, .streaming_response = false }, @@ -681,6 +694,7 @@ pub fn ServerRouter(comptime Impl: type) type { if (!@hasDecl(Impl, "getIndex")) @compileError("ServerRouter: Impl missing required method 'getIndex'"); if (!@hasDecl(Impl, "createIndex")) @compileError("ServerRouter: Impl missing required method 'createIndex'"); if (!@hasDecl(Impl, "dropIndex")) @compileError("ServerRouter: Impl missing required method 'dropIndex'"); + if (!@hasDecl(Impl, "executeGraphMetricAction")) @compileError("ServerRouter: Impl missing required method 'executeGraphMetricAction'"); if (!@hasDecl(Impl, "linearMerge")) @compileError("ServerRouter: Impl missing required method 'linearMerge'"); if (!@hasDecl(Impl, "queryTable")) @compileError("ServerRouter: Impl missing required method 'queryTable'"); if (!@hasDecl(Impl, "listTableRepairIssues")) @compileError("ServerRouter: Impl missing required method 'listTableRepairIssues'"); @@ -758,6 +772,7 @@ pub fn ServerRouter(comptime Impl: type) type { try server.get("/tables/:tableName/indexes/:indexName", httpx.Handler.bind(self.impl, getIndex)); try server.post("/tables/:tableName/indexes/:indexName", httpx.Handler.bind(self.impl, createIndex)); try server.delete("/tables/:tableName/indexes/:indexName", httpx.Handler.bind(self.impl, dropIndex)); + try server.post("/tables/:tableName/indexes/:indexName/graph-metrics/:metricName::action", httpx.Handler.bind(self.impl, executeGraphMetricAction)); try server.post("/tables/:tableName/merge", httpx.Handler.bind(self.impl, linearMerge)); try server.post("/tables/:tableName/query", httpx.Handler.bind(self.impl, queryTable)); try server.post("/tables/:tableName/repair/issues", httpx.Handler.bind(self.impl, listTableRepairIssues)); @@ -1115,6 +1130,16 @@ pub fn ServerRouter(comptime Impl: type) type { return impl.dropIndex(ctx, table_name, index_name); } + /// Execute a graph metric operational action + /// POST /tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action} + fn executeGraphMetricAction(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { + const table_name = ctx.param("tableName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: tableName" }); + const index_name = ctx.param("indexName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: indexName" }); + const metric_name = ctx.param("metricName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: metricName" }); + const action = ctx.param("action") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: action" }); + return impl.executeGraphMetricAction(ctx, table_name, index_name, metric_name, action); + } + /// Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge /// POST /tables/{tableName}/merge fn linearMerge(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { @@ -1332,6 +1357,7 @@ pub fn ServerRouter(comptime Impl: type) type { // fn getIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response // fn createIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response // fn dropIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response +// fn executeGraphMetricAction(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8, metric_name: []const u8, action: []const u8) !httpx.Response // fn linearMerge(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn queryTable(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn listTableRepairIssues(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/types.zig b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/types.zig index b428fdd266..d60699bf6d 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/types.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/types.zig @@ -1446,6 +1446,49 @@ pub const BackupRequest = struct { } }; +/// Additive details for a committed batch that needs operator action. The open string code is forward-compatible with older SDKs; clients should treat unknown codes as non-retryable when `retryable` is false. +pub const BatchCommittedFailure = struct { + /// Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + code: []const u8, + /// Actionable operator guidance. + message: []const u8, + /// Optional stable reason within the failure category, such as `build_budget_exceeded`. + reason: ?[]const u8 = null, + /// Whether replaying the document mutation is safe. Committed repair outcomes are false. + retryable: bool, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "code", "code", false }, + .{ "message", "message", false }, + .{ "reason", "reason", true }, + .{ "retryable", "retryable", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("code"); + try jw.write(self.code); + try jw.objectField("message"); + try jw.write(self.message); + if (self.reason) |value| { + try jw.objectField("reason"); + try jw.write(value); + } + try jw.objectField("retryable"); + try jw.write(self.retryable); + try jw.endObject(); + } +}; + /// Batch insert, delete, and transform operations in a single request. **Atomicity**: - **Single shard**: Operations are atomic within shard boundaries - **Multiple shards**: Uses distributed 2-phase commit (2PC) for atomic cross-shard writes **How distributed transactions work**: 1. Metadata server allocates HLC timestamp and selects coordinator shard 2. Coordinator writes transaction record, participants write intents 3. After all intents succeed, coordinator commits transaction 4. Participants are notified asynchronously to resolve intents 5. Recovery loop ensures notifications complete even after coordinator failure **Performance**: - Single-shard batches: < 5ms latency - Cross-shard transactions: ~20ms latency - Intent resolution: < 30 seconds worst-case (via recovery loop) **Guarantees**: - All writes succeed or all fail (atomicity across all shards) - Coordinator failure is recoverable (new leader resumes notifications) - Idempotent resolution (duplicate notifications are safe) **Benefits**: - Reduces network overhead compared to individual requests - More efficient indexing (updates are batched) - Automatic distributed transactions when operations span shards The inserts are upserts - existing keys are overwritten, new keys are created. pub const BatchRequest = struct { /// Map of document IDs to document objects. Each key is the unique identifier for the document. Best practices: - Use consistent key naming schemes (e.g., "user:123", "article:456") - Key length affects storage and performance - keep them reasonably short - Keys are sorted lexicographically, so choose prefixes that support range scans @@ -1495,7 +1538,7 @@ pub const BatchRequest = struct { }; pub const BatchResponse = struct { - /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal enrichment failure needs operator repair and will not be retried indefinitely. + /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal background materialization failure needs operator repair and will not be retried indefinitely. Inspect `failure` when present; retrying the document write is unnecessary. status: ?[]const u8 = null, /// Number of documents successfully inserted inserted: ?i64 = null, @@ -1503,6 +1546,7 @@ pub const BatchResponse = struct { deleted: ?i64 = null, /// Number of documents successfully transformed transformed: ?i64 = null, + failure: ?BatchCommittedFailure = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -1510,6 +1554,7 @@ pub const BatchResponse = struct { .{ "inserted", "inserted", true }, .{ "deleted", "deleted", true }, .{ "transformed", "transformed", true }, + .{ "failure", "failure", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -1538,6 +1583,10 @@ pub const BatchResponse = struct { try jw.objectField("transformed"); try jw.write(value); } + if (self.failure) |value| { + try jw.objectField("failure"); + try jw.write(value); + } try jw.endObject(); } }; @@ -4045,6 +4094,10 @@ pub const GlobalStatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -4090,6 +4143,8 @@ pub const GlobalStatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -4222,6 +4277,20 @@ pub const GlobalStatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -4297,6 +4366,137 @@ pub const GraphMatchOperationLimitExceededError = struct { actual: i64, }; +pub const GraphMetricActionResponse = struct { + status: antfly_indexes_openapi.GraphMetricStatus, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "status", "status", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("status"); + try jw.write(self.status); + try jw.endObject(); + } +}; + +pub const GraphMetricProfile = struct { + /// Name of the graph query or graph metric query that used the metric. + query_name: []const u8, + /// Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + source: []const u8, + /// Graph index that owns the metric. + index_name: []const u8, + /// Graph metric name within the index. + metric_name: []const u8, + /// Effective freshness mode requested for this metric use. + freshness: []const u8, + /// Published generation and freshness status observed by the query. + status: antfly_indexes_openapi.GraphMetricStatus, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "query_name", "query_name", false }, + .{ "source", "source", false }, + .{ "index_name", "index_name", false }, + .{ "metric_name", "metric_name", false }, + .{ "freshness", "freshness", false }, + .{ "status", "status", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("query_name"); + try jw.write(self.query_name); + try jw.objectField("source"); + try jw.write(self.source); + try jw.objectField("index_name"); + try jw.write(self.index_name); + try jw.objectField("metric_name"); + try jw.write(self.metric_name); + try jw.objectField("freshness"); + try jw.write(self.freshness); + try jw.objectField("status"); + try jw.write(self.status); + try jw.endObject(); + } +}; + +pub const GraphMetricRerankScoreDetails = struct { + /// Graph index that provided the metric score. + index_name: []const u8, + /// Graph metric used as a score feature. + metric_name: []const u8, + /// Hit score before graph metric rerank composition. + base_score: f64, + /// Weight applied to the base score. + base_weight: f64, + /// Published metric score for this hit, or null when the hit was missing from the metric generation. + metric_score: OpenApiOptionalNullable(f64) = .absent, + /// Metric feature value used in the formula after applying missing_score fallback if needed. + metric_score_used: f64, + /// Weight applied to the metric score feature. + metric_weight: f64, + /// True when metric_score was missing and the request's missing_score fallback was used. + missing_score_used: bool, + /// Final hit score after graph metric rerank composition. + final_score: f64, + /// Published graph metric score generation used for this hit. + published_generation: i64, + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("index_name"); + try jw.write(self.index_name); + try jw.objectField("metric_name"); + try jw.write(self.metric_name); + try jw.objectField("base_score"); + try jw.write(self.base_score); + try jw.objectField("base_weight"); + try jw.write(self.base_weight); + switch (self.metric_score) { + .absent => {}, + .null_value => { + try jw.objectField("metric_score"); + try jw.write(@as(?u8, null)); + }, + .value => |value| { + try jw.objectField("metric_score"); + try jw.write(value); + }, + } + try jw.objectField("metric_score_used"); + try jw.write(self.metric_score_used); + try jw.objectField("metric_weight"); + try jw.write(self.metric_weight); + try jw.objectField("missing_score_used"); + try jw.write(self.missing_score_used); + try jw.objectField("final_score"); + try jw.write(self.final_score); + try jw.objectField("published_generation"); + try jw.write(self.published_generation); + try jw.endObject(); + } +}; + pub const GraphPathWeightDomainError = struct { status: i32, @"error": []const u8, @@ -6808,6 +7008,8 @@ pub const QueryHit = struct { _distance: ?f32 = null, /// Scores partitioned by index when using RRF search. _index_scores: ?std.json.ArrayHashMap(f64) = null, + /// Optional score provenance for ranking features applied to this hit. + _score_details: ?QueryScoreDetails = null, _source: ?std.json.ArrayHashMap(std.json.Value) = null, /// Stable ancestry envelope for derived document hierarchy hits. Present when the hit is a derived unit/chunk/embedding artifact or when a source-level group includes nested matches. Standard fields include `level`, `parent_doc_key`, optional `parent_unit_id`, `artifact` or `matched_artifact`, `matches`, and `ancestors` with response-local or requested DB-backed source/unit context when available. V0.2-compatible implicit rollup requests continue to use the deprecated `chunks` field instead of `matches`. hierarchy: ?QueryHitHierarchy = null, @@ -6820,6 +7022,7 @@ pub const QueryHit = struct { .{ "_score", "_score", false }, .{ "_distance", "_distance", true }, .{ "_index_scores", "_index_scores", true }, + .{ "_score_details", "_score_details", true }, .{ "_source", "_source", true }, .{ "hierarchy", "hierarchy", true }, .{ "_sort", "_sort", true }, @@ -6847,6 +7050,10 @@ pub const QueryHit = struct { try jw.objectField("_index_scores"); try jw.write(value); } + if (self._score_details) |value| { + try jw.objectField("_score_details"); + try jw.write(value); + } if (self._source) |value| { try jw.objectField("_source"); try jw.write(value); @@ -7083,6 +7290,8 @@ pub const QueryProfile = struct { reranker: ?RerankerProfile = null, /// Result merge statistics (present for hybrid search). merge: ?MergeProfile = null, + /// Graph metric freshness and generation details for metric-aware query work. + graph_metrics: ?[]const GraphMetricProfile = null, /// Sort execution statistics (present when the query used ordered page options and profiling was enabled). sort: ?SortProfile = null, @@ -7092,6 +7301,7 @@ pub const QueryProfile = struct { .{ "join", "join", true }, .{ "reranker", "reranker", true }, .{ "merge", "merge", true }, + .{ "graph_metrics", "graph_metrics", true }, .{ "sort", "sort", true }, }; @@ -7121,6 +7331,10 @@ pub const QueryProfile = struct { try jw.objectField("merge"); try jw.write(value); } + if (self.graph_metrics) |value| { + try jw.objectField("graph_metrics"); + try jw.write(value); + } if (self.sort) |value| { try jw.objectField("sort"); try jw.write(value); @@ -7183,6 +7397,10 @@ pub const QueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -7224,6 +7442,8 @@ pub const QueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -7356,6 +7576,20 @@ pub const QueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -7424,6 +7658,8 @@ pub const QueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -7441,6 +7677,7 @@ pub const QueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -7471,6 +7708,10 @@ pub const QueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -7505,6 +7746,8 @@ pub const QueryResultBase = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -7521,6 +7764,7 @@ pub const QueryResultBase = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -7550,6 +7794,10 @@ pub const QueryResultBase = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -7570,6 +7818,34 @@ pub const QueryResultBase = struct { } }; +/// Optional score provenance for ranking features that changed the final hit score. +pub const QueryScoreDetails = struct { + /// Score contribution from an explicit graph_metric_rerank request. + graph_metric_rerank: ?GraphMetricRerankScoreDetails = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "graph_metric_rerank", "graph_metric_rerank", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } + try jw.endObject(); + } +}; + /// A transient query dependency or read-availability failure that is safe to retry. pub const QueryTemporarilyUnavailableError = struct { /// Stable machine-readable retry classification. @@ -9120,6 +9396,10 @@ pub const RetrievalQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -9163,6 +9443,8 @@ pub const RetrievalQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -9296,6 +9578,20 @@ pub const RetrievalQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -10174,6 +10470,10 @@ pub const StatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -10219,6 +10519,8 @@ pub const StatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -10353,6 +10655,20 @@ pub const StatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -10429,6 +10745,8 @@ pub const StatefulQueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -10446,6 +10764,7 @@ pub const StatefulQueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -10476,6 +10795,10 @@ pub const StatefulQueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/root.zig b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/root.zig index 56afabb9cc..d1aeb2f713 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/root.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/root.zig @@ -36,6 +36,7 @@ pub const BackupListResponse = types.BackupListResponse; pub const BackupMetadataUnavailableError = types.BackupMetadataUnavailableError; pub const BackupOutcomeAmbiguousConflict = types.BackupOutcomeAmbiguousConflict; pub const BackupRequest = types.BackupRequest; +pub const BatchCommittedFailure = types.BatchCommittedFailure; pub const BatchRequest = types.BatchRequest; pub const BatchResponse = types.BatchResponse; pub const ByteRange = types.ByteRange; @@ -95,6 +96,9 @@ pub const GlobalStatefulQueryRequest = types.GlobalStatefulQueryRequest; pub const GraphAnchorFilterRequiresIndexError = types.GraphAnchorFilterRequiresIndexError; pub const GraphDistinctBudgetExceededError = types.GraphDistinctBudgetExceededError; pub const GraphMatchOperationLimitExceededError = types.GraphMatchOperationLimitExceededError; +pub const GraphMetricActionResponse = types.GraphMetricActionResponse; +pub const GraphMetricProfile = types.GraphMetricProfile; +pub const GraphMetricRerankScoreDetails = types.GraphMetricRerankScoreDetails; pub const GraphPathWeightDomainError = types.GraphPathWeightDomainError; pub const GraphQueryUnprocessableError = types.GraphQueryUnprocessableError; pub const GraphQueryUnsupportedError = types.GraphQueryUnsupportedError; @@ -165,6 +169,7 @@ pub const QueryRequest = types.QueryRequest; pub const QueryResponses = types.QueryResponses; pub const QueryResult = types.QueryResult; pub const QueryResultBase = types.QueryResultBase; +pub const QueryScoreDetails = types.QueryScoreDetails; pub const QueryTemporarilyUnavailableError = types.QueryTemporarilyUnavailableError; pub const QueryUnprocessableError = types.QueryUnprocessableError; pub const RawQuery = types.RawQuery; diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig index ba843589c9..03c21d8bad 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig @@ -537,6 +537,18 @@ pub const DropIndexPathParams = struct { index_name: []const u8, }; +/// Execute a graph metric operational action +pub const ExecuteGraphMetricActionPathParams = struct { + /// Name of the table + table_name: []const u8, + /// Name of the graph index + index_name: []const u8, + /// Name of the configured graph metric + metric_name: []const u8, + /// Operational action to apply to the graph metric materialization + action: []const u8, +}; + /// Synchronize data from external sources (Shopify, Postgres, S3) using a linear merge pub const LinearMergePathParams = struct { /// Name of the table @@ -812,6 +824,7 @@ pub const routes = [_]Route{ .{ .method = "GET", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "getIndex", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "createIndex", .request_body = .buffered, .streaming_response = false }, .{ .method = "DELETE", .path = "/tables/{tableName}/indexes/{indexName}", .operation_id = "dropIndex", .request_body = .none, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/indexes/{indexName}/graph-metrics/{metricName}:{action}", .operation_id = "executeGraphMetricAction", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/merge", .operation_id = "linearMerge", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/query", .operation_id = "queryTable", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/tables/{tableName}/repair/issues", .operation_id = "listTableRepairIssues", .request_body = .buffered, .streaming_response = false }, @@ -906,6 +919,7 @@ pub const routes = [_]Route{ // fn getIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response // fn createIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response // fn dropIndex(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8) !httpx.Response +// fn executeGraphMetricAction(self: *Impl, ctx: *httpx.Context, table_name: []const u8, index_name: []const u8, metric_name: []const u8, action: []const u8) !httpx.Response // fn linearMerge(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn queryTable(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn listTableRepairIssues(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/types.zig b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/types.zig index 2c5c19b76c..84fe091d09 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/types.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/types.zig @@ -1588,6 +1588,49 @@ pub const BackupRequest = struct { } }; +/// Additive details for a committed batch that needs operator action. The open string code is forward-compatible with older SDKs; clients should treat unknown codes as non-retryable when `retryable` is false. +pub const BatchCommittedFailure = struct { + /// Stable machine-readable failure code, such as `graph_metric_materialization_rejected`. + code: []const u8, + /// Actionable operator guidance. + message: []const u8, + /// Optional stable reason within the failure category, such as `build_budget_exceeded`. + reason: ?[]const u8 = null, + /// Whether replaying the document mutation is safe. Committed repair outcomes are false. + retryable: bool, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "code", "code", false }, + .{ "message", "message", false }, + .{ "reason", "reason", true }, + .{ "retryable", "retryable", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("code"); + try jw.write(self.code); + try jw.objectField("message"); + try jw.write(self.message); + if (self.reason) |value| { + try jw.objectField("reason"); + try jw.write(value); + } + try jw.objectField("retryable"); + try jw.write(self.retryable); + try jw.endObject(); + } +}; + /// Batch insert, delete, and transform operations in a single request. **Atomicity**: - **Single shard**: Operations are atomic within shard boundaries - **Multiple shards**: Uses distributed 2-phase commit (2PC) for atomic cross-shard writes **How distributed transactions work**: 1. Metadata server allocates HLC timestamp and selects coordinator shard 2. Coordinator writes transaction record, participants write intents 3. After all intents succeed, coordinator commits transaction 4. Participants are notified asynchronously to resolve intents 5. Recovery loop ensures notifications complete even after coordinator failure **Performance**: - Single-shard batches: < 5ms latency - Cross-shard transactions: ~20ms latency - Intent resolution: < 30 seconds worst-case (via recovery loop) **Guarantees**: - All writes succeed or all fail (atomicity across all shards) - Coordinator failure is recoverable (new leader resumes notifications) - Idempotent resolution (duplicate notifications are safe) **Benefits**: - Reduces network overhead compared to individual requests - More efficient indexing (updates are batched) - Automatic distributed transactions when operations span shards The inserts are upserts - existing keys are overwritten, new keys are created. pub const BatchRequest = struct { /// Map of document IDs to document objects. Each key is the unique identifier for the document. Best practices: - Use consistent key naming schemes (e.g., "user:123", "article:456") - Key length affects storage and performance - keep them reasonably short - Keys are sorted lexicographically, so choose prefixes that support range scans @@ -1637,7 +1680,7 @@ pub const BatchRequest = struct { }; pub const BatchResponse = struct { - /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal enrichment failure needs operator repair and will not be retried indefinitely. + /// Durable commit outcome. `committed_pending` means requested visibility or participant propagation is still completing. `committed_repair_required` means the primary write committed, but a terminal background materialization failure needs operator repair and will not be retried indefinitely. Inspect `failure` when present; retrying the document write is unnecessary. status: ?[]const u8 = null, /// Number of documents successfully inserted inserted: ?i64 = null, @@ -1645,6 +1688,7 @@ pub const BatchResponse = struct { deleted: ?i64 = null, /// Number of documents successfully transformed transformed: ?i64 = null, + failure: ?BatchCommittedFailure = null, /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. pub const openApiFieldMetadata = .{ @@ -1652,6 +1696,7 @@ pub const BatchResponse = struct { .{ "inserted", "inserted", true }, .{ "deleted", "deleted", true }, .{ "transformed", "transformed", true }, + .{ "failure", "failure", true }, }; pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { @@ -1680,6 +1725,10 @@ pub const BatchResponse = struct { try jw.objectField("transformed"); try jw.write(value); } + if (self.failure) |value| { + try jw.objectField("failure"); + try jw.write(value); + } try jw.endObject(); } }; @@ -4358,6 +4407,10 @@ pub const GlobalStatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -4403,6 +4456,8 @@ pub const GlobalStatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -4535,6 +4590,20 @@ pub const GlobalStatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -4610,6 +4679,137 @@ pub const GraphMatchOperationLimitExceededError = struct { actual: i64, }; +pub const GraphMetricActionResponse = struct { + status: antfly_indexes_openapi.GraphMetricStatus, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "status", "status", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("status"); + try jw.write(self.status); + try jw.endObject(); + } +}; + +pub const GraphMetricProfile = struct { + /// Name of the graph query or graph metric query that used the metric. + query_name: []const u8, + /// Profile source, such as `graph_query`, `graph_metric`, or `graph_metric_rerank`. + source: []const u8, + /// Graph index that owns the metric. + index_name: []const u8, + /// Graph metric name within the index. + metric_name: []const u8, + /// Effective freshness mode requested for this metric use. + freshness: []const u8, + /// Published generation and freshness status observed by the query. + status: antfly_indexes_openapi.GraphMetricStatus, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "query_name", "query_name", false }, + .{ "source", "source", false }, + .{ "index_name", "index_name", false }, + .{ "metric_name", "metric_name", false }, + .{ "freshness", "freshness", false }, + .{ "status", "status", false }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("query_name"); + try jw.write(self.query_name); + try jw.objectField("source"); + try jw.write(self.source); + try jw.objectField("index_name"); + try jw.write(self.index_name); + try jw.objectField("metric_name"); + try jw.write(self.metric_name); + try jw.objectField("freshness"); + try jw.write(self.freshness); + try jw.objectField("status"); + try jw.write(self.status); + try jw.endObject(); + } +}; + +pub const GraphMetricRerankScoreDetails = struct { + /// Graph index that provided the metric score. + index_name: []const u8, + /// Graph metric used as a score feature. + metric_name: []const u8, + /// Hit score before graph metric rerank composition. + base_score: f64, + /// Weight applied to the base score. + base_weight: f64, + /// Published metric score for this hit, or null when the hit was missing from the metric generation. + metric_score: OpenApiOptionalNullable(f64) = .absent, + /// Metric feature value used in the formula after applying missing_score fallback if needed. + metric_score_used: f64, + /// Weight applied to the metric score feature. + metric_weight: f64, + /// True when metric_score was missing and the request's missing_score fallback was used. + missing_score_used: bool, + /// Final hit score after graph metric rerank composition. + final_score: f64, + /// Published graph metric score generation used for this hit. + published_generation: i64, + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + try jw.objectField("index_name"); + try jw.write(self.index_name); + try jw.objectField("metric_name"); + try jw.write(self.metric_name); + try jw.objectField("base_score"); + try jw.write(self.base_score); + try jw.objectField("base_weight"); + try jw.write(self.base_weight); + switch (self.metric_score) { + .absent => {}, + .null_value => { + try jw.objectField("metric_score"); + try jw.write(@as(?u8, null)); + }, + .value => |value| { + try jw.objectField("metric_score"); + try jw.write(value); + }, + } + try jw.objectField("metric_score_used"); + try jw.write(self.metric_score_used); + try jw.objectField("metric_weight"); + try jw.write(self.metric_weight); + try jw.objectField("missing_score_used"); + try jw.write(self.missing_score_used); + try jw.objectField("final_score"); + try jw.write(self.final_score); + try jw.objectField("published_generation"); + try jw.write(self.published_generation); + try jw.endObject(); + } +}; + pub const GraphPathWeightDomainError = struct { status: i32, @"error": []const u8, @@ -7157,6 +7357,8 @@ pub const QueryHit = struct { _distance: ?f32 = null, /// Scores partitioned by index when using RRF search. _index_scores: ?std.json.ArrayHashMap(f64) = null, + /// Optional score provenance for ranking features applied to this hit. + _score_details: ?QueryScoreDetails = null, _source: ?std.json.ArrayHashMap(std.json.Value) = null, /// Stable ancestry envelope for derived document hierarchy hits. Present when the hit is a derived unit/chunk/embedding artifact or when a source-level group includes nested matches. Standard fields include `level`, `parent_doc_key`, optional `parent_unit_id`, `artifact` or `matched_artifact`, `matches`, and `ancestors` with response-local or requested DB-backed source/unit context when available. V0.2-compatible implicit rollup requests continue to use the deprecated `chunks` field instead of `matches`. hierarchy: ?QueryHitHierarchy = null, @@ -7169,6 +7371,7 @@ pub const QueryHit = struct { .{ "_score", "_score", false }, .{ "_distance", "_distance", true }, .{ "_index_scores", "_index_scores", true }, + .{ "_score_details", "_score_details", true }, .{ "_source", "_source", true }, .{ "hierarchy", "hierarchy", true }, .{ "_sort", "_sort", true }, @@ -7196,6 +7399,10 @@ pub const QueryHit = struct { try jw.objectField("_index_scores"); try jw.write(value); } + if (self._score_details) |value| { + try jw.objectField("_score_details"); + try jw.write(value); + } if (self._source) |value| { try jw.objectField("_source"); try jw.write(value); @@ -7432,6 +7639,8 @@ pub const QueryProfile = struct { reranker: ?RerankerProfile = null, /// Result merge statistics (present for hybrid search). merge: ?MergeProfile = null, + /// Graph metric freshness and generation details for metric-aware query work. + graph_metrics: ?[]const GraphMetricProfile = null, /// Sort execution statistics (present when the query used ordered page options and profiling was enabled). sort: ?SortProfile = null, @@ -7441,6 +7650,7 @@ pub const QueryProfile = struct { .{ "join", "join", true }, .{ "reranker", "reranker", true }, .{ "merge", "merge", true }, + .{ "graph_metrics", "graph_metrics", true }, .{ "sort", "sort", true }, }; @@ -7470,6 +7680,10 @@ pub const QueryProfile = struct { try jw.objectField("merge"); try jw.write(value); } + if (self.graph_metrics) |value| { + try jw.objectField("graph_metrics"); + try jw.write(value); + } if (self.sort) |value| { try jw.objectField("sort"); try jw.write(value); @@ -7532,6 +7746,10 @@ pub const QueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -7573,6 +7791,8 @@ pub const QueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -7705,6 +7925,20 @@ pub const QueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -7773,6 +8007,8 @@ pub const QueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -7790,6 +8026,7 @@ pub const QueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -7820,6 +8057,10 @@ pub const QueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -7854,6 +8095,8 @@ pub const QueryResultBase = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -7870,6 +8113,7 @@ pub const QueryResultBase = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -7899,6 +8143,10 @@ pub const QueryResultBase = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); @@ -7919,6 +8167,34 @@ pub const QueryResultBase = struct { } }; +/// Optional score provenance for ranking features that changed the final hit score. +pub const QueryScoreDetails = struct { + /// Score contribution from an explicit graph_metric_rerank request. + graph_metric_rerank: ?GraphMetricRerankScoreDetails = null, + + /// OpenAPI wire names and nullability consumed by compatible typed JSON parsers. + pub const openApiFieldMetadata = .{ + .{ "graph_metric_rerank", "graph_metric_rerank", true }, + }; + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !@This() { + return try openApiParseObject(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !@This() { + return try openApiParseObjectFromValue(@This(), openApiFieldMetadata, allocator, source, options); + } + + pub fn jsonStringify(self: @This(), jw: anytype) !void { + try jw.beginObject(); + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } + try jw.endObject(); + } +}; + /// A transient query dependency or read-availability failure that is safe to retry. pub const QueryTemporarilyUnavailableError = struct { /// Stable machine-readable retry classification. @@ -9501,6 +9777,10 @@ pub const RetrievalQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -9544,6 +9824,8 @@ pub const RetrievalQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -9677,6 +9959,20 @@ pub const RetrievalQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -10568,6 +10864,10 @@ pub const StatefulQueryRequest = struct { profile: ?bool = null, /// Optional reranker configuration to improve result relevance. Rerankers use cross-encoder models that score query-document pairs directly, providing more accurate relevance scores than embedding similarity alone. **When to use:** - Results need high precision (e.g., RAG, question answering) - You have semantic or hybrid search results to refine - Latency trade-off is acceptable (reranking adds 100-500ms typically) **Best practice:** Set `candidate_count` to the bounded retrieval window (often 50-100) and use the query `limit` for the final page size. Antfly retrieves and globally merges that window, calls the reranker once, then applies pruning, offset, and limit at the coordinator. Example: ```json { "provider": "antfly", "model": "cross-encoder/ms-marco-MiniLM-L-6-v2", "field": "content" } ``` reranker: ?antfly_reranking_openapi.RerankerConfig = null, + /// Direct top-k read from a published graph metric generation. Results are returned in graph_metric_results under the requested name or the metric name when no explicit name is supplied. + graph_metric: ?antfly_indexes_openapi.GraphMetricQuery = null, + /// Blend a published graph metric feature into ordinary search hit scores. Requests may require either any published generation or a generation that is fresh with respect to graph writes. + graph_metric_rerank: ?antfly_indexes_openapi.GraphMetricRerank = null, analyses: ?Analyses = null, /// Declarative graph matching, traversal, and path queries. A nested node `filter` is a typed, non-scoring stored-document predicate. It shares familiar scalar syntax with document queries but deliberately excludes analyzer-backed and index-only clauses. A request may contain at most 64 named graph operations, of which at most 8 may be named `match` operations. Each operation key is a GraphIdentifier under the versioned policy published in the GraphIdentifier schema. Put multiple counts over one pattern in the same `match` return object so they share one complete anchor scan. graph_queries: ?antfly_indexes_openapi.GraphQueries = null, @@ -10613,6 +10913,8 @@ pub const StatefulQueryRequest = struct { .{ "count", "count", true }, .{ "profile", "profile", true }, .{ "reranker", "reranker", false }, + .{ "graph_metric", "graph_metric", false }, + .{ "graph_metric_rerank", "graph_metric_rerank", false }, .{ "analyses", "analyses", true }, .{ "graph_queries", "graph_queries", false }, .{ "document_renderer", "document_renderer", true }, @@ -10747,6 +11049,20 @@ pub const StatefulQueryRequest = struct { try jw.objectField("reranker"); try jw.write(@as(?u8, null)); } + if (self.graph_metric) |value| { + try jw.objectField("graph_metric"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric"); + try jw.write(@as(?u8, null)); + } + if (self.graph_metric_rerank) |value| { + try jw.objectField("graph_metric_rerank"); + try jw.write(value); + } else if (jw.options.emit_null_optional_fields) { + try jw.objectField("graph_metric_rerank"); + try jw.write(@as(?u8, null)); + } if (self.analyses) |value| { try jw.objectField("analyses"); try jw.write(value); @@ -10823,6 +11139,8 @@ pub const StatefulQueryResult = struct { aggregations: ?std.json.ArrayHashMap(AggregationResult) = null, /// Analysis results like PCA and t-SNE per index embeddings. analyses: ?std.json.ArrayHashMap(AnalysesResult) = null, + /// Results from direct graph metric reads. + graph_metric_results: ?std.json.ArrayHashMap(antfly_indexes_openapi.GraphMetricResult) = null, /// Detailed execution profile (present when `profile: true` in request). profile: ?std.json.Value = null, /// Duration of the query in milliseconds. @@ -10840,6 +11158,7 @@ pub const StatefulQueryResult = struct { .{ "hits", "hits", true }, .{ "aggregations", "aggregations", true }, .{ "analyses", "analyses", true }, + .{ "graph_metric_results", "graph_metric_results", true }, .{ "profile", "profile", true }, .{ "took", "took", false }, .{ "status", "status", false }, @@ -10870,6 +11189,10 @@ pub const StatefulQueryResult = struct { try jw.objectField("analyses"); try jw.write(value); } + if (self.graph_metric_results) |value| { + try jw.objectField("graph_metric_results"); + try jw.write(value); + } if (self.profile) |value| { try jw.objectField("profile"); try jw.write(value); diff --git a/zig/pkg/antfly/src/runtime_artifact_lib.zig b/zig/pkg/antfly/src/runtime_artifact_lib.zig index 35152af84d..a6232b1432 100644 --- a/zig/pkg/antfly/src/runtime_artifact_lib.zig +++ b/zig/pkg/antfly/src/runtime_artifact_lib.zig @@ -41,6 +41,7 @@ const cli_runtime = if (unit_options.unit == .cli) @import("cli_runtime.zig") el // archive does not code-generate a second copy of the HA storage closure. const ha_runtime = if (unit_options.unit == .distributed) @import("cmd/ha.zig") else struct {}; const data_runtime = if (unit_options.unit == .distributed) @import("data/runtime.zig") else struct {}; +const graph_metric_maintenance_runtime = if (unit_options.unit == .distributed) @import("cmd/graph_metric_maintenance.zig") else struct {}; const metadata_runtime = if (unit_options.unit == .distributed) @import("metadata/runtime.zig") else struct {}; const serverless_runtime = if (unit_options.unit == .serverless) @import("cmd/serverless.zig") else struct {}; const inference_runtime = if (unit_options.unit == .inference) @import("inference_runtime/runtime.zig") else struct {}; @@ -225,6 +226,10 @@ fn runData(init: std.process.Init, _: []const u8, args: *std.process.Args.Iterat return data_runtime.runFromIterator(init, "antfly", args); } +fn runGraphMetricMaintenance(init: std.process.Init, _: []const u8, args: *std.process.Args.Iterator) !void { + return graph_metric_maintenance_runtime.runFromIterator(init, "antfly", args); +} + fn runHa(init: std.process.Init, _: []const u8, args: *std.process.Args.Iterator) !void { return ha_runtime.runFromIterator(init, "antfly", args); } @@ -254,6 +259,10 @@ fn dataEntry(context: *const bridge.Context) callconv(.c) c_int { return runtimeEntry(context, "data", runData); } +fn graphMetricMaintenanceEntry(context: *const bridge.Context) callconv(.c) c_int { + return runtimeEntry(context, "graph_metric_maintenance", runGraphMetricMaintenance); +} + fn haEntry(context: *const bridge.Context) callconv(.c) c_int { return runtimeEntry(context, "ha", runHa); } @@ -289,6 +298,7 @@ comptime { // C ABI library names link this exact compiled artifact. _ = storage_kernel_exports; exportInternal(&dataEntry, "antfly_runtime_data"); + exportInternal(&graphMetricMaintenanceEntry, "antfly_runtime_graph_metric_maintenance"); exportInternal(&haEntry, "antfly_runtime_ha"); exportInternal(&metadataEntry, "antfly_runtime_metadata"); exportInternal(&standaloneEntry, "antfly_runtime_standalone"); diff --git a/zig/pkg/antfly/src/runtime_artifact_main.zig b/zig/pkg/antfly/src/runtime_artifact_main.zig index ed602af770..012be57021 100644 --- a/zig/pkg/antfly/src/runtime_artifact_main.zig +++ b/zig/pkg/antfly/src/runtime_artifact_main.zig @@ -22,6 +22,7 @@ const inference_process_supervisor = @import("antfly_platform").inference_proces extern fn antfly_runtime_cli(context: *const bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_data(context: *const bridge.Context) callconv(.c) c_int; +extern fn antfly_runtime_graph_metric_maintenance(context: *const bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_inference(context: *const bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_metadata(context: *const bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_standalone(context: *const bridge.Context) callconv(.c) c_int; @@ -95,6 +96,7 @@ fn mainImpl(init: std.process.Init) anyerror!void { const code = switch (role_options.role) { .cli => antfly_runtime_cli(&context), .data => antfly_runtime_data(&context), + .graph_metric_maintenance => antfly_runtime_graph_metric_maintenance(&context), .inference => antfly_runtime_inference(&context), .metadata => antfly_runtime_metadata(&context), .standalone => if (worker_invocation) antfly_runtime_inference(&context) else antfly_runtime_standalone(&context), diff --git a/zig/pkg/antfly/src/runtime_callback_abi.zig b/zig/pkg/antfly/src/runtime_callback_abi.zig index cfbfd5e5a8..e6a9fecb73 100644 --- a/zig/pkg/antfly/src/runtime_callback_abi.zig +++ b/zig/pkg/antfly/src/runtime_callback_abi.zig @@ -225,6 +225,10 @@ test "boundary dispatcher preserves local calls and maps cross-unit calls" { return error.ProposalDropped; } + fn indexGenerationMismatch(_: *u32) anyerror!void { + return error.IndexGenerationMismatch; + } + fn ambiguousFail(_: *u32) anyerror!void { return error.MetadataMutationOutcomeUnknown; } @@ -276,6 +280,10 @@ test "boundary dispatcher preserves local calls and maps cross-unit calls" { error.ProposalDropped, TestBoundary.call("retryable_fail", &callbacks.foreignDispatch, &callbacks.retryableFail, .{&base}), ); + try std.testing.expectError( + error.IndexGenerationMismatch, + TestBoundary.call("retryable_fail", &callbacks.foreignDispatch, &callbacks.indexGenerationMismatch, .{&base}), + ); try std.testing.expectError( error.MetadataMutationOutcomeUnknown, TestBoundary.call("ambiguous_fail", &callbacks.foreignDispatch, &callbacks.ambiguousFail, .{&base}), diff --git a/zig/pkg/antfly/src/runtime_error_abi.zig b/zig/pkg/antfly/src/runtime_error_abi.zig index 12b0303056..3c8786794b 100644 --- a/zig/pkg/antfly/src/runtime_error_abi.zig +++ b/zig/pkg/antfly/src/runtime_error_abi.zig @@ -307,6 +307,7 @@ pub const Detail = enum(c_int) { ha_sync_commit_wait_standby_not_in_policy, deadline_exceeded, pre_decision_deadline_exceeded, + graph_metric_action_partial_outcome, graph_distinct_budget_exceeded, graph_anchor_filter_requires_index, graph_match_operation_limit_exceeded, @@ -346,8 +347,10 @@ pub const Detail = enum(c_int) { unsupported_generator_provider, generate_request_failed, generation_rate_limit, + index_generation_mismatch, unsupported_tensor_type, generation_capacity_unavailable, + generation_transition_active, invalid_table_storage_settings, vector_store_requires_local_single_shard_table, vector_store_requires_empty_table, @@ -426,6 +429,7 @@ pub fn statusFromError(err: anyerror) Status { error.WriteOutcomeUnknown => status(.retryable, .write_outcome_unknown), error.RaftBatchWriteOutcomeUnknown => status(.retryable, .raft_batch_write_outcome_unknown), error.RaftBatchWritePartialOutcome => status(.retryable, .raft_batch_write_partial_outcome), + error.GraphMetricActionPartialOutcome => status(.retryable, .graph_metric_action_partial_outcome), error.DocIdentityUnavailable => status(.retryable, .doc_identity_unavailable), error.HAReadOnlyStandby => status(.unavailable, .ha_read_only_standby), error.HAPromotedStandbyRequiresPrimaryOpen => status(.unavailable, .ha_promoted_standby_requires_primary_open), @@ -438,10 +442,12 @@ pub fn statusFromError(err: anyerror) Status { error.NotLeader => status(.retryable, .not_leader), error.LeaderUnavailable => status(.unavailable, .leader_unavailable), error.TopologyChanged => status(.retryable, .topology_changed), + error.IndexGenerationMismatch => status(.retryable, .index_generation_mismatch), error.IdentityReadGenerationChanged => status(.conflict, .identity_read_generation_changed), error.DocIdentityNamespaceMismatch => status(.conflict, .doc_identity_namespace_mismatch), error.TableGenerationChanged => status(.conflict, .table_generation_changed), error.GenerationDurabilityUncertain => status(.retryable, .generation_durability_uncertain), + error.GenerationTransitionActive => status(.retryable, .generation_transition_active), error.IndexRebuilding => status(.retryable, .index_rebuilding), error.TableVisibilityTimeout => status(.timeout, .table_visibility_timeout), error.WriterLocked => status(.retryable, .writer_locked), @@ -788,10 +794,12 @@ fn detailErrorName(comptime detail: Detail) []const u8 { .internal_failure => "InternalFailure", .not_leader => "NotLeader", .topology_changed => "TopologyChanged", + .index_generation_mismatch => "IndexGenerationMismatch", .identity_read_generation_changed => "IdentityReadGenerationChanged", .doc_identity_namespace_mismatch => "DocIdentityNamespaceMismatch", .table_generation_changed => "TableGenerationChanged", .generation_durability_uncertain => "GenerationDurabilityUncertain", + .generation_transition_active => "GenerationTransitionActive", .index_rebuilding => "IndexRebuilding", .table_visibility_timeout => "TableVisibilityTimeout", .writer_locked => "WriterLocked", @@ -1001,6 +1009,7 @@ fn detailErrorName(comptime detail: Detail) []const u8 { .ha_sync_commit_wait_standby_not_in_policy => "HASyncCommitWaitStandbyNotInPolicy", .deadline_exceeded => "DeadlineExceeded", .pre_decision_deadline_exceeded => "PreDecisionDeadlineExceeded", + .graph_metric_action_partial_outcome => "GraphMetricActionPartialOutcome", .graph_distinct_budget_exceeded => "GraphDistinctBudgetExceeded", .graph_anchor_filter_requires_index => "GraphAnchorFilterRequiresIndex", .graph_match_operation_limit_exceeded => "GraphMatchOperationLimitExceeded", @@ -1037,6 +1046,10 @@ fn detailErrorName(comptime detail: Detail) []const u8 { } test "stable status preserves public boundary semantics" { + try std.testing.expectEqual(error.GenerationTransitionActive, errorFromStatus(statusFromError(error.GenerationTransitionActive))); + try std.testing.expectEqual(@intFromEnum(Code.retryable), statusFromError(error.GenerationTransitionActive).code); + try std.testing.expectEqual(error.IndexGenerationMismatch, errorFromStatus(statusFromError(error.IndexGenerationMismatch))); + try std.testing.expectEqual(@intFromEnum(Code.retryable), statusFromError(error.IndexGenerationMismatch).code); try std.testing.expect(Status.ok.isOk()); try std.testing.expectEqual(error.TableNotFound, errorFromStatus(statusFromError(error.TableNotFound))); try std.testing.expectEqual(error.TableVisibilityTimeout, errorFromStatus(statusFromError(error.TableVisibilityTimeout))); diff --git a/zig/pkg/antfly/src/serverless/api/http_handler.zig b/zig/pkg/antfly/src/serverless/api/http_handler.zig index c2fd6722c6..d4091d8ba6 100644 --- a/zig/pkg/antfly/src/serverless/api/http_handler.zig +++ b/zig/pkg/antfly/src/serverless/api/http_handler.zig @@ -45,6 +45,7 @@ const db_query_graph = @import("../../storage/db/query/graph_exec.zig"); const db_embedder = @import("../../storage/db/enrichment/embedder.zig"); const distributed_stats_mod = @import("../../search/distributed_stats.zig"); const graph_mod = @import("../../graph/graph.zig"); +const graph_metric_rerank = @import("../../graph/metric_rerank.zig"); const graph_pattern_mod = @import("../../graph/pattern.zig"); const graph_work_budget_mod = @import("../../graph/work_budget.zig"); const graph_node_admission = @import("../../graph/node_admission.zig"); @@ -74,8 +75,23 @@ const managed_embedder = @import("../../inference/managed_embedder.zig"); const scraping = @import("antfly_scraping"); const platform_time = @import("antfly_platform").time; const graph_segment_mod = @import("../graph_segment/mod.zig"); +const graph_metric_segment_mod = @import("../graph_metric_segment/mod.zig"); const foreign_mod = @import("../../foreign/mod.zig"); const query_execution = @import("query_execution.zig"); + +const SyncWaitCancellation = struct { + upstream: CancellationToken, + deadline_ns: u64, + + fn token(self: *const @This()) CancellationToken { + return .{ .ptr = self, .is_cancelled_fn = isCancelled }; + } + + fn isCancelled(ptr: *const anyopaque) bool { + const state: *const @This() = @ptrCast(@alignCast(ptr)); + return state.upstream.isCancelled() or platform_time.monotonicNs() >= state.deadline_ns; + } +}; const json_helpers = @import("../../api/json_helpers.zig"); const ParsedJsonPathValue = json_helpers.ParsedJsonPathValue; const parseJsonValueAlloc = json_helpers.parseJsonValueAlloc; @@ -739,6 +755,8 @@ pub const HttpHandler = struct { else => return err, }; validateServerlessIndexCatalog(self.alloc, indexes_json) catch |err| switch (err) { + error.UnsupportedGraphMetricRefreshMode => return try textResponse(self.alloc, 400, "manual graph metric refresh is not supported in serverless; use background refresh"), + error.GraphMetricConfigurationLimitExceeded => return try textResponse(self.alloc, 400, "graph metric configuration exceeds serverless limits (maximum 16 graph metric indexes, 16 metrics per graph, 64 metrics total, 64 edge types per filter, 256-byte index names, and 128-byte metric names)"), error.UnsupportedServerlessArtifactIndexSources => return try unsupportedArtifactIndexSourcesResponse(self.alloc), error.UnsupportedCreateTableRequest, error.InvalidTableIndexMetadata => return try textResponse(self.alloc, 400, "unsupported table index configuration"), else => return err, @@ -785,7 +803,9 @@ pub const HttpHandler = struct { return error.InternalFailure; }; defer status.deinit(self.alloc); - const body = encodeServerlessIndexListAlloc(self.alloc, table.indexes_json, status) catch |err| { + const metric_statuses = try self.graphMetricIndexStatusesAlloc(table_name, table.indexes_json, null, .none); + defer freeServerlessGraphMetricStatuses(self.alloc, metric_statuses); + const body = encodeServerlessIndexListWithGraphMetricsAlloc(self.alloc, table.indexes_json, status, metric_statuses) catch |err| { std.log.err("table index list encode failed table={s} err={}", .{ table_name, err }); return error.InternalFailure; }; @@ -801,7 +821,9 @@ pub const HttpHandler = struct { return error.InternalFailure; }; defer status.deinit(self.alloc); - const body = (encodeServerlessSingleIndexAlloc(self.alloc, table.indexes_json, index_name, status) catch |err| { + const metric_statuses = try self.graphMetricIndexStatusesAlloc(table_name, table.indexes_json, index_name, .none); + defer freeServerlessGraphMetricStatuses(self.alloc, metric_statuses); + const body = (encodeServerlessSingleIndexWithGraphMetricsAlloc(self.alloc, table.indexes_json, index_name, status, metric_statuses) catch |err| { std.log.err("table index encode failed table={s} index={s} err={}", .{ table_name, index_name, err }); return error.InternalFailure; }) orelse { @@ -811,6 +833,108 @@ pub const HttpHandler = struct { return try jsonSliceResponse(self.alloc, 200, body); } + fn graphMetricIndexStatusesAlloc( + self: *HttpHandler, + table_name: []const u8, + indexes_json: []const u8, + only_index_name: ?[]const u8, + cancellation: CancellationToken, + ) ![]ServerlessGraphMetricStatus { + try cancellation.check(); + const specs = try build_mod.graph_metric_config.parseIndexSpecsAlloc(self.alloc, indexes_json); + defer build_mod.graph_metric_config.freeIndexSpecs(self.alloc, specs); + var status_count: usize = 0; + for (specs) |spec| { + if (only_index_name) |name| if (!std.mem.eql(u8, spec.index_name, name)) continue; + status_count = std.math.add(usize, status_count, spec.configs.len) catch return error.OutOfMemory; + } + if (status_count == 0) return try self.alloc.alloc(ServerlessGraphMetricStatus, 0); + + const statuses = try self.alloc.alloc(ServerlessGraphMetricStatus, status_count); + var initialized: usize = 0; + errdefer { + for (statuses[0..initialized]) |*status| status.deinit(self.alloc); + self.alloc.free(statuses); + } + + const namespace = self.catalog.resolveTableNamespaceAlloc(table_name) catch |err| switch (err) { + error.FileNotFound => null, + else => return err, + }; + defer if (namespace) |value| self.alloc.free(value); + var maybe_session: ?query_mod.QuerySession = if (namespace) |value| + self.query.openHeadSession(value) catch |err| switch (err) { + error.FileNotFound => null, + else => return err, + } + else + null; + defer if (maybe_session) |*session| session.deinit(); + if (maybe_session) |*session| session.setCancellation(cancellation); + + for (specs) |spec| { + if (only_index_name) |name| if (!std.mem.eql(u8, spec.index_name, name)) continue; + for (spec.configs) |config| { + try cancellation.check(); + const owned_index_name = try self.alloc.dupe(u8, spec.index_name); + var index_name_moved = false; + errdefer if (!index_name_moved) self.alloc.free(owned_index_name); + const owned_metric_name = try self.alloc.dupe(u8, config.name); + var metric_name_moved = false; + errdefer if (!metric_name_moved) self.alloc.free(owned_metric_name); + var status = ServerlessGraphMetricStatus{ + .index_name = owned_index_name, + .metric_name = owned_metric_name, + .kind = config.kind, + .config_fingerprint = build_mod.lake_graph_metric.configFingerprint(config), + .state = .pending, + }; + index_name_moved = true; + metric_name_moved = true; + var status_moved = false; + errdefer if (!status_moved) status.deinit(self.alloc); + + if (maybe_session) |*session| { + const graph_index = session.findNamedArtifactIndex(.graph_segment, spec.index_name); + const artifact_name = try graph_metric_segment_mod.artifactNameAlloc(self.alloc, spec.index_name, config.name); + defer self.alloc.free(artifact_name); + const metric_index = session.findNamedArtifactIndex(.graph_metric_segment, artifact_name); + if (graph_index != null and metric_index != null) { + const metric_ref = session.artifactRef(metric_index.?).?; + const graph_ref = session.artifactRef(graph_index.?).?; + status.published_generation = if (metric_ref.published_generation != 0) + metric_ref.published_generation + else + session.manifest.version; + status.materializer_fingerprint = metric_ref.materializer_fingerprint; + const source_checksum = blk: { + artifacts_mod.validateSha256ArtifactIdentity(graph_ref.artifact_id, graph_ref.checksum) catch break :blk null; + break :blk artifacts_mod.sha256DigestFromChecksum(graph_ref.checksum) catch null; + }; + const valid_identity = metric_ref.graph_metric_config_fingerprint == status.config_fingerprint; + const valid_source = if (source_checksum) |digest| + std.mem.eql(u8, &digest, &metric_ref.graph_metric_source_checksum) + else + false; + const current_policy = metric_ref.materializer_fingerprint == build_mod.lake_graph_metric.materializerFingerprint(.{}); + const current_format = metric_ref.metadata_version == graph_metric_segment_mod.wire_version; + status.state = if (!valid_identity or !valid_source or !current_policy or !current_format) + .stale + else switch (metric_ref.graph_metric_materialization_state) { + .ready => .ready, + .rejected => .rejected, + }; + status.rejection_reason = @enumFromInt(@intFromEnum(metric_ref.graph_metric_rejection_reason)); + } + } + statuses[initialized] = status; + initialized += 1; + status_moved = true; + } + } + return statuses; + } + fn handleCreateTableIndex(self: *HttpHandler, table_name: []const u8, index_name: []const u8, body: []const u8) !HttpResponse { if (try self.requireMutableRoute()) |resp| return resp; var table = (try self.catalog.getTableAlloc(self.alloc, table_name)) orelse return try textResponse(self.alloc, 404, "not found"); @@ -838,6 +962,8 @@ pub const HttpHandler = struct { const next_indexes_json = try indexes_api.addIndexToTableIndexesJson(self.alloc, table.indexes_json, index_name, expanded_index_json); defer self.alloc.free(next_indexes_json); validateServerlessIndexCatalog(self.alloc, next_indexes_json) catch |err| switch (err) { + error.UnsupportedGraphMetricRefreshMode => return try textResponse(self.alloc, 400, "manual graph metric refresh is not supported in serverless; use background refresh"), + error.GraphMetricConfigurationLimitExceeded => return try textResponse(self.alloc, 400, "graph metric configuration exceeds serverless limits (maximum 16 graph metric indexes, 16 metrics per graph, 64 metrics total, 64 edge types per filter, 256-byte index names, and 128-byte metric names)"), error.UnsupportedServerlessArtifactIndexSources => return try unsupportedArtifactIndexSourcesResponse(self.alloc), error.UnsupportedCreateTableRequest => return try textResponse(self.alloc, 400, "unsupported index configuration"), error.InvalidTableIndexMetadata => return try textResponse(self.alloc, 400, "invalid index configuration"), @@ -945,13 +1071,16 @@ pub const HttpHandler = struct { var resp = try public_table_http.handleTableBatch(self.alloc, table_name, body, self.tableApi(cancellation)); defer resp.deinit(self.alloc); return switch (resp.status) { - 201 => blk: { + 201, 202 => blk: { var arena_impl = std.heap.ArenaAllocator.init(self.alloc); defer arena_impl.deinit(); const parsed = try parseJsonResponseBody(metadata_openapi.BatchResponse, arena_impl.allocator(), resp.body); - break :blk try jsonResponse(self.alloc, 201, parsed); + break :blk try jsonResponse(self.alloc, resp.status, parsed); }, - else => try textResponse(self.alloc, resp.status, resp.body), + else => if (resp.json) + try jsonSliceResponse(self.alloc, resp.status, resp.body) + else + try textResponse(self.alloc, resp.status, resp.body), }; } @@ -1170,7 +1299,14 @@ pub const HttpHandler = struct { }; } - fn executePublishedSearch(self: *HttpHandler, namespace: []const u8, table_name: ?[]const u8, body: []const u8, cancellation: CancellationToken) !SearchExecution { + fn executePublishedSearch( + self: *HttpHandler, + namespace: []const u8, + table_name: ?[]const u8, + body: []const u8, + cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, + ) !SearchExecution { try cancellation.check(); var status = try self.catalog.buildStatus(namespace); errdefer status.deinit(self.alloc); @@ -1187,6 +1323,12 @@ pub const HttpHandler = struct { var session = try self.query.openHeadSession(namespace); errdefer session.deinit(); session.setCancellation(cancellation); + session.setDiagnostics(diagnostics); + // Install the runtime before any query work. Graph-metric reranking + // happens immediately after this function returns its pinned session; + // deferring setIo until graph traversal silently serialized all of its + // independent immutable range reads. + session.setIo(self.io); var execution_stats = query_mod.QuerySearchExecutionStats{}; const hits = try query_mod.searchIndexedPlanWithStatsAlloc(self.alloc, &session, plan, &execution_stats); @@ -1248,10 +1390,10 @@ pub const HttpHandler = struct { var owned = parsed_join; owned.deinit(self.alloc); } - return try self.executeSupportedJoinedPublicTableQueryRequest(table_name, body, parsed_join.join, parsed_join.foreign_sources, cancellation); + return try self.executeSupportedJoinedPublicTableQueryRequest(table_name, body, parsed_join.join, parsed_join.foreign_sources, cancellation, null); } - return try self.executePlainPublicTableQueryJsonValueAlloc(table_name, body, raw_request.value, cancellation); + return try self.executePlainPublicTableQueryJsonValueAlloc(table_name, body, raw_request.value, cancellation, null); } fn executeForeignPublicTableQueryJsonValueAlloc( @@ -1606,7 +1748,13 @@ pub const HttpHandler = struct { return hits; } - fn executePlainPublicTableQueryJsonAlloc(self: *HttpHandler, table_name: []const u8, body: []const u8, cancellation: CancellationToken) anyerror![]u8 { + fn executePlainPublicTableQueryJsonAlloc( + self: *HttpHandler, + table_name: []const u8, + body: []const u8, + cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, + ) anyerror![]u8 { var raw_request = ant_json.parseFromSlice(std.json.Value, self.alloc, body, .{}) catch return error.InvalidQueryRequest; defer raw_request.deinit(); @@ -1616,6 +1764,7 @@ pub const HttpHandler = struct { body, raw_request.value, cancellation, + diagnostics, ); } @@ -1625,12 +1774,13 @@ pub const HttpHandler = struct { body: []const u8, raw_request: std.json.Value, cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, ) anyerror![]u8 { try cancellation.check(); const namespace = self.catalog.resolveTableNamespaceAlloc(table_name) catch return error.FileNotFound; defer self.alloc.free(namespace); - const graph_response = self.handleTablePublicGraphQueryRequestValue(table_name, namespace, body, raw_request, cancellation) catch |err| switch (err) { + const graph_response = self.handleTablePublicGraphQueryRequestValue(table_name, namespace, body, raw_request, cancellation, diagnostics) catch |err| switch (err) { // Once the graph boundary has recognized the request, unsupported // semantics are an exact-execution capability response, not a // malformed request or an internal server failure. @@ -1655,7 +1805,7 @@ pub const HttpHandler = struct { }; defer if (aggregations_json) |json| self.alloc.free(json); - var execution = self.executePublishedSearch(namespace, table_name, body, cancellation) catch |err| { + var execution = self.executePublishedSearch(namespace, table_name, body, cancellation, diagnostics) catch |err| { switch (err) { error.InvalidQueryRequest, error.EmbeddingIndexNotFound, @@ -2274,6 +2424,7 @@ pub const HttpHandler = struct { join: SupportedJoinRequest, foreign_sources: foreign_mod.PostgresSourceMap, cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, ) anyerror![]u8 { try cancellation.check(); var contract_request = std.json.parseFromSlice(metadata_openapi.QueryRequest, self.alloc, body, .{ @@ -2288,7 +2439,7 @@ pub const HttpHandler = struct { const primary_body = rewrite.body; defer self.alloc.free(primary_body); - const primary_json = self.executePlainPublicTableQueryJsonAlloc(table_name, primary_body, cancellation) catch |err| { + const primary_json = self.executePlainPublicTableQueryJsonAlloc(table_name, primary_body, cancellation, diagnostics) catch |err| { std.log.warn("serverless joined query rejected rewritten left input table={s} err={}", .{ table_name, err }); return err; }; @@ -2700,15 +2851,28 @@ pub const HttpHandler = struct { fn handleTableQueryRequest(self: *HttpHandler, table_name: []const u8, body: []const u8, cancellation: CancellationToken) !HttpResponse { try cancellation.check(); + var diagnostics = api_operation.RequestDiagnostics{}; var resp = try public_table_http.handleTableQueryRequest( self.alloc, table_name, body, null, - self.tableApi(cancellation), + self.tableApiWithDiagnostics(cancellation, &diagnostics), ); defer resp.deinit(self.alloc); try cancellation.check(); + if (resp.status == 422) { + if (diagnostics.graph_metric_rejection) |*diagnostic| { + const rejection_body = try public_table_http.graphMetricMaterializationRejectedBodyWithContext( + self.alloc, + diagnostic.graphIndexName(), + diagnostic.metricName(), + diagnostic.materializer_fingerprint, + ); + defer self.alloc.free(rejection_body); + return try jsonSliceResponse(self.alloc, 422, rejection_body); + } + } return try adaptPublicTableQueryResponse(self.alloc, resp); } @@ -2729,6 +2893,7 @@ pub const HttpHandler = struct { body, raw_request.value, cancellation, + null, ); } @@ -2739,6 +2904,7 @@ pub const HttpHandler = struct { body: []const u8, raw_request: std.json.Value, cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, ) !?HttpResponse { try cancellation.check(); if (raw_request != .object) return error.InvalidQueryRequest; @@ -2751,8 +2917,11 @@ pub const HttpHandler = struct { ); return error.UnsupportedQueryRequest; } - const graph_request = raw_request.object.get("graph_queries") orelse return null; - if (graph_request == .null) return error.InvalidQueryRequest; + const graph_request = raw_request.object.get("graph_queries"); + if (graph_request) |value| if (value == .null) return error.InvalidQueryRequest; + const has_graph_metric = public_search_request.hasNonNullField(raw_request.object, "graph_metric"); + const has_graph_metric_rerank = public_search_request.hasNonNullField(raw_request.object, "graph_metric_rerank"); + if (graph_request == null and !has_graph_metric and !has_graph_metric_rerank) return null; const unsupported_controls = [_][]const u8{ "aggregations", @@ -2785,9 +2954,6 @@ pub const HttpHandler = struct { }) catch return error.InvalidQueryRequest; defer parsed_request.deinit(); const request = parsed_request.value; - if (request.graph_queries == null) - return error.InvalidQueryRequest; - const started_ns = platform_time.monotonicNs(); const graph_queries = public_graph_query.parseCanonicalGraphQueriesAlloc(self.alloc, request) catch |err| { std.log.warn("serverless public graph request admission failed table={s} err={}", .{ table_name, err }); @@ -2795,25 +2961,30 @@ pub const HttpHandler = struct { }; defer public_graph_query.freeNamedGraphQueries(self.alloc, graph_queries); + var metric_requests = try query_api.parseGraphMetricRequestsAlloc(self.alloc, body); + defer metric_requests.deinit(self.alloc); + var req: db_types.SearchRequest = .{ .count_only = request.count == true, .profile = request.profile == true, .graph_queries = graph_queries, + .graph_metric_queries = metric_requests.queries, + .graph_metric_rerank = metric_requests.rerank, .limit = if (request.limit) |limit| std.math.cast(u32, limit) orelse 10 else 10, .offset = if (request.offset) |offset| std.math.cast(u32, offset) orelse 0 else 0, .cancellation = cancellation, }; - const canonical_operations = raw_request.object.get("graph_queries") orelse - return error.InvalidQueryRequest; - req.graph_query_transport = graph_wire_envelope.captureCanonicalOperationsAlloc( - self.alloc, - canonical_operations, - graph_queries, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => return error.InvalidQueryRequest, - }; - defer req.graph_query_transport.?.deinit(self.alloc); + if (graph_request) |canonical_operations| { + req.graph_query_transport = graph_wire_envelope.captureCanonicalOperationsAlloc( + self.alloc, + canonical_operations, + graph_queries, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.InvalidQueryRequest, + }; + } + defer if (req.graph_query_transport) |*transport| transport.deinit(self.alloc); var search_hits: []db_types.SearchHit = &.{}; defer if (search_hits.len > 0) freeDbSearchHits(self.alloc, search_hits); var search_total_hits: u32 = 0; @@ -2823,14 +2994,20 @@ pub const HttpHandler = struct { var session: query_mod.QuerySession = undefined; var session_initialized = false; defer if (session_initialized) session.deinit(); + var graph_metric_rerank_status: ?db_types.GraphMetricStatus = null; + defer if (graph_metric_rerank_status) |*status| status.deinit(self.alloc); if (requestHasSearchInputs(request)) { - var search_request = request; - search_request.graph_queries = null; + var search_request = requestWithoutGraphControls(request); + if (metric_requests.rerank) |rerank| { + try db_types.validateGraphMetricRerankWindow(rerank, req.offset, req.limit); + search_request.offset = 0; + search_request.limit = @intCast(db_types.graphMetricRerankCandidateCount(rerank, req.offset, req.limit)); + } const search_body = try std.json.Stringify.valueAlloc(self.alloc, search_request, .{}); defer self.alloc.free(search_body); - var execution = self.executePublishedSearch(namespace, table_name, search_body, cancellation) catch |err| switch (err) { + var execution = self.executePublishedSearch(namespace, table_name, search_body, cancellation, diagnostics) catch |err| switch (err) { error.InvalidQueryRequest, error.EmbeddingIndexNotFound, error.InvalidEmbeddingDimensions, @@ -2850,6 +3027,21 @@ pub const HttpHandler = struct { req.profile = execution.profile_requested; search_hits = try allocDbSearchHitsAlloc(self.alloc, execution.hits); search_total_hits = publicGraphSeedTotalHits(execution.hits.len, req.limit); + + session = execution.takeSession(); + session_initialized = true; + if (metric_requests.rerank) |rerank| { + if (req.count_only) return error.UnsupportedQueryRequest; + const reranked = try self.applyPublicGraphMetricRerank( + &session, + search_hits, + rerank, + req.offset, + req.limit, + ); + graph_metric_rerank_status = reranked.status; + search_hits = reranked.hits; + } try initial_sets.append(self.alloc, .{ .name = "$query_results", .hits = search_hits, @@ -2881,9 +3073,6 @@ pub const HttpHandler = struct { .total_hits = search_total_hits, }); } - - session = execution.takeSession(); - session_initialized = true; } else { session = self.query.openHeadSession(namespace) catch |err| switch (err) { error.FileNotFound => return try textResponse(self.alloc, 404, "not found"), @@ -2891,7 +3080,14 @@ pub const HttpHandler = struct { }; session_initialized = true; session.setCancellation(cancellation); + session.setDiagnostics(diagnostics); + } + session.setIo(self.io); + if (metric_requests.rerank) |rerank| { + _ = rerank; + if (!requestHasSearchInputs(request)) return error.UnsupportedQueryRequest; } + const results = self.executePublicGraphQueriesAlloc(&session, table_name, graph_queries, initial_sets.items) catch |err| { std.log.warn("serverless public graph request execution failed table={s} err={}", .{ table_name, err }); return err; @@ -2901,11 +3097,19 @@ pub const HttpHandler = struct { if (results.len > 0) self.alloc.free(results); } + const metric_results = try self.executePublicGraphMetricQueriesAlloc(&session, metric_requests.queries); + defer { + for (metric_results) |*metric_result| metric_result.deinit(self.alloc); + if (metric_results.len > 0) self.alloc.free(metric_results); + } + const result: db_types.SearchResult = .{ .alloc = self.alloc, .hits = search_hits, .total_hits = search_total_hits, .graph_results = results, + .graph_metric_results = metric_results, + .graph_metric_rerank_status = graph_metric_rerank_status, }; var response = query_api.encodeQueryResponses( @@ -2922,6 +3126,775 @@ pub const HttpHandler = struct { return try typedJsonResponse(metadata_openapi.QueryResponses, self.alloc, 200, response.json); } + fn executePublicGraphMetricQueriesAlloc( + self: *HttpHandler, + session: *query_mod.QuerySession, + queries: []const db_types.NamedGraphMetricQuery, + ) ![]db_types.GraphMetricResult { + const results = try self.alloc.alloc(db_types.GraphMetricResult, queries.len); + errdefer self.alloc.free(results); + var initialized: usize = 0; + errdefer for (results[0..initialized]) |*result| result.deinit(self.alloc); + for (queries, 0..) |named, i| { + try session.checkCancellation(); + var metric = try query_mod.graphMetricTopAlloc(self.alloc, session, named.query.index_name, named.query.metric_name, named.query.top_k); + defer metric.deinit(self.alloc); + const scores = try metric.takePublicScoresAlloc(self.alloc, session); + errdefer { + for (scores) |*score| score.deinit(self.alloc); + self.alloc.free(scores); + } + const name = try self.alloc.dupe(u8, named.name); + errdefer self.alloc.free(name); + const index_name = try self.alloc.dupe(u8, named.query.index_name); + errdefer self.alloc.free(index_name); + const metric_name = try self.alloc.dupe(u8, named.query.metric_name); + errdefer self.alloc.free(metric_name); + var status = try self.graphMetricStatusAlloc( + named.query.metric_name, + metric.config_fingerprint, + metric.edge_filter, + metric.converged, + metric.iterations_completed, + metric.delta, + metric.metadata_version, + metric.published_generation, + metric.edge_generation, + metric.computed_at_ms, + ); + errdefer status.deinit(self.alloc); + results[i] = .{ + .name = name, + .index_name = index_name, + .metric_name = metric_name, + .scores = scores, + .status = status, + }; + initialized += 1; + } + return results; + } + + const PublicGraphMetricColumns = struct { + columns: []query_mod.graph_metric_reader.ScoreColumn, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.columns) |*column| column.deinit(alloc); + if (self.columns.len > 0) alloc.free(self.columns); + self.* = undefined; + } + }; + + const CachedPublicGraphMetricColumn = struct { + scores: []?f64, + memory: query_mod.runtime.GraphMetricReadBudget.Reservation = .{}, + /// Stable indexes into the original graph result. This slice is owned + /// by the request workspace and shared by columns loaded together. + source_rows: []const usize, + }; + + fn metricDependencyIndex(names: []const []const u8, name: []const u8) ?usize { + for (names, 0..) |candidate, i| if (std.mem.eql(u8, candidate, name)) return i; + return null; + } + + fn releaseUnusedPublicGraphMetricColumns( + self: *HttpHandler, + dependency_names: []const []const u8, + cached_columns: []?CachedPublicGraphMetricColumn, + first_future_names: []const []const u8, + second_future_names: []const []const u8, + ) void { + std.debug.assert(dependency_names.len == cached_columns.len); + for (dependency_names, cached_columns) |name, *maybe_column| { + if (metricDependencyIndex(first_future_names, name) != null or + metricDependencyIndex(second_future_names, name) != null) + { + continue; + } + var column = maybe_column.* orelse continue; + if (column.scores.len > 0) self.alloc.free(column.scores); + column.memory.deinit(); + maybe_column.* = null; + } + } + + fn loadPublicGraphMetricColumns( + self: *HttpHandler, + session: *query_mod.QuerySession, + graph_index_name: []const u8, + metric_names: []const []const u8, + nodes: []const graph_query_mod.GraphResultNode, + source_rows: []const usize, + dependency_names: []const []const u8, + statuses: []db_types.GraphMetricStatus, + status_initialized: []bool, + ) !PublicGraphMetricColumns { + if (statuses.len != dependency_names.len or status_initialized.len != dependency_names.len) + return error.InvalidQueryRequest; + const columns = try self.alloc.alloc(query_mod.graph_metric_reader.ScoreColumn, metric_names.len); + var initialized_columns: usize = 0; + errdefer { + for (columns[0..initialized_columns]) |*column| column.deinit(self.alloc); + if (columns.len > 0) self.alloc.free(columns); + } + + // Metric segments contain only nodes local to the query table. Keep + // table-qualified identities out of the lookup entirely; flattening + // them to `key` could alias an unrelated local node with the same key. + var local_node_count: usize = 0; + for (source_rows) |source_row| { + if (source_row >= nodes.len) return error.InvalidQueryRequest; + local_node_count += @intFromBool(graphMetricLocalNodeId(nodes[source_row]) != null); + } + var adapter_memory = try session.reserveGraphMetricMemory(std.math.mul(usize, local_node_count, @sizeOf([]const u8) + @sizeOf(usize)) catch + return error.GraphMetricQueryBudgetExceeded); + defer adapter_memory.deinit(); + const node_ids = try self.alloc.alloc([]const u8, local_node_count); + defer self.alloc.free(node_ids); + const local_node_indexes = try self.alloc.alloc(usize, local_node_count); + defer self.alloc.free(local_node_indexes); + var local_node_index: usize = 0; + for (source_rows, 0..) |source_row, result_index| { + const node = nodes[source_row]; + const node_id = graphMetricLocalNodeId(node) orelse continue; + node_ids[local_node_index] = node_id; + local_node_indexes[local_node_index] = result_index; + local_node_index += 1; + } + + var point_score_columns = try query_mod.graph_metric_reader.scoreColumnsScopedAlloc( + self.alloc, + session, + graph_index_name, + metric_names, + node_ids, + ); + defer point_score_columns.deinit(self.alloc); + for (metric_names, point_score_columns.columns, 0..) |metric_name, *point_scores, i| { + const dependency_index = metricDependencyIndex(dependency_names, metric_name) orelse + return error.InvalidQueryRequest; + if (!status_initialized[dependency_index]) { + statuses[dependency_index] = try self.graphMetricStatusAlloc(metric_name, point_scores.config_fingerprint, point_scores.edge_filter, point_scores.converged, point_scores.iterations_completed, point_scores.delta, point_scores.metadata_version, point_scores.published_generation, point_scores.edge_generation, point_scores.computed_at_ms); + status_initialized[dependency_index] = true; + } + var local_column = try point_scores.takeColumn(); + errdefer local_column.deinit(self.alloc); + var aligned_memory: query_mod.runtime.GraphMetricReadBudget.Reservation = .{}; + errdefer aligned_memory.deinit(); + if (local_node_count != source_rows.len) { + const aligned_bytes = std.math.mul(usize, source_rows.len, @sizeOf(?f64)) catch + return error.GraphMetricQueryBudgetExceeded; + // The local result remains live while the aligned column is + // allocated, so charge the peak expansion rather than only + // the eventual delta. + aligned_memory = try session.reserveGraphMetricMemory(aligned_bytes); + } + const aligned = try scatterLocalGraphMetricScoresAlloc( + self.alloc, + source_rows.len, + local_node_indexes, + local_column.scores, + ); + if (local_node_count != source_rows.len) { + // Scatter consumed/freed the local allocation. + local_column.memory.deinit(); + columns[i] = .{ .scores = aligned, .memory = aligned_memory }; + } else columns[i] = .{ .scores = aligned, .memory = local_column.memory }; + initialized_columns += 1; + } + return .{ .columns = columns }; + } + + fn ensurePublicGraphMetricColumns( + self: *HttpHandler, + session: *query_mod.QuerySession, + graph_index_name: []const u8, + requested_names: []const []const u8, + nodes: []const graph_query_mod.GraphResultNode, + source_rows: []const usize, + dependency_names: []const []const u8, + statuses: []db_types.GraphMetricStatus, + status_initialized: []bool, + cached_columns: []?CachedPublicGraphMetricColumn, + ) !void { + var missing_names: [graph_query_mod.graph_metric_dependency_limit][]const u8 = undefined; + var missing_count: usize = 0; + for (requested_names) |name| { + const dependency_index = metricDependencyIndex(dependency_names, name) orelse + return error.InvalidQueryRequest; + if (cached_columns[dependency_index] != null) continue; + missing_names[missing_count] = name; + missing_count += 1; + } + if (missing_count == 0) return; + var loaded = try self.loadPublicGraphMetricColumns( + session, + graph_index_name, + missing_names[0..missing_count], + nodes, + source_rows, + dependency_names, + statuses, + status_initialized, + ); + defer loaded.deinit(self.alloc); + for (missing_names[0..missing_count], loaded.columns) |name, *column| { + const dependency_index = metricDependencyIndex(dependency_names, name).?; + cached_columns[dependency_index] = .{ .scores = column.scores, .memory = column.memory, .source_rows = source_rows }; + column.* = .{ .scores = &.{} }; + } + } + + fn publicGraphMetricStageColumns( + dependency_names: []const []const u8, + stage_names: []const []const u8, + cached_columns: []const ?CachedPublicGraphMetricColumn, + active_rows: []const usize, + out: [][]?f64, + ) !void { + if (out.len != stage_names.len) return error.InvalidQueryRequest; + for (stage_names, out) |name, *column| { + const dependency_index = metricDependencyIndex(dependency_names, name) orelse + return error.InvalidQueryRequest; + const cached = cached_columns[dependency_index] orelse return error.InvalidQueryRequest; + // Every selection transition rebases all resident columns with the + // direct parent indexes returned by the shared selector. A cache + // entry from another lineage is corrupt request state; recovering + // its position with per-row binary searches hides that invariant + // and turns repeated clauses into O(columns * rows * log(rows)). + if (!std.mem.eql(usize, cached.source_rows, active_rows)) + return error.InvalidQueryRequest; + column.* = cached.scores; + } + } + + fn rebasePublicGraphMetricColumns( + self: *HttpHandler, + session: *query_mod.QuerySession, + cached_columns: []?CachedPublicGraphMetricColumn, + current_rows: []const usize, + next_rows: []const usize, + selected_parent_indexes: []const usize, + ) !void { + if (next_rows.len != selected_parent_indexes.len) return error.InvalidQueryRequest; + if (cached_columns.len > graph_query_mod.graph_metric_dependency_limit) + return error.InvalidQueryRequest; + + // Rebase transactionally. A late allocation or corrupt parent index + // must leave every cache entry on the old lineage; otherwise earlier + // entries would retain source_rows pointing at a selection the caller + // is about to free while later entries still describe current_rows. + var replacements: [graph_query_mod.graph_metric_dependency_limit]?[]?f64 = @splat(null); + var reservations: [graph_query_mod.graph_metric_dependency_limit]query_mod.runtime.GraphMetricReadBudget.Reservation = @splat(.{}); + defer for (&reservations) |*reservation| reservation.deinit(); + errdefer for (replacements[0..cached_columns.len]) |maybe_scores| if (maybe_scores) |scores| self.alloc.free(scores); + for (cached_columns, 0..) |maybe_column, column_index| { + const cached = maybe_column orelse continue; + if (!std.mem.eql(usize, cached.source_rows, current_rows) or cached.scores.len != current_rows.len) + return error.InvalidQueryRequest; + const retained_bytes = std.math.mul(usize, next_rows.len, @sizeOf(?f64)) catch + return error.GraphMetricQueryBudgetExceeded; + reservations[column_index] = try session.reserveGraphMetricMemory(retained_bytes); + const rebased = try self.alloc.alloc(?f64, next_rows.len); + // Register ownership before validating/copying the selector. The + // transaction rollback must also cover malformed parent ordinals. + replacements[column_index] = rebased; + for (selected_parent_indexes, rebased) |parent_index, *score| { + if (parent_index >= cached.scores.len) return error.InvalidQueryRequest; + score.* = cached.scores[parent_index]; + } + } + for (cached_columns, replacements[0..cached_columns.len], reservations[0..cached_columns.len]) |*maybe_column, *maybe_replacement, *reservation| { + const cached = if (maybe_column.*) |*value| value else continue; + const rebased = maybe_replacement.*.?; + if (cached.scores.len > 0) self.alloc.free(cached.scores); + cached.memory.deinit(); + cached.scores = rebased; + cached.memory = reservation.*; + reservation.* = .{}; + cached.source_rows = next_rows; + maybe_replacement.* = null; + } + } + + fn applyPublicGraphMetricShape( + self: *HttpHandler, + session: *query_mod.QuerySession, + query: graph_query_mod.GraphQuery, + result: *db_types.GraphSearchResult, + ) !void { + try graph_query_mod.validateGraphMetricQueryShape(query); + const read_plan = try graph_query_mod.MetricReadPlan.init(query); + const dependency_names = read_plan.dependencies.buffer; + const dependency_count = read_plan.dependencies.len; + if (dependency_count == 0) return; + if (graphMetricPostProcessingNeeded(query) and result.nodes.len > graph_query_mod.graph_metric_candidate_limit) return error.QueryCandidateBudgetExceeded; + + const statuses = try self.alloc.alloc(db_types.GraphMetricStatus, dependency_count); + var status_initialized: [graph_query_mod.graph_metric_dependency_limit]bool = @splat(false); + var statuses_owned = true; + errdefer if (statuses_owned) { + for (statuses, status_initialized[0..dependency_count]) |*status, initialized| if (initialized) status.deinit(self.alloc); + self.alloc.free(statuses); + }; + const filter_names_buffer = read_plan.filters.buffer; + const filter_name_count = read_plan.filters.len; + const order_names_buffer = read_plan.orders.buffer; + const order_name_count = read_plan.orders.len; + const projection_names_buffer = read_plan.projections.buffer; + const projection_name_count = read_plan.projections.len; + + const initial_row_bytes = std.math.mul(usize, result.nodes.len, @sizeOf(usize)) catch + return error.GraphMetricQueryBudgetExceeded; + var active_row_memory = try session.reserveGraphMetricMemory(initial_row_bytes); + defer active_row_memory.deinit(); + const initial_rows = try self.alloc.alloc(usize, result.nodes.len); + for (initial_rows, 0..) |*row, i| row.* = i; + var active_rows: []usize = initial_rows; + defer if (active_rows.len > 0) self.alloc.free(active_rows); + + var cached_columns: [graph_query_mod.graph_metric_dependency_limit]?CachedPublicGraphMetricColumn = @splat(null); + defer for (cached_columns[0..dependency_count]) |*maybe_column| if (maybe_column.*) |*column| { + if (column.scores.len > 0) self.alloc.free(column.scores); + column.memory.deinit(); + }; + var stage_column_buffer: [graph_query_mod.graph_metric_dependency_limit][]?f64 = undefined; + + // Resolve selective filters first so later I/O only covers survivors. + // Stable original-row ordinals compose across stages; resident columns + // are rebased transactionally and nodes move once after selection. + if (filter_name_count > 0) { + try self.ensurePublicGraphMetricColumns( + session, + query.index_name, + filter_names_buffer[0..filter_name_count], + result.nodes, + active_rows, + dependency_names[0..dependency_count], + statuses, + status_initialized[0..dependency_count], + cached_columns[0..dependency_count], + ); + const filter_columns = stage_column_buffer[0..filter_name_count]; + try publicGraphMetricStageColumns( + dependency_names[0..dependency_count], + filter_names_buffer[0..filter_name_count], + cached_columns[0..dependency_count], + active_rows, + filter_columns, + ); + var filter_query = query; + filter_query.metrics = &.{}; + filter_query.order_by = &.{}; + // The selector retains its candidate permutation while copying + // the selected prefix; reserve both arrays before entering it. + var selection_memory = try session.reserveGraphMetricMemory(std.math.mul(usize, active_rows.len, 2 * @sizeOf(usize)) catch + return error.GraphMetricQueryBudgetExceeded); + defer selection_memory.deinit(); + const selected = try graph_query_mod.GraphQueryEngine.selectLoadedMetricCandidateIndexesAlloc( + self.alloc, + filter_names_buffer[0..filter_name_count], + filter_columns, + filter_query, + order_name_count == 0, + active_rows.len, + ); + defer self.alloc.free(selected); + self.releaseUnusedPublicGraphMetricColumns( + dependency_names[0..dependency_count], + cached_columns[0..dependency_count], + order_names_buffer[0..order_name_count], + projection_names_buffer[0..projection_name_count], + ); + const filtered_rows = blk: { + var memory = try session.reserveGraphMetricMemory(std.math.mul(usize, selected.len, @sizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded); + errdefer memory.deinit(); + const rows = try self.alloc.alloc(usize, selected.len); + errdefer self.alloc.free(rows); + for (selected, rows) |source_index, *row| { + if (source_index >= active_rows.len) return error.InvalidQueryRequest; + row.* = active_rows[source_index]; + } + try self.rebasePublicGraphMetricColumns( + session, + cached_columns[0..dependency_count], + active_rows, + rows, + selected, + ); + break :blk .{ .rows = rows, .memory = memory }; + }; + if (active_rows.len > 0) self.alloc.free(active_rows); + active_row_memory.deinit(); + active_rows = filtered_rows.rows; + active_row_memory = filtered_rows.memory; + } + + if (order_name_count > 0) { + try self.ensurePublicGraphMetricColumns( + session, + query.index_name, + order_names_buffer[0..order_name_count], + result.nodes, + active_rows, + dependency_names[0..dependency_count], + statuses, + status_initialized[0..dependency_count], + cached_columns[0..dependency_count], + ); + const order_columns = stage_column_buffer[0..order_name_count]; + try publicGraphMetricStageColumns( + dependency_names[0..dependency_count], + order_names_buffer[0..order_name_count], + cached_columns[0..dependency_count], + active_rows, + order_columns, + ); + var order_query = query; + order_query.metrics = &.{}; + order_query.where_metric = &.{}; + var selection_memory = try session.reserveGraphMetricMemory(std.math.mul(usize, active_rows.len, 2 * @sizeOf(usize)) catch + return error.GraphMetricQueryBudgetExceeded); + defer selection_memory.deinit(); + const selected = try graph_query_mod.GraphQueryEngine.selectLoadedMetricCandidateIndexesAlloc( + self.alloc, + order_names_buffer[0..order_name_count], + order_columns, + order_query, + true, + active_rows.len, + ); + defer self.alloc.free(selected); + self.releaseUnusedPublicGraphMetricColumns( + dependency_names[0..dependency_count], + cached_columns[0..dependency_count], + projection_names_buffer[0..projection_name_count], + &.{}, + ); + const ordered_rows = blk: { + var memory = try session.reserveGraphMetricMemory(std.math.mul(usize, selected.len, @sizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded); + errdefer memory.deinit(); + const rows = try self.alloc.alloc(usize, selected.len); + errdefer self.alloc.free(rows); + for (selected, rows) |source_index, *row| { + if (source_index >= active_rows.len) return error.InvalidQueryRequest; + row.* = active_rows[source_index]; + } + try self.rebasePublicGraphMetricColumns( + session, + cached_columns[0..dependency_count], + active_rows, + rows, + selected, + ); + break :blk .{ .rows = rows, .memory = memory }; + }; + if (active_rows.len > 0) self.alloc.free(active_rows); + active_row_memory.deinit(); + active_rows = ordered_rows.rows; + active_row_memory = ordered_rows.memory; + } + + try self.ensurePublicGraphMetricColumns( + session, + query.index_name, + projection_names_buffer[0..projection_name_count], + result.nodes, + active_rows, + dependency_names[0..dependency_count], + statuses, + status_initialized[0..dependency_count], + cached_columns[0..dependency_count], + ); + for (status_initialized[0..dependency_count]) |initialized| if (!initialized) + return error.InvalidQueryRequest; + + const projected_value_count = std.math.mul(usize, active_rows.len, query.metrics.len) catch + return error.GraphMetricQueryBudgetExceeded; + var projected_retained_bytes = std.math.mul(usize, projected_value_count, @sizeOf(graph_query_mod.GraphMetricValue)) catch + return error.GraphMetricQueryBudgetExceeded; + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, active_rows.len, @sizeOf(graph_query_mod.GraphResultNode)) catch + return error.GraphMetricQueryBudgetExceeded) catch return error.GraphMetricQueryBudgetExceeded; + const selection_words = std.math.divCeil(usize, result.nodes.len, @bitSizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded; + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, selection_words, @sizeOf(usize)) catch + return error.GraphMetricQueryBudgetExceeded) catch return error.GraphMetricQueryBudgetExceeded; + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, dependency_count, @sizeOf([]u8)) catch + return error.GraphMetricQueryBudgetExceeded) catch return error.GraphMetricQueryBudgetExceeded; + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, dependency_count, @sizeOf(db_types.GraphMetricStatus)) catch + return error.GraphMetricQueryBudgetExceeded) catch return error.GraphMetricQueryBudgetExceeded; + for (statuses[0..dependency_count]) |status| { + // The status and the projection dictionary intentionally own + // independent copies: status may be omitted from the response + // while projected metric names must remain valid. + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, status.name.len, 2) catch + return error.GraphMetricQueryBudgetExceeded) catch + return error.GraphMetricQueryBudgetExceeded; + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, std.math.mul(usize, status.edge_filter.types.len, @sizeOf([]const u8)) catch + return error.GraphMetricQueryBudgetExceeded) catch return error.GraphMetricQueryBudgetExceeded; + for (status.edge_filter.types) |edge_type| { + projected_retained_bytes = std.math.add(usize, projected_retained_bytes, edge_type.len) catch + return error.GraphMetricQueryBudgetExceeded; + } + } + try session.chargeGraphMetricRetained(projected_retained_bytes); + + const metric_value_names = try self.alloc.alloc([]u8, dependency_count); + var initialized_metric_names: usize = 0; + var metric_names_owned = true; + errdefer if (metric_names_owned) { + for (metric_value_names[0..initialized_metric_names]) |name| self.alloc.free(name); + self.alloc.free(metric_value_names); + }; + for (statuses[0..dependency_count], 0..) |status, i| { + metric_value_names[i] = try self.alloc.dupe(u8, status.name); + initialized_metric_names += 1; + } + // Keep the old slab alive until the single final node move has + // installed every replacement view. Filtering and ordering above only + // changed stable ordinals, so no intermediate node or metric copies + // are required. + const projection_columns = stage_column_buffer[0..projection_name_count]; + try publicGraphMetricStageColumns( + dependency_names[0..dependency_count], + projection_names_buffer[0..projection_name_count], + cached_columns[0..dependency_count], + active_rows, + projection_columns, + ); + const projection_metric_value_names = try self.alloc.alloc([]const u8, projection_name_count); + defer self.alloc.free(projection_metric_value_names); + for (projection_names_buffer[0..projection_name_count], 0..) |name, i| { + const dependency_index = metricDependencyIndex(dependency_names[0..dependency_count], name) orelse + return error.InvalidQueryRequest; + projection_metric_value_names[i] = metric_value_names[dependency_index]; + } + const previous_metric_values_slab = result.metric_values_slab; + const replacement_metric_values_slab = try graph_query_mod.GraphQueryEngine.materializeSelectedMetricColumns( + self.alloc, + projection_metric_value_names, + projection_columns, + active_rows, + &result.nodes, + ); + result.metric_values_slab = replacement_metric_values_slab; + if (previous_metric_values_slab.len > 0) self.alloc.free(previous_metric_values_slab); + for (result.metric_value_names) |name| self.alloc.free(name); + if (result.metric_value_names.len > 0) self.alloc.free(result.metric_value_names); + result.metric_value_names = metric_value_names; + metric_names_owned = false; + for (result.metric_status) |*status| status.deinit(self.alloc); + if (result.metric_status.len > 0) self.alloc.free(result.metric_status); + result.metric_status = statuses; + statuses_owned = false; + if (!query.include_metric_status) { + try stripServerlessGraphMetricStatus(self.alloc, result); + } + if (result.paths.len > 0) try self.rebuildPublicGraphPathsFromNodes(result); + try self.rebuildPublicGraphHitsFromNodes(session.cancellation, result); + } + + fn rebuildPublicGraphPathsFromNodes(self: *HttpHandler, result: *db_types.GraphSearchResult) !void { + const paths = try self.alloc.alloc(db_types.GraphPath, result.nodes.len); + var initialized: usize = 0; + errdefer { + for (paths[0..initialized]) |path| graph_paths.freePath(self.alloc, path); + self.alloc.free(paths); + } + for (result.nodes, 0..) |node, i| { + paths[i] = try graphResultNodeToDbPathAlloc(self.alloc, node); + initialized += 1; + } + for (result.paths) |path| graph_paths.freePath(self.alloc, path); + self.alloc.free(result.paths); + result.paths = paths; + } + + const PublicGraphHitKey = struct { + source_table: ?[]const u8, + id: []const u8, + }; + + const PublicGraphHitKeyContext = struct { + pub fn hash(_: @This(), key: PublicGraphHitKey) u64 { + var hasher = std.hash.Wyhash.init(0x6772_6170_682d_6869); + if (key.source_table) |table| { + hasher.update(&.{1}); + hasher.update(table); + } else { + hasher.update(&.{0}); + } + hasher.update(&.{0}); + hasher.update(key.id); + return hasher.final(); + } + + pub fn eql(_: @This(), lhs: PublicGraphHitKey, rhs: PublicGraphHitKey) bool { + if (!std.mem.eql(u8, lhs.id, rhs.id)) return false; + if (lhs.source_table == null or rhs.source_table == null) return lhs.source_table == null and rhs.source_table == null; + return std.mem.eql(u8, lhs.source_table.?, rhs.source_table.?); + } + }; + + fn rebuildPublicGraphHitsFromNodes( + self: *HttpHandler, + cancellation: CancellationToken, + result: *db_types.GraphSearchResult, + ) !void { + var hit_indexes = std.HashMapUnmanaged(PublicGraphHitKey, usize, PublicGraphHitKeyContext, std.hash_map.default_max_load_percentage).empty; + defer hit_indexes.deinit(self.alloc); + try hit_indexes.ensureTotalCapacity(self.alloc, @intCast(result.hits.len)); + for (result.hits, 0..) |hit, i| { + if (i % 4096 == 0) try cancellation.check(); + const gop = hit_indexes.getOrPutAssumeCapacity(.{ .source_table = hit.source_table, .id = hit.id }); + if (!gop.found_existing) gop.value_ptr.* = i; + } + + const hits = try self.alloc.alloc(db_types.SearchHit, result.nodes.len); + errdefer if (hits.len > 0) self.alloc.free(hits); + var initialized: usize = 0; + errdefer for (hits[0..initialized]) |*hit| hit.deinit(self.alloc); + for (result.nodes, 0..) |node, i| { + if (i % 4096 == 0) try cancellation.check(); + if (hit_indexes.get(.{ .source_table = node.table, .id = node.key })) |existing_index| { + const existing = result.hits[existing_index]; + hits[i] = try existing.clone(self.alloc); + hits[i].score = clampGraphMetricScore(node.distance); + } else { + hits[i] = try newPublicGraphHitAlloc(self.alloc, node); + } + initialized += 1; + } + for (result.hits) |*hit| hit.deinit(self.alloc); + if (result.hits.len > 0) self.alloc.free(result.hits); + result.hits = hits; + // Pattern-query totals count matches, while traversal totals count + // result nodes. Metric filtering/sorting applies to nodes only. + if (result.matches.len == 0) result.total_hits = @intCast(result.nodes.len); + } + + fn newPublicGraphHitAlloc(alloc: Allocator, node: graph_query_mod.GraphResultNode) !db_types.SearchHit { + const id = try alloc.dupe(u8, node.key); + errdefer alloc.free(id); + const source_table = if (node.table) |table| try alloc.dupe(u8, table) else null; + return .{ .id = id, .source_table = source_table, .score = clampGraphMetricScore(node.distance) }; + } + + const PublicGraphMetricRerankResult = struct { + status: db_types.GraphMetricStatus, + hits: []db_types.SearchHit, + }; + + fn applyPublicGraphMetricRerank( + self: *HttpHandler, + session: *query_mod.QuerySession, + hits: []db_types.SearchHit, + rerank: db_types.GraphMetricRerank, + offset: u32, + limit: u32, + ) !PublicGraphMetricRerankResult { + const node_ids = try self.alloc.alloc([]const u8, hits.len); + defer self.alloc.free(node_ids); + for (hits, 0..) |hit, i| node_ids[i] = hit.id; + var metric = try query_mod.graphMetricScoresAlloc(self.alloc, session, rerank.index_name, rerank.metric_name, node_ids); + defer metric.deinit(self.alloc); + try session.checkCancellation(); + const selected = try graph_metric_rerank.selectPageAlloc( + self.alloc, + hits, + metric.scores, + .{ + .base_weight = rerank.base_weight, + .metric_weight = rerank.weight, + .missing_score = rerank.missing_score, + }, + offset, + limit, + ); + defer self.alloc.free(selected); + try session.checkCancellation(); + var status = try self.graphMetricStatusAlloc( + rerank.metric_name, + metric.config_fingerprint, + metric.edge_filter, + metric.converged, + metric.iterations_completed, + metric.delta, + metric.metadata_version, + metric.published_generation, + metric.edge_generation, + metric.computed_at_ms, + ); + errdefer status.deinit(self.alloc); + for (selected) |selection| { + try session.checkCancellation(); + const hit = &hits[selection.original_index]; + const index_name = try self.alloc.dupe(u8, rerank.index_name); + errdefer self.alloc.free(index_name); + const metric_name = try self.alloc.dupe(u8, rerank.metric_name); + if (hit.score_details) |*old| old.deinit(self.alloc); + hit.score_details = .{ + .index_name = index_name, + .metric_name = metric_name, + .base_score = selection.base_score, + .base_weight = rerank.base_weight, + .metric_score = selection.metric_score, + .metric_score_used = selection.metric_score_used, + .metric_weight = rerank.weight, + .missing_score_used = selection.metric_score == null, + .final_score = selection.final_score, + .published_generation = metric.published_generation, + }; + hit.score = selection.final_score; + } + const retained = try self.alloc.alloc(bool, hits.len); + defer self.alloc.free(retained); + @memset(retained, false); + const kept = try self.alloc.alloc(db_types.SearchHit, selected.len); + for (selected, 0..) |selection, i| { + retained[selection.original_index] = true; + kept[i] = hits[selection.original_index]; + hits[selection.original_index] = undefined; + } + for (hits, retained) |*hit, keep| if (!keep) hit.deinit(self.alloc); + if (hits.len > 0) self.alloc.free(hits); + return .{ .status = status, .hits = kept }; + } + + fn graphMetricStatusAlloc( + self: *HttpHandler, + metric_name: []const u8, + config_fingerprint: u64, + edge_filter: graph_mod.GraphMetricEdgeFilter, + converged: bool, + iterations_completed: u32, + delta: f64, + metadata_version: u16, + published_generation: u64, + edge_generation: u64, + computed_at_ms: u64, + ) !db_types.GraphMetricStatus { + const name = try self.alloc.dupe(u8, metric_name); + errdefer self.alloc.free(name); + const owned_filter = try edge_filter.cloneAlloc(self.alloc); + return .{ + .name = name, + .state = .fresh, + .phase = .complete, + .edge_filter = owned_filter, + .metadata_version = metadata_version, + .config_fingerprint = config_fingerprint, + .published_generation = published_generation, + .edge_generation = edge_generation, + .target_edge_generation = edge_generation, + .progress = 1, + .converged = converged, + .iterations_completed = iterations_completed, + .delta = delta, + .computed_at_ms = computed_at_ms, + }; + } + fn handleQuerySearch(self: *HttpHandler, namespace: []const u8, body: []const u8, cancellation: CancellationToken) !HttpResponse { const aggregations_json = parsePublicAggregationsJsonAlloc(self.alloc, body) catch |err| switch (err) { error.InvalidQueryRequest => return try textResponse(self.alloc, 400, "invalid query request"), @@ -2929,7 +3902,7 @@ pub const HttpHandler = struct { }; defer if (aggregations_json) |json| self.alloc.free(json); - var execution = self.executePublishedSearch(namespace, null, body, cancellation) catch |err| { + var execution = self.executePublishedSearch(namespace, null, body, cancellation, null) catch |err| { switch (err) { error.InvalidQueryRequest, error.EmbeddingIndexNotFound, @@ -3020,7 +3993,7 @@ pub const HttpHandler = struct { const namespace = self.catalog.resolveTableNamespaceAlloc(table_name) catch return try textResponse(self.alloc, 404, "not found"); defer self.alloc.free(namespace); - var execution = self.executePublishedSearch(namespace, table_name, body, cancellation) catch |err| switch (err) { + var execution = self.executePublishedSearch(namespace, table_name, body, cancellation, null) catch |err| switch (err) { error.InvalidQueryRequest, error.EmbeddingIndexNotFound, error.InvalidEmbeddingDimensions, @@ -3278,13 +4251,14 @@ pub const HttpHandler = struct { } return err; }; + initialized += 1; + try self.applyPublicGraphMetricShape(session, named_query.query, &results[idx]); try available_sets.append(self.alloc, .{ .name = results[idx].name, .hits = results[idx].hits, .total_hits = results[idx].total_hits, .graph_result = &results[idx], }); - initialized += 1; } // The shared request budget is stack-owned. Preserve its output // charges across all named operations, then detach release hooks only @@ -3802,6 +4776,15 @@ pub const HttpHandler = struct { if (matches.len > 0) self.alloc.free(matches); } + const nodes = if (graphMetricDependenciesNeeded(named_query.query)) + try collectUniquePublicPatternNodesAlloc(self.alloc, matches) + else + try self.alloc.alloc(graph_query_mod.GraphResultNode, 0); + errdefer { + for (nodes) |*node| node.deinit(self.alloc); + if (nodes.len > 0) self.alloc.free(nodes); + } + const hits = try self.buildPatternDocumentHitsAlloc(source_table, named_query.query, matches, request_cache); errdefer { for (hits) |*hit| hit.deinit(self.alloc); @@ -3810,7 +4793,7 @@ pub const HttpHandler = struct { return .{ .name = try self.alloc.dupe(u8, named_query.name), - .nodes = &.{}, + .nodes = nodes, .paths = &.{}, .matches = matches, .hits = hits, @@ -4717,9 +5700,17 @@ pub const HttpHandler = struct { } fn tableApi(self: *HttpHandler, cancellation: CancellationToken) public_table_http.TableApi { + return self.tableApiWithDiagnostics(cancellation, null); + } + + fn tableApiWithDiagnostics( + self: *HttpHandler, + cancellation: CancellationToken, + diagnostics: ?*api_operation.RequestDiagnostics, + ) public_table_http.TableApi { return .{ .ptr = self, - .request = .{ .cancellation = cancellation }, + .request = .{ .cancellation = cancellation, .diagnostics = diagnostics }, .vtable = &.{ .execute_table_batch = executePublicTableBatch, .execute_table_query_request = executePublicTableQueryRequest, @@ -4763,6 +5754,8 @@ pub const HttpHandler = struct { }; defer status.deinit(self.alloc); if (!status.publish_admitted) return error.Backpressured; + request.cancellation.check() catch return error.Canceled; + try self.preflightPublicTableBatchSyncLevel(req.sync_level, status); const namespace = self.catalog.resolveTableNamespaceAlloc(table_name) catch |err| switch (err) { error.NamespaceNotFound => return error.NotFound, @@ -4792,7 +5785,7 @@ pub const HttpHandler = struct { }; defer result.deinit(self.alloc); - self.enforcePublicTableBatchSyncLevel(table_name, req.sync_level, result.end_lsn, status, request.cancellation) catch |err| { + self.enforcePublicTableBatchSyncLevel(table_name, namespace, req.sync_level, result.end_lsn, request.cancellation) catch |err| { if (err == error.InternalFailure) { std.log.err("serverless public table batch sync wait failed table={s} sync_level={} end_lsn={} err={}", .{ table_name, @@ -4805,38 +5798,59 @@ pub const HttpHandler = struct { }; } + fn preflightPublicTableBatchSyncLevel( + self: *HttpHandler, + sync_level: db_types.SyncLevel, + status: catalog_types.BuildStatus, + ) public_table_http.TableApi.ExecuteBatchError!void { + const requires_background_materialization = switch (sync_level) { + .propose, .write => false, + .full_text => status.chunk_preview_enabled, + .enrichments, .full_index => status.enrichment_enabled or + status.chunk_preview_enabled or + status.chunk_embeddings_enabled or + status.rerank_terms_enabled, + }; + switch (sync_level) { + .propose, .write => {}, + .full_text, .enrichments, .full_index => if (requires_background_materialization) { + const runtime = self.runtime_metrics orelse return error.UnsupportedSyncLevel; + if (!runtime.supportsSynchronousMaterialization()) return error.UnsupportedSyncLevel; + }, + } + if (sync_level != .full_index) return; + if (status.graph_metrics_rejected != 0) return error.GraphMetricMaterializationRejected; + } + fn enforcePublicTableBatchSyncLevel( self: *HttpHandler, table_name: []const u8, + namespace: []const u8, sync_level: db_types.SyncLevel, end_lsn: u64, - status_before_write: catalog_types.BuildStatus, cancellation: CancellationToken, ) public_table_http.TableApi.ExecuteBatchError!void { - const requires_background_materialization = - status_before_write.enrichment_enabled or - status_before_write.chunk_preview_enabled or - status_before_write.chunk_embeddings_enabled or - status_before_write.rerank_terms_enabled; - switch (sync_level) { .propose, .write => return, - .full_text => {}, - .enrichments, .full_index => if (requires_background_materialization and self.runtime_metrics == null) { - return error.UnsupportedSyncLevel; - }, + .full_text, .enrichments, .full_index => {}, } - const timeout_ns = 30 * std.time.ns_per_s; const start_ns = platform_time.monotonicNs(); + const deadline_ns = start_ns +| timeout_ns; + const sync_cancellation_state = SyncWaitCancellation{ + .upstream = cancellation, + .deadline_ns = deadline_ns, + }; + const sync_cancellation = sync_cancellation_state.token(); while (true) { - cancellation.check() catch return error.Canceled; - const build_result = self.catalog.buildTable(table_name) catch |err| switch (err) { - error.NamespaceNotFound => return error.NotFound, + sync_cancellation.check() catch return error.CommittedPending; + const build_result = self.catalog.buildTableWithCancellation(table_name, sync_cancellation) catch |err| switch (err) { + error.Canceled => return error.CommittedPending, + error.NamespaceNotFound => return error.CommittedRepairRequired, error.HeadChanged => null, else => { std.log.err("serverless public table batch build failed table={s} sync_level={} err={}", .{ table_name, sync_level, err }); - return error.InternalFailure; + return error.CommittedRepairRequired; }, }; if (build_result) |build| { @@ -4844,26 +5858,32 @@ pub const HttpHandler = struct { owned_build.deinit(self.alloc); } - if (self.runtime_metrics) |runtime| { - _ = runtime.runOnce() catch |err| { - std.log.err("serverless public table batch maintenance run failed table={s} sync_level={} err={}", .{ table_name, sync_level, err }); - return error.InternalFailure; - }; - } - var status = self.catalog.tableBuildStatus(table_name) catch |err| switch (err) { - error.NamespaceNotFound => return error.NotFound, + error.NamespaceNotFound => return error.CommittedRepairRequired, else => { std.log.err("serverless public table batch post-build status failed table={s} sync_level={} err={}", .{ table_name, sync_level, err }); - return error.InternalFailure; + return error.CommittedRepairRequired; }, }; defer status.deinit(self.alloc); + if (sync_level == .full_index and status.graph_metrics_rejected != 0) return error.CommittedGraphMetricMaterializationRejected; if (tableSyncLevelSatisfied(sync_level, end_lsn, status)) return; - if (platform_time.monotonicNs() -| start_ns >= timeout_ns) { + + if (syncLevelNeedsBackgroundMaterialization(sync_level, status)) { + const runtime = self.runtime_metrics orelse return error.CommittedRepairRequired; + _ = runtime.runNamespaceMaterializationOnceWithCancellation(namespace, sync_cancellation) catch |err| switch (err) { + error.Canceled => return error.CommittedPending, + else => { + std.log.err("serverless public table batch targeted materialization failed table={s} namespace={s} sync_level={} err={}", .{ table_name, namespace, sync_level, err }); + return error.CommittedRepairRequired; + }, + }; + } + if (cancellation.isCancelled()) return error.CommittedPending; + if (platform_time.monotonicNs() >= deadline_ns) { std.log.err( - "serverless public table batch sync timeout table={s} sync_level={} end_lsn={} published={} latest={} pending_rebuild={} enrichment_complete={} chunk_preview_complete={} chunk_embeddings_complete={} rerank_terms_complete={} active_stage={any}", + "serverless public table batch sync timeout table={s} sync_level={} end_lsn={} published={} latest={} pending_rebuild={} enrichment_complete={} chunk_preview_complete={} chunk_embeddings_complete={} rerank_terms_complete={} graph_metrics_configured={} graph_metrics_pending={} graph_metrics_rejected={} active_stage={any}", .{ table_name, sync_level, @@ -4875,10 +5895,13 @@ pub const HttpHandler = struct { status.chunk_preview_complete, status.chunk_embeddings_complete, status.rerank_terms_complete, + status.graph_metrics_configured, + status.graph_metrics_pending, + status.graph_metrics_rejected, status.enrichment_active_stage, }, ); - return error.UnsupportedSyncLevel; + return error.CommittedPending; } sleepNs(10 * std.time.ns_per_ms); @@ -4900,6 +5923,18 @@ pub const HttpHandler = struct { }; } + fn syncLevelNeedsBackgroundMaterialization(sync_level: db_types.SyncLevel, status: catalog_types.BuildStatus) bool { + return switch (sync_level) { + .propose, .write => false, + .full_text => status.chunk_preview_enabled and !status.chunk_preview_complete, + .enrichments => !status.enrichment_complete, + .full_index => (status.enrichment_enabled and !status.enrichment_complete) or + (status.chunk_preview_enabled and !status.chunk_preview_complete) or + (status.chunk_embeddings_enabled and !status.chunk_embeddings_complete) or + (status.rerank_terms_enabled and !status.rerank_terms_complete), + }; + } + fn fullTextSyncSatisfied(status: catalog_types.BuildStatus) bool { if (status.artifact_actions.document_segment == .rebuild) return false; if (status.full_text_index_actions.len > 0) { @@ -4917,6 +5952,8 @@ pub const HttpHandler = struct { } fn fullIndexSyncSatisfied(status: catalog_types.BuildStatus) bool { + if (status.graph_metrics_pending != 0 or status.graph_metrics_rejected != 0) return false; + if (status.head_republish_recommended or status.pending_materialization_rebuild) return false; if (!status.enrichment_complete) return false; if (!fullTextSyncSatisfied(status)) return false; if (status.artifact_actions.dense_vector == .rebuild) return false; @@ -4964,6 +6001,7 @@ pub const HttpHandler = struct { error.DocIdentityUnavailable => return error.DocIdentityUnavailable, error.UnsupportedExactSort => return error.UnsupportedExactSort, error.QueryCandidateBudgetExceeded => return error.QueryCandidateBudgetExceeded, + error.GraphMetricQueryBudgetExceeded => return error.GraphMetricQueryBudgetExceeded, error.GraphTraversalQueryBudgetExceeded => return error.QueryCandidateBudgetExceeded, error.GraphWorkBudgetExceeded => return error.GraphWorkBudgetExceeded, error.GraphMinWeightDomainViolation => return error.GraphMinWeightDomainViolation, @@ -5034,7 +6072,7 @@ pub const HttpHandler = struct { ptr: *anyopaque, alloc: Allocator, table_name: []const u8, - _: api_operation.RequestContext, + request: api_operation.RequestContext, ) public_table_http.TableApi.ExecuteListIndexesError![]u8 { const self: *HttpHandler = @ptrCast(@alignCast(ptr)); var table = (self.catalog.getTableAlloc(self.alloc, table_name) catch |err| { @@ -5047,7 +6085,12 @@ pub const HttpHandler = struct { return error.InternalFailure; }; defer status.deinit(self.alloc); - return encodeServerlessIndexListAlloc(alloc, table.indexes_json, status) catch |err| { + const metric_statuses = self.graphMetricIndexStatusesAlloc(table_name, table.indexes_json, null, request.cancellation) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return error.InternalFailure, + }; + defer freeServerlessGraphMetricStatuses(self.alloc, metric_statuses); + return encodeServerlessIndexListWithGraphMetricsAlloc(alloc, table.indexes_json, status, metric_statuses) catch |err| { std.log.err("serverless public table index list encode failed table={s} err={}", .{ table_name, err }); return error.InternalFailure; }; @@ -5058,7 +6101,7 @@ pub const HttpHandler = struct { alloc: Allocator, table_name: []const u8, index_name: []const u8, - _: api_operation.RequestContext, + request: api_operation.RequestContext, ) public_table_http.TableApi.ExecuteGetIndexError![]u8 { const self: *HttpHandler = @ptrCast(@alignCast(ptr)); var table = (self.catalog.getTableAlloc(self.alloc, table_name) catch |err| { @@ -5071,7 +6114,12 @@ pub const HttpHandler = struct { return error.InternalFailure; }; defer status.deinit(self.alloc); - return (encodeServerlessSingleIndexAlloc(alloc, table.indexes_json, index_name, status) catch |err| { + const metric_statuses = self.graphMetricIndexStatusesAlloc(table_name, table.indexes_json, index_name, request.cancellation) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return error.InternalFailure, + }; + defer freeServerlessGraphMetricStatuses(self.alloc, metric_statuses); + return (encodeServerlessSingleIndexWithGraphMetricsAlloc(alloc, table.indexes_json, index_name, status, metric_statuses) catch |err| { std.log.err("serverless public table index encode failed table={s} index={s} err={}", .{ table_name, index_name, err }); return error.InternalFailure; }) orelse error.NotFound; @@ -5134,8 +6182,9 @@ pub const HttpHandler = struct { const response_body = indexes_api.encodeCreatedIndexConfig(alloc, index_name, normalized_index_json) catch return error.InternalFailure; errdefer alloc.free(response_body); validateServerlessIndexCatalog(alloc, next_indexes_json) catch |err| switch (err) { + error.UnsupportedCreateTableRequest, error.UnsupportedGraphMetricRefreshMode, error.InvalidTableIndexMetadata => return error.InvalidIndexRequest, + error.GraphMetricConfigurationLimitExceeded => return error.GraphMetricConfigurationLimitExceeded, error.UnsupportedServerlessArtifactIndexSources => return error.UnsupportedArtifactIndexSources, - error.UnsupportedCreateTableRequest, error.InvalidTableIndexMetadata => return error.InvalidIndexRequest, else => return error.InternalFailure, }; request.ensureActive() catch |err| switch (err) { @@ -5272,10 +6321,22 @@ const ServerlessGraphAdmissionContext = struct { filter: graph_pattern_mod.NodeFilter, }; +const AdmittedAdjacencyReader = struct { + allocation: graph_work_budget_mod.RetainedAllocator, + reader: graph_segment_mod.AdjacencyReader, + + fn translate(self: *@This(), err: anyerror) anyerror { + if (err == error.OutOfMemory and self.allocation.denied) return error.QueryCandidateBudgetExceeded; + if (err == error.GraphMetricBuildBudgetExceeded) return error.GraphTraversalQueryBudgetExceeded; + return err; + } +}; + const CachedPublicGraphSegment = struct { index_name: []u8, - segment: graph_segment_mod.Segment, - adjacency_index: graph_segment_mod.AdjacencyIndex, + segment: graph_segment_mod.Segment = .{ .adjacencies = &.{} }, + adjacency_index: graph_segment_mod.AdjacencyIndex = .{}, + paged: ?*AdmittedAdjacencyReader = null, /// Canonical edge metadata aligned with segment.neighbor_tables. Building /// it once avoids serializing the same table qualifier for every edge scan /// and clone in a request. @@ -5387,6 +6448,7 @@ const PublicGraphRequestCache = struct { published_body_blocks: std.ArrayListUnmanaged([]u8) = .empty, segments: std.ArrayListUnmanaged(CachedPublicGraphSegment) = .empty, filter_cache: db_query_graph.PreparedPatternFilterCache, + graph_read_remaining: u64 = 512 * 1024 * 1024, fn init( handler: *HttpHandler, @@ -5409,6 +6471,11 @@ const PublicGraphRequestCache = struct { for (self.published_body_blocks.items) |block| self.handler.alloc.free(block); self.published_body_blocks.deinit(self.handler.alloc); for (self.segments.items) |*entry| { + if (entry.paged) |paged| { + paged.reader.deinit(); + std.debug.assert(paged.allocation.live_bytes == 0); + self.handler.alloc.destroy(paged); + } self.handler.alloc.free(entry.index_name); entry.adjacency_index.deinit(self.handler.alloc); for (entry.neighbor_table_metadata) |metadata| self.handler.alloc.free(metadata); @@ -5910,6 +6977,38 @@ const PublicGraphRequestCache = struct { return .{ .items = refs, .retained_bytes = retained_bytes }; } + fn pagedGraphSegment(self: *PublicGraphRequestCache, index_name: []const u8, artifact_ref: manifest_mod.ArtifactRef) !?*const CachedPublicGraphSegment { + const prior = self.retained_lease.bytes; + try self.reserveRetained(@sizeOf(AdmittedAdjacencyReader) + index_name.len); + errdefer self.retained_lease.resize(prior) catch unreachable; + const paged = try self.handler.alloc.create(AdmittedAdjacencyReader); + errdefer self.handler.alloc.destroy(paged); + paged.allocation = .{ .backing = self.handler.alloc, .budget = self.work_budget }; + paged.reader = (graph_segment_mod.AdjacencyReader.init(paged.allocation.allocator(), self.session.artifacts, artifact_ref, self.session.cancellation, &self.graph_read_remaining) catch |err| return paged.translate(err)) orelse { + self.handler.alloc.destroy(paged); + try self.retained_lease.resize(prior); + return null; + }; + errdefer paged.reader.deinit(); + try self.reserveRetained(try std.math.mul(usize, paged.reader.tables.len, @sizeOf([]u8))); + const metadata = try self.handler.alloc.alloc([]u8, paged.reader.tables.len); + errdefer self.handler.alloc.free(metadata); + var initialized: usize = 0; + errdefer for (metadata[0..initialized]) |value| self.handler.alloc.free(value); + for (paged.reader.tables, metadata) |table, *value| { + const reserved = try std.math.add(usize, try std.math.mul(usize, table.len, 6), "{\"target_table\":\"\"}".len); + try self.reserveRetained(reserved); + value.* = try std.json.Stringify.valueAlloc(self.handler.alloc, .{ .target_table = table }, .{}); + initialized += 1; + try self.retained_lease.resize(self.retained_lease.bytes - (reserved - value.len)); + } + const name = try self.handler.alloc.dupe(u8, index_name); + errdefer self.handler.alloc.free(name); + try self.ensureRetainedListCapacity(CachedPublicGraphSegment, &self.segments, self.segments.items.len + 1); + self.segments.appendAssumeCapacity(.{ .index_name = name, .paged = paged, .neighbor_table_metadata = metadata }); + return &self.segments.items[self.segments.items.len - 1]; + } + fn graphSegment(self: *PublicGraphRequestCache, index_name: []const u8) !*const CachedPublicGraphSegment { for (self.segments.items) |*entry| { if (std.mem.eql(u8, entry.index_name, index_name)) return entry; @@ -5917,22 +7016,26 @@ const PublicGraphRequestCache = struct { const graph_index = query_mod.graph_reader.findGraphArtifactIndex(self.session, index_name) orelse return error.GraphSegmentNotFound; const artifact_ref = self.session.artifactRef(graph_index) orelse return error.GraphSegmentNotFound; + if (try self.pagedGraphSegment(index_name, artifact_ref)) |entry| return entry; const payload_len = std.math.cast(usize, artifact_ref.byte_len) orelse return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); + if (payload_len > self.graph_read_remaining) return error.GraphTraversalQueryBudgetExceeded; + self.graph_read_remaining -= payload_len; var payload_lease = try graph_work_budget_mod.RetainedLease.init(self.work_budget, payload_len); defer payload_lease.deinit(); const payload = try self.session.fetchArtifactAlloc(graph_index); defer self.handler.alloc.free(payload); if (payload.len != payload_len) return error.InvalidGraphSegment; - var persistent_bytes = graph_segment_mod.decodedRetainedBytes(payload) catch |err| switch (err) { - error.UnsupportedGraphSegmentVersion => return err, + var view_lease = try graph_work_budget_mod.RetainedLease.init(self.work_budget, try graph_segment_mod.codec.compact.viewRetainedBytes(payload)); + defer view_lease.deinit(); + var view = graph_segment_mod.codec.compact.viewAlloc(self.handler.alloc, payload, .{}, self.session.cancellation) catch |err| switch (err) { + error.OutOfMemory, error.Canceled, error.UnsupportedGraphSegmentVersion => return err, else => return error.InvalidGraphSegment, }; - const adjacency_count = blk: { - if (payload.len < 14) return error.InvalidGraphSegment; - break :blk std.mem.readInt(u32, payload[10..14], .little); - }; + defer view.deinit(self.handler.alloc); + var persistent_bytes = try view.decodedBytes(); + const adjacency_count = view.adjacencies.len; const map_capacity = graph_work_budget_mod.hashMapCapacityForCount( adjacency_count, std.hash_map.default_max_load_percentage, @@ -5943,7 +7046,7 @@ const PublicGraphRequestCache = struct { graph_work_budget_mod.hashMapRetainedBytes([]const u8, usize, map_capacity) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes), ) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); - const table_count = std.mem.readInt(u32, payload[6..10], .little); + const table_count = view.tables.len; persistent_bytes = std.math.add( usize, persistent_bytes, @@ -5952,16 +7055,11 @@ const PublicGraphRequestCache = struct { ) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); // JSON string escaping expands one source byte to at most six bytes. // Reserve that hard upper bound before metadata serialization. - var table_pos: usize = 14; var reserved_metadata_payload_bytes: usize = 0; - for (0..table_count) |_| { - if (table_pos > payload.len or payload.len - table_pos < 4) return error.InvalidGraphSegment; - const table_len = std.mem.readInt(u32, payload[table_pos..][0..4], .little); - table_pos += 4; - if (table_len > payload.len - table_pos) return error.InvalidGraphSegment; + for (view.tables) |table| { const metadata_len = std.math.add( usize, - std.math.mul(usize, table_len, 6) catch + std.math.mul(usize, table.len, 6) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes), "{\"target_table\":\"\"}".len, ) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); @@ -5969,7 +7067,6 @@ const PublicGraphRequestCache = struct { return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); reserved_metadata_payload_bytes = std.math.add(usize, reserved_metadata_payload_bytes, metadata_len) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); - table_pos += table_len; } persistent_bytes = std.math.add(usize, persistent_bytes, index_name.len) catch return self.work_budget.exhaust(.retained_state_bytes, self.work_budget.max_retained_state_bytes); @@ -5977,7 +7074,7 @@ const PublicGraphRequestCache = struct { try self.reserveRetained(persistent_bytes); errdefer self.retained_lease.resize(prior_retained) catch unreachable; - var segment = try graph_segment_mod.decodeAlloc(self.handler.alloc, payload); + var segment = try graph_segment_mod.codec.compact.decodeViewAlloc(self.handler.alloc, view, self.session.cancellation); errdefer graph_segment_mod.freeSegment(self.handler.alloc, &segment); var adjacency_index = try graph_segment_mod.AdjacencyIndex.initWithCancellation( self.handler.alloc, @@ -6206,12 +7303,17 @@ const ServerlessPatternEdgeReader = struct { return error.GraphExternalAliasSourceUnsupported; var owned_bytes: usize = 0; for (probes, 0..) |probe, probe_index| { - const adjacency = self.cached.adjacency_index.find(self.cached.segment, probe.source) orelse continue; - const lookup = graph_segment_mod.findEdgeByTypeAndNeighbor( - adjacency.out_edges, - probe.edge_type, - probe.target, - ); + var paged_edge: ?graph_segment_mod.Edge = null; + defer if (paged_edge) |*edge| edge.deinit(self.cached.paged.?.reader.alloc); + const lookup: graph_segment_mod.EdgeLookup = if (self.cached.paged) |paged| blk: { + var remaining = public_graph_max_edges_scanned -| self.budget.edges_scanned; + const initial = remaining; + paged_edge = paged.reader.probe(probe.source, probe.edge_type, probe.target, &remaining) catch |err| return paged.translate(err); + break :blk .{ .edge = paged_edge, .inspected = initial - remaining }; + } else blk: { + const adjacency = self.cached.adjacency_index.find(self.cached.segment, probe.source) orelse continue; + break :blk graph_segment_mod.findEdgeByTypeAndNeighbor(adjacency.out_edges, probe.edge_type, probe.target); + }; try self.budget.admitEdges(lookup.inspected); if (lookup.edge) |edge| { const metadata = self.cached.edgeMetadata(edge); @@ -6286,13 +7388,20 @@ fn allocPublicSegmentEdgesBounded( // Serverless snapshots contain one table-local graph segment. Never alias a // cross-table identity into that local key space. if (table != null) return try alloc.alloc(graph_mod.Edge, 0); - const adjacency = cached.adjacency_index.find(cached.segment, key) orelse - return try alloc.alloc(graph_mod.Edge, 0); + var paged_adjacency: ?graph_segment_mod.Adjacency = null; + defer if (paged_adjacency) |*adjacency| adjacency.deinit(cached.paged.?.reader.alloc); + const adjacency = if (cached.paged) |paged| blk: { + var remaining = public_graph_max_edges_scanned -| budget.edges_scanned; + const initial = remaining; + paged_adjacency = paged.reader.adjacencyFiltered(key, edge_types, direction, max_edges, &remaining, include_qualified_targets, true) catch |err| return paged.translate(err); + try budget.admitEdges(initial - remaining); + break :blk paged_adjacency orelse return try alloc.alloc(graph_mod.Edge, 0); + } else cached.adjacency_index.find(cached.segment, key) orelse return try alloc.alloc(graph_mod.Edge, 0); // Charge physical adjacency work even when a mirrored self-loop is later // suppressed from the logical result. const scanned = (if (direction == .out or direction == .both) adjacency.out_edges.len else 0) + (if (direction == .in or direction == .both) adjacency.in_edges.len else 0); - try budget.admitEdges(scanned); + if (cached.paged == null) try budget.admitEdges(scanned); var edge_count: usize = 0; var owned_bytes: usize = 0; @@ -6709,13 +7818,264 @@ fn freeOwnedGraphEdge(alloc: Allocator, edge: graph_mod.Edge) void { } fn requestHasSearchInputs(request: metadata_openapi.QueryRequest) bool { - return request.full_text_search != null or + return request.query != null or + request.full_text_search != null or request.embeddings != null or request.semantic_search != null or request.filter_query != null or request.exclusion_query != null; } +fn requestWithoutGraphControls(request: metadata_openapi.QueryRequest) metadata_openapi.QueryRequest { + var search_request = request; + // Graph controls execute against the same pinned session after ordinary + // retrieval and must never leak into the ordinary search planner. + search_request.graph_queries = null; + search_request.graph_metric = null; + search_request.graph_metric_rerank = null; + return search_request; +} + +fn graphMetricPostProcessingNeeded(query: graph_query_mod.GraphQuery) bool { + return query.where_metric.len > 0 or query.order_by.len > 0; +} + +fn graphMetricLocalNodeId(node: graph_query_mod.GraphResultNode) ?[]const u8 { + if (node.table != null) return null; + return node.key; +} + +/// Takes ownership of `local_scores` on success and expands it into graph +/// result order. Qualified rows are deliberately left null. +fn scatterLocalGraphMetricScoresAlloc( + alloc: Allocator, + node_count: usize, + local_node_indexes: []const usize, + local_scores: []?f64, +) ![]?f64 { + if (local_node_indexes.len != local_scores.len or local_scores.len > node_count) + return error.InvalidQueryRequest; + if (local_scores.len == node_count) { + for (local_node_indexes, 0..) |result_index, expected_index| { + if (result_index != expected_index) return error.InvalidQueryRequest; + } + return local_scores; + } + const scores = try alloc.alloc(?f64, node_count); + errdefer alloc.free(scores); + @memset(scores, null); + var previous_index: ?usize = null; + for (local_node_indexes, local_scores) |result_index, score| { + if (result_index >= node_count or (previous_index != null and result_index <= previous_index.?)) + return error.InvalidQueryRequest; + scores[result_index] = score; + previous_index = result_index; + } + alloc.free(local_scores); + return scores; +} + +test "serverless graph metric column rebasing releases retired memory and rolls back failures" { + const alloc = std.testing.allocator; + var handler = HttpHandler{ .alloc = alloc, .api = undefined, .catalog = undefined, .manifests = undefined, .progress = undefined, .query = undefined, .runtime_status = undefined }; + var session = query_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + const bytes = 4 * @sizeOf(?f64); + session.graph_metric_read_budget.limits.max_retained_bytes = 2 * bytes; + const scores = try alloc.dupe(?f64, &.{ 0, 1, 2, 3 }); + var columns = [_]?HttpHandler.CachedPublicGraphMetricColumn{.{ .scores = scores, .source_rows = &.{ 0, 1, 2, 3 }, .memory = try session.reserveGraphMetricMemory(bytes) }}; + defer { + alloc.free(columns[0].?.scores); + columns[0].?.memory.deinit(); + } + for (0..100) |_| { + try handler.rebasePublicGraphMetricColumns(&session, &columns, &.{ 0, 1, 2, 3 }, &.{ 0, 1, 2, 3 }, &.{ 0, 1, 2, 3 }); + try std.testing.expectEqual(@as(u64, bytes), session.graph_metric_read_budget.retained_bytes); + } + try std.testing.expectError(error.InvalidQueryRequest, handler.rebasePublicGraphMetricColumns(&session, &columns, &.{ 0, 1, 2, 3 }, &.{ 0, 1, 2, 3 }, &.{ 0, 1, 2, 4 })); + try std.testing.expectEqual(@as(u64, bytes), session.graph_metric_read_budget.retained_bytes); + try std.testing.expectEqualSlices(?f64, &.{ 0, 1, 2, 3 }, columns[0].?.scores); +} + +test "serverless graph metric lookup preserves qualified node identity" { + const local: graph_query_mod.GraphResultNode = .{ .key = "shared", .depth = 0, .distance = 0 }; + const qualified: graph_query_mod.GraphResultNode = .{ .key = "shared", .table = "entities", .depth = 0, .distance = 0 }; + try std.testing.expectEqualStrings("shared", graphMetricLocalNodeId(local).?); + try std.testing.expect(graphMetricLocalNodeId(qualified) == null); + + const local_scores = try std.testing.allocator.alloc(?f64, 1); + local_scores[0] = 42; + const scattered = try scatterLocalGraphMetricScoresAlloc( + std.testing.allocator, + 2, + &.{0}, + local_scores, + ); + defer std.testing.allocator.free(scattered); + try std.testing.expectEqual(@as(?f64, 42), scattered[0]); + try std.testing.expectEqual(@as(?f64, null), scattered[1]); +} + +test "serverless graph metric qualified scatter is allocation-failure safe" { + const Runner = struct { + fn run(alloc: Allocator) !void { + const local_scores = try alloc.alloc(?f64, 1); + local_scores[0] = 7; + const scattered = scatterLocalGraphMetricScoresAlloc(alloc, 2, &.{0}, local_scores) catch |err| { + alloc.free(local_scores); + return err; + }; + defer alloc.free(scattered); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +fn graphMetricDependenciesNeeded(query: graph_query_mod.GraphQuery) bool { + return query.metrics.len > 0 or graphMetricPostProcessingNeeded(query); +} + +fn appendUniqueMetricDependency( + names: *[graph_query_mod.graph_metric_dependency_limit][]const u8, + count: *usize, + name: []const u8, +) void { + for (names[0..count.*]) |existing| if (std.mem.eql(u8, existing, name)) return; + std.debug.assert(count.* < names.len); + names[count.*] = name; + count.* += 1; +} + +fn clampGraphMetricScore(value: f64) f32 { + if (std.math.isNan(value)) return 0; + const max = std.math.floatMax(f32); + return @floatCast(std.math.clamp(value, -max, max)); +} + +fn stripServerlessGraphMetricStatus(alloc: Allocator, result: *db_types.GraphSearchResult) !void { + // Projected values intern names in the status list while scoring. Give the + // output-bounded projection independent ownership before hiding and + // releasing that internal lifecycle metadata. + for (result.nodes) |*node| { + for (node.metrics) |*metric| try metric.ensureNameOwned(alloc); + } + for (result.metric_status) |*status| status.deinit(alloc); + if (result.metric_status.len > 0) alloc.free(result.metric_status); + result.metric_status = &.{}; +} + +fn graphResultNodeToDbPathAlloc(alloc: Allocator, node: graph_query_mod.GraphResultNode) !db_types.GraphPath { + const source_nodes = node.path orelse return error.InvalidGraphQueryResult; + const source_edges = node.path_edges orelse return error.InvalidGraphQueryResult; + if (source_edges.len + 1 != source_nodes.len) return error.InvalidGraphQueryResult; + + const nodes = try alloc.alloc([]const u8, source_nodes.len); + var initialized_nodes: usize = 0; + errdefer { + for (nodes[0..initialized_nodes]) |item| alloc.free(item); + alloc.free(nodes); + } + for (source_nodes, 0..) |item, i| { + nodes[i] = try alloc.dupe(u8, item); + initialized_nodes += 1; + } + + var node_tables: []?[]const u8 = &.{}; + var initialized_tables: usize = 0; + errdefer { + for (node_tables[0..initialized_tables]) |table| if (table) |value| alloc.free(value); + if (node_tables.len > 0) alloc.free(node_tables); + } + if (node.path_tables) |source_tables| { + if (source_tables.len != source_nodes.len) return error.InvalidGraphQueryResult; + node_tables = try alloc.alloc(?[]const u8, source_tables.len); + @memset(node_tables, null); + for (source_tables, 0..) |table, i| { + if (table) |value| node_tables[i] = try alloc.dupe(u8, value); + initialized_tables += 1; + } + } + + const edges = try alloc.alloc(graph_paths.PathEdge, source_edges.len); + var initialized_edges: usize = 0; + errdefer { + for (edges[0..initialized_edges]) |edge| { + alloc.free(edge.source); + alloc.free(edge.target); + alloc.free(edge.edge_type); + if (edge.metadata.len > 0) alloc.free(edge.metadata); + } + alloc.free(edges); + } + for (source_edges, 0..) |edge, i| { + const source = try alloc.dupe(u8, edge.source); + errdefer alloc.free(source); + const target = try alloc.dupe(u8, edge.target); + errdefer alloc.free(target); + const edge_type = try alloc.dupe(u8, edge.edge_type); + errdefer alloc.free(edge_type); + const metadata = if (edge.metadata.len > 0) try alloc.dupe(u8, edge.metadata) else ""; + edges[i] = .{ + .source = source, + .target = target, + .edge_type = edge_type, + .weight = edge.weight, + .metadata = metadata, + .traversal_direction = edge.traversal_direction, + }; + initialized_edges += 1; + } + return .{ + .nodes = nodes, + .node_tables = node_tables, + .edges = edges, + .total_weight = node.distance, + .length = node.depth, + }; +} + +fn collectUniquePublicPatternNodesAlloc( + alloc: Allocator, + matches: []const db_types.GraphPatternMatch, +) ![]graph_query_mod.GraphResultNode { + var binding_count: usize = 0; + for (matches) |match| binding_count = std.math.add(usize, binding_count, match.bindings.len) catch + return error.QueryCandidateBudgetExceeded; + + var seen = graph_node_identity.BorrowedMap(void){}; + defer seen.deinit(alloc); + try seen.ensureTotalCapacity(alloc, binding_count); + var nodes = std.ArrayListUnmanaged(graph_query_mod.GraphResultNode).empty; + errdefer { + for (nodes.items) |*result_node| result_node.deinit(alloc); + nodes.deinit(alloc); + } + try nodes.ensureTotalCapacity(alloc, binding_count); + + for (matches) |match| { + for (match.bindings) |binding| { + const identity = graph_node_identity.Ref{ .table = binding.node.table, .key = binding.node.key }; + if (seen.contains(identity)) continue; + const key = try alloc.dupe(u8, binding.node.key); + const table = if (binding.node.table) |value| + alloc.dupe(u8, value) catch |err| { + alloc.free(key); + return err; + } + else + null; + nodes.appendAssumeCapacity(.{ + .key = key, + .table = table, + .depth = binding.node.depth, + .distance = binding.node.distance, + }); + const stored = nodes.items[nodes.items.len - 1]; + seen.putAssumeCapacityNoClobber(.{ .table = stored.table, .key = stored.key }, {}); + } + } + return try nodes.toOwnedSlice(alloc); +} + fn firstEdgeType(edge_types: ?[]const []const u8) ?[]const u8 { const values = edge_types orelse return null; if (values.len != 1) return null; @@ -7298,10 +8658,67 @@ const ServerlessIndexStatus = struct { chunked_source_count: usize = 0, }; +const GraphMetricMaterializationState = enum { + unsupported, + pending, + ready, + rejected, + stale, + unavailable, +}; + +const ServerlessGraphMetricStatus = struct { + index_name: []u8, + metric_name: []u8, + kind: graph_mod.GraphMetricKind, + state: GraphMetricMaterializationState = .pending, + rejection_reason: graph_metric_segment_mod.RejectionReason = .none, + config_fingerprint: u64, + materializer_fingerprint: u64 = 0, + published_generation: u64 = 0, + + fn deinit(self: *ServerlessGraphMetricStatus, alloc: Allocator) void { + alloc.free(self.index_name); + alloc.free(self.metric_name); + self.* = undefined; + } +}; + +fn freeServerlessGraphMetricStatuses(alloc: Allocator, statuses: []ServerlessGraphMetricStatus) void { + for (statuses) |*status| status.deinit(alloc); + if (statuses.len > 0) alloc.free(statuses); +} + +fn graphMetricStatusesForIndex( + statuses: []const ServerlessGraphMetricStatus, + index_name: []const u8, +) []const ServerlessGraphMetricStatus { + var start: ?usize = null; + var end: usize = 0; + for (statuses, 0..) |status, i| { + if (!std.mem.eql(u8, status.index_name, index_name)) { + if (start != null) break; + continue; + } + if (start == null) start = i; + end = i + 1; + } + return if (start) |value| statuses[value..end] else &.{}; +} + fn encodeServerlessIndexListAlloc( alloc: Allocator, indexes_json: []const u8, status: catalog_types.BuildStatus, +) ![]u8 { + return try encodeServerlessIndexListWithGraphMetricsAlloc(alloc, indexes_json, status, &.{}); +} + +fn encodeServerlessIndexListWithGraphMetricsAlloc( + alloc: Allocator, + indexes_json: []const u8, + status: catalog_types.BuildStatus, + graph_metric_statuses: []const ServerlessGraphMetricStatus, ) ![]u8 { const config_map_json = try indexes_api.encodeIndexConfigMap(alloc, indexes_json); defer alloc.free(config_map_json); @@ -7316,7 +8733,7 @@ fn encodeServerlessIndexListAlloc( while (it.next()) |entry| { if (!first) try out.append(alloc, ','); first = false; - try appendServerlessIndexEntry(alloc, &out, entry.key_ptr.*, entry.value_ptr.*, status); + try appendServerlessIndexEntry(alloc, &out, entry.key_ptr.*, entry.value_ptr.*, status, graph_metric_statuses); } try out.append(alloc, ']'); return try out.toOwnedSlice(alloc); @@ -7327,6 +8744,16 @@ fn encodeServerlessSingleIndexAlloc( indexes_json: []const u8, index_name: []const u8, status: catalog_types.BuildStatus, +) !?[]u8 { + return try encodeServerlessSingleIndexWithGraphMetricsAlloc(alloc, indexes_json, index_name, status, &.{}); +} + +fn encodeServerlessSingleIndexWithGraphMetricsAlloc( + alloc: Allocator, + indexes_json: []const u8, + index_name: []const u8, + status: catalog_types.BuildStatus, + graph_metric_statuses: []const ServerlessGraphMetricStatus, ) !?[]u8 { const config_json = try indexes_api.encodeSingleIndexConfig(alloc, indexes_json, index_name); defer if (config_json) |value| alloc.free(value); @@ -7337,7 +8764,7 @@ fn encodeServerlessSingleIndexAlloc( var out = std.ArrayListUnmanaged(u8).empty; defer out.deinit(alloc); - try appendServerlessIndexEntry(alloc, &out, index_name, config, status); + try appendServerlessIndexEntry(alloc, &out, index_name, config, status, graph_metric_statuses); return try out.toOwnedSlice(alloc); } @@ -7347,14 +8774,16 @@ fn appendServerlessIndexEntry( index_name: []const u8, config: std.json.Value, status: catalog_types.BuildStatus, + graph_metric_statuses: []const ServerlessGraphMetricStatus, ) !void { const config_json = try std.fmt.allocPrint(alloc, "{f}", .{std.json.fmt(config, .{})}); defer alloc.free(config_json); - const runtime = try serverlessIndexStatus(index_name, config, status); + const index_metric_statuses = graphMetricStatusesForIndex(graph_metric_statuses, index_name); + const runtime = try serverlessIndexStatus(index_name, config, status, index_metric_statuses); try out.appendSlice(alloc, "{\"config\":"); try out.appendSlice(alloc, config_json); try out.appendSlice(alloc, ",\"status\":"); - try appendServerlessIndexStatusJson(alloc, out, runtime); + try appendServerlessIndexStatusJson(alloc, out, runtime, index_metric_statuses); try out.appendSlice(alloc, ",\"shard_status\":{}}"); } @@ -7362,6 +8791,7 @@ fn appendServerlessIndexStatusJson( alloc: Allocator, out: *std.ArrayListUnmanaged(u8), status: ServerlessIndexStatus, + graph_metric_statuses: []const ServerlessGraphMetricStatus, ) !void { try out.appendSlice(alloc, "{\"readiness\":{\"state\":"); try out.appendSlice(alloc, if (status.readiness_ready) "\"ready\"" else "\"pending\""); @@ -7501,6 +8931,34 @@ fn appendServerlessIndexStatusJson( defer alloc.free(encoded_chunk_count); try out.appendSlice(alloc, encoded_chunk_count); } + if (graph_metric_statuses.len > 0) { + try out.appendSlice(alloc, ",\"graph_metrics\":["); + for (graph_metric_statuses, 0..) |metric, i| { + if (i > 0) try out.append(alloc, ','); + const encoded_name = try std.json.Stringify.valueAlloc(alloc, metric.metric_name, .{}); + defer alloc.free(encoded_name); + try out.print( + alloc, + "{{\"name\":{s},\"kind\":\"{s}\",\"state\":\"{s}\",\"config_fingerprint\":\"{x:0>16}\",\"materializer_fingerprint\":\"{x:0>16}\",\"published_generation\":{}", + .{ + encoded_name, + @tagName(metric.kind), + @tagName(metric.state), + metric.config_fingerprint, + metric.materializer_fingerprint, + metric.published_generation, + }, + ); + if (metric.rejection_reason != .none) { + try out.print(alloc, ",\"rejection_reason\":\"{s}\"", .{@tagName(metric.rejection_reason)}); + } + if (metric.state == .unsupported) { + try out.appendSlice(alloc, ",\"unavailable_reason\":\"graph_metric_publication_not_enabled\",\"retryable\":false"); + } + try out.append(alloc, '}'); + } + try out.append(alloc, ']'); + } try out.append(alloc, '}'); } @@ -7508,6 +8966,7 @@ fn serverlessIndexStatus( index_name: []const u8, config: std.json.Value, status: catalog_types.BuildStatus, + graph_metric_statuses: []const ServerlessGraphMetricStatus, ) !ServerlessIndexStatus { if (config != .object) return error.InvalidTableIndexMetadata; const kind = switch (indexes_api.inferIndexType(index_name, config) orelse return error.InvalidTableIndexMetadata) { @@ -7566,6 +9025,13 @@ fn serverlessIndexStatus( 0 else 0; + var graph_metric_blocked = false; + for (graph_metric_statuses) |metric| { + if (metric.state != .ready) { + graph_metric_blocked = true; + break; + } + } const materialization_blocker: ?[]const u8 = if (std.mem.eql(u8, kind, "full_text")) blk: { if (full_text_action) |action| { if (action.chunked_source_count > 0 and status.pending_materialization_families.chunk_preview) break :blk "chunk_preview"; @@ -7581,7 +9047,10 @@ fn serverlessIndexStatus( if (indexUsesChunkEmbeddings(config) and status.pending_materialization_families.chunk_embeddings) break :blk "chunk_embeddings"; if (status.pending_materialization_families.dense_vector) break :blk "dense_vector"; break :blk null; - } else null; + } else if (std.mem.eql(u8, kind, "graph") and graph_metric_blocked) + "graph_metric" + else + null; const is_vector_driver = std.mem.eql(u8, kind, "embeddings") and !try isSparseEmbeddingsIndex(config) and status.vector_compaction_driver_index_name != null and std.mem.eql(u8, status.vector_compaction_driver_index_name.?, index_name); const readiness_ready = config_published and built and materialization_blocker == null; return .{ @@ -7707,7 +9176,7 @@ test "serverless readiness serializes durable incarnation as an opaque token" { .backfill_active = false, .doc_count = 0, .total_indexed = 0, - }); + }, &.{}); try ant_json.testing.expectSubsetJsonText( alloc, "{\"readiness\":{\"state\":\"ready\",\"queryable\":true,\"complete\":true,\"pending_reasons\":[]}}", @@ -7728,7 +9197,7 @@ test "serverless pending readiness is explicitly non-queryable and incomplete" { .backfill_active = true, .doc_count = 0, .total_indexed = 0, - }); + }, &.{}); try ant_json.testing.expectSubsetJsonText( alloc, "{\"readiness\":{\"state\":\"pending\",\"queryable\":false,\"complete\":false,\"pending_reasons\":[\"publication\"]}}", @@ -7741,6 +9210,11 @@ fn validateServerlessIndexCatalog(alloc: Allocator, indexes_json: []const u8) !v error.OutOfMemory => return err, else => return error.InvalidTableIndexMetadata, }; + const graph_metric_specs = build_mod.graph_metric_config.parseIndexSpecsAlloc(alloc, indexes_json) catch |err| switch (err) { + error.OutOfMemory, error.UnsupportedGraphMetricRefreshMode, error.GraphMetricConfigurationLimitExceeded => return err, + else => return error.InvalidTableIndexMetadata, + }; + defer build_mod.graph_metric_config.freeIndexSpecs(alloc, graph_metric_specs); var parsed = try std.json.parseFromSlice(JsonValueMap, alloc, indexes_json, .{}); defer parsed.deinit(); @@ -7951,6 +9425,29 @@ fn freeDbSearchHits(alloc: Allocator, hits: []db_types.SearchHit) void { alloc.free(hits); } +fn pagePublicGraphMetricRerankedHits( + alloc: Allocator, + hits: []db_types.SearchHit, + offset: u32, + limit: u32, +) ![]db_types.SearchHit { + const start = @min(@as(usize, offset), hits.len); + const keep_len = @min(@as(usize, limit), hits.len - start); + if (start == 0 and keep_len == hits.len) return hits; + + const kept = try alloc.alloc(db_types.SearchHit, keep_len); + for (hits, 0..) |*hit, i| { + if (i >= start and i < start + keep_len) { + kept[i - start] = hit.*; + hit.* = undefined; + } else { + hit.deinit(alloc); + } + } + alloc.free(hits); + return kept; +} + fn allocQueryHitsAlloc( alloc: Allocator, hits: []const query_mod.QuerySearchHit, @@ -10488,6 +11985,63 @@ test "serverless index catalog validation rejects malformed configs" { try std.testing.expectError(error.InvalidTableIndexMetadata, validateServerlessIndexCatalog(alloc, \\{"relations":{"type":"graph","source":{"artifact":"relations_v1","path":"$.relations[0]"}}} )); + try std.testing.expectError(error.UnsupportedGraphMetricRefreshMode, validateServerlessIndexCatalog(alloc, + \\{"relations":{"type":"graph","metrics":{"rank":{"kind":"pagerank","refresh":"manual"}}}} + )); +} + +test "serverless graph metric node post-processing preserves hydrated hits and pattern totals" { + const alloc = std.testing.allocator; + var handler: HttpHandler = undefined; + handler.alloc = alloc; + + const nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + nodes[0] = .{ .key = try alloc.dupe(u8, "doc-a"), .depth = 0, .distance = 0.75, .path = null, .path_edges = null }; + const hits = try alloc.alloc(db_types.SearchHit, 1); + hits[0] = .{ .id = try alloc.dupe(u8, "doc-a"), .stored_data = try alloc.dupe(u8, "{\"title\":\"kept\"}") }; + const matches = try alloc.alloc(db_types.GraphPatternMatch, 1); + matches[0] = .{ .bindings = &.{}, .path = &.{} }; + var result = db_types.GraphSearchResult{ + .name = try alloc.dupe(u8, "pattern"), + .nodes = nodes, + .matches = matches, + .hits = hits, + .total_hits = 7, + }; + defer result.deinit(alloc); + + try handler.rebuildPublicGraphHitsFromNodes(.none, &result); + try std.testing.expectEqual(@as(usize, 1), result.hits.len); + try std.testing.expectEqualStrings("{\"title\":\"kept\"}", result.hits[0].stored_data.?); + try std.testing.expectEqual(@as(u32, 7), result.total_hits); +} + +test "serverless hidden graph metric status leaves projected names independently owned" { + const alloc = std.testing.allocator; + const statuses = try alloc.alloc(db_types.GraphMetricStatus, 1); + statuses[0] = .{ .name = try alloc.dupe(u8, "pagerank") }; + const values = try alloc.alloc(graph_query_mod.GraphMetricValue, 1); + values[0] = .{ .name = statuses[0].name, .score = 0.75, .name_owned = false }; + const nodes = try alloc.alloc(graph_query_mod.GraphResultNode, 1); + nodes[0] = .{ + .key = try alloc.dupe(u8, "doc-a"), + .depth = 0, + .distance = 0, + .metrics = values, + }; + var result = db_types.GraphSearchResult{ + .name = try alloc.dupe(u8, "neighbors"), + .nodes = nodes, + .hits = &.{}, + .total_hits = 1, + .metric_status = statuses, + }; + defer result.deinit(alloc); + + try stripServerlessGraphMetricStatus(alloc, &result); + try std.testing.expectEqual(@as(usize, 0), result.metric_status.len); + try std.testing.expect(result.nodes[0].metrics[0].name_owned); + try std.testing.expectEqualStrings("pagerank", result.nodes[0].metrics[0].name); } test "serverless index catalog rejects artifact-backed sources before publication" { @@ -11479,11 +13033,21 @@ test "http handler honors public serverless sync levels on table batch writes" { defer unsupported.deinit(alloc); try std.testing.expectEqual(@as(u16, 400), unsupported.status); try std.testing.expect(std.mem.indexOf(u8, unsupported.body, "unsupported sync_level") != null); + try std.testing.expectEqual(@as(u64, 0), try wal_store.latestLsn("enriched")); } test "serverless full_index sync waits for enrichment and index publication" { var status = readyServerlessBuildStatusForSyncTest(); try std.testing.expect(HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); + try std.testing.expect(!HttpHandler.syncLevelNeedsBackgroundMaterialization(.full_text, status)); + try std.testing.expect(!HttpHandler.syncLevelNeedsBackgroundMaterialization(.full_index, status)); + + status.chunk_preview_enabled = true; + status.chunk_preview_complete = false; + try std.testing.expect(HttpHandler.syncLevelNeedsBackgroundMaterialization(.full_text, status)); + try std.testing.expect(HttpHandler.syncLevelNeedsBackgroundMaterialization(.full_index, status)); + status.chunk_preview_enabled = false; + status.chunk_preview_complete = true; status.enrichment_complete = false; try std.testing.expect(!HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); @@ -11533,10 +13097,50 @@ test "serverless full_index sync waits for enrichment and index publication" { try std.testing.expect(!HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); status.artifact_actions.dense_vector = .reuse; + status.graph_metrics_configured = 1; + status.graph_metrics_pending = 1; + try std.testing.expect(!HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); + status.graph_metrics_pending = 0; + status.graph_metrics_rejected = 1; + try std.testing.expect(!HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); + status.graph_metrics_rejected = 0; + try std.testing.expect(HttpHandler.tableSyncLevelSatisfied(.full_index, 10, status)); + try std.testing.expect(!HttpHandler.tableSyncLevelSatisfied(.full_index, 11, status)); try std.testing.expect(HttpHandler.tableSyncLevelSatisfied(.enrichments, 10, status)); } +test "serverless sync wait cancellation combines request cancellation and deadline" { + var upstream_signal = std.atomic.Value(bool).init(false); + const expired = SyncWaitCancellation{ + .upstream = CancellationToken.fromAtomic(&upstream_signal), + .deadline_ns = 0, + }; + try std.testing.expect(expired.token().isCancelled()); + + const upstream_only = SyncWaitCancellation{ + .upstream = CancellationToken.fromAtomic(&upstream_signal), + .deadline_ns = std.math.maxInt(u64), + }; + try std.testing.expect(!upstream_only.token().isCancelled()); + upstream_signal.store(true, .release); + try std.testing.expect(upstream_only.token().isCancelled()); +} + +test "serverless full_index graph metric preflight fails before commit" { + var status = readyServerlessBuildStatusForSyncTest(); + status.graph_metrics_configured = 1; + var handler: HttpHandler = undefined; + try handler.preflightPublicTableBatchSyncLevel(.full_index, status); + try handler.preflightPublicTableBatchSyncLevel(.write, status); + + status.graph_metrics_rejected = 1; + try std.testing.expectError( + error.GraphMetricMaterializationRejected, + handler.preflightPublicTableBatchSyncLevel(.full_index, status), + ); +} + fn readyServerlessBuildStatusForSyncTest() catalog_types.BuildStatus { const ready_actions: catalog_types.ArtifactPublicationActions = .{ .document_segment = .reuse, @@ -11620,6 +13224,68 @@ fn readyServerlessBuildStatusForSyncTest() catalog_types.BuildStatus { }; } +test "serverless graph index status exposes rejected metric policy and blocker" { + const alloc = std.testing.allocator; + const status = readyServerlessBuildStatusForSyncTest(); + var metric_status = ServerlessGraphMetricStatus{ + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .kind = .pagerank, + .state = .rejected, + .rejection_reason = .build_budget_exceeded, + .config_fingerprint = 0x11, + .materializer_fingerprint = 0x22, + .published_generation = 7, + }; + defer metric_status.deinit(alloc); + const metric_statuses = [_]ServerlessGraphMetricStatus{metric_status}; + const encoded = (try encodeServerlessSingleIndexWithGraphMetricsAlloc( + alloc, + "{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"pagerank\":{\"kind\":\"pagerank\"}}}}", + "graph_idx", + status, + &metric_statuses, + )).?; + defer alloc.free(encoded); + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, encoded, .{}); + defer parsed.deinit(); + const runtime = parsed.value.object.get("status").?.object; + try std.testing.expect(runtime.get("materialization_blocked").?.bool); + try std.testing.expectEqualStrings("graph_metric", runtime.get("materialization_blocker").?.string); + const metric = runtime.get("graph_metrics").?.array.items[0].object; + try std.testing.expectEqualStrings("rejected", metric.get("state").?.string); + try std.testing.expectEqualStrings("build_budget_exceeded", metric.get("rejection_reason").?.string); + try std.testing.expectEqualStrings("0000000000000022", metric.get("materializer_fingerprint").?.string); +} + +test "serverless graph index status exposes disabled publication as terminal" { + const alloc = std.testing.allocator; + const status = readyServerlessBuildStatusForSyncTest(); + var metric_status = ServerlessGraphMetricStatus{ + .index_name = try alloc.dupe(u8, "graph_idx"), + .metric_name = try alloc.dupe(u8, "pagerank"), + .kind = .pagerank, + .state = .unsupported, + .config_fingerprint = 0x11, + }; + defer metric_status.deinit(alloc); + const metric_statuses = [_]ServerlessGraphMetricStatus{metric_status}; + const encoded = (try encodeServerlessSingleIndexWithGraphMetricsAlloc( + alloc, + "{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"pagerank\":{\"kind\":\"pagerank\"}}}}", + "graph_idx", + status, + &metric_statuses, + )).?; + defer alloc.free(encoded); + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, encoded, .{}); + defer parsed.deinit(); + const metric = parsed.value.object.get("status").?.object.get("graph_metrics").?.array.items[0].object; + try std.testing.expectEqualStrings("unsupported", metric.get("state").?.string); + try std.testing.expectEqualStrings("graph_metric_publication_not_enabled", metric.get("unavailable_reason").?.string); + try std.testing.expect(!metric.get("retryable").?.bool); +} + test "http handler serves published graph query endpoints" { const alloc = std.testing.allocator; @@ -12688,6 +14354,23 @@ test "serverless conjunctive anchors are enumerated in borrowed bounded pages" { ); } +test "serverless ordinary search planning strips every graph metric control" { + const alloc = std.testing.allocator; + var parsed = try std.json.parseFromSlice( + metadata_openapi.QueryRequest, + alloc, + \\{"full_text_search":{"query":"needle"},"graph_metric":{"index":"graph_idx","metric":"pagerank"},"graph_metric_rerank":{"index":"graph_idx","metric":"pagerank"},"graph_queries":{"walk":{"index":"graph_idx","traverse":{"start":{"keys":["doc:a"]}}}}} + , + .{ .allocate = .alloc_always }, + ); + defer parsed.deinit(); + const search_request = requestWithoutGraphControls(parsed.value); + try std.testing.expect(search_request.full_text_search != null); + try std.testing.expect(search_request.graph_metric == null); + try std.testing.expect(search_request.graph_metric_rerank == null); + try std.testing.expect(search_request.graph_queries == null); +} + test "serverless graph HTTP result copies are allocation-failure safe" { const alloc = std.testing.allocator; var node_path = [_][]u8{ @constCast("doc-a"), @constCast("doc-b") }; diff --git a/zig/pkg/antfly/src/serverless/artifacts/fs_store.zig b/zig/pkg/antfly/src/serverless/artifacts/fs_store.zig index 69fd20ebae..9ecd804c31 100644 --- a/zig/pkg/antfly/src/serverless/artifacts/fs_store.zig +++ b/zig/pkg/antfly/src/serverless/artifacts/fs_store.zig @@ -19,8 +19,35 @@ const artifact_store = @import("store.zig"); const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; pub const FsStore = struct { + const verified_file_cache_limit: usize = 4096; + + const VerifiedFile = struct { + inode: std.Io.File.INode, + byte_len: u64, + mtime_ns: i128, + ctime_ns: i128, + + fn fromStat(file_stat: std.Io.File.Stat) VerifiedFile { + return .{ + .inode = file_stat.inode, + .byte_len = file_stat.size, + .mtime_ns = file_stat.mtime.toNanoseconds(), + .ctime_ns = file_stat.ctime.toNanoseconds(), + }; + } + + fn matchesStat(self: VerifiedFile, file_stat: std.Io.File.Stat) bool { + return self.inode == file_stat.inode and + self.byte_len == file_stat.size and + self.mtime_ns == file_stat.mtime.toNanoseconds() and + self.ctime_ns == file_stat.ctime.toNanoseconds(); + } + }; + alloc: Allocator, root_dir: []u8, + verified_mu: std.atomic.Mutex = .unlocked, + verified_files: std.StringHashMapUnmanaged(VerifiedFile) = .empty, pub fn init(alloc: Allocator, root_dir: []const u8) !FsStore { var io_impl = threadedIo(); @@ -33,6 +60,11 @@ pub const FsStore = struct { } pub fn deinit(self: *FsStore) void { + lockAtomic(&self.verified_mu); + var it = self.verified_files.keyIterator(); + while (it.next()) |key| self.alloc.free(key.*); + self.verified_files.deinit(self.alloc); + self.verified_mu.unlock(); self.alloc.free(self.root_dir); self.* = undefined; } @@ -46,7 +78,12 @@ pub const FsStore = struct { } pub fn put(self: *FsStore, alloc: Allocator, contents: []const u8) !artifact_store.ArtifactMetadata { - const checksum = try sha256StringAlloc(alloc, contents); + return try self.putWithCancellation(alloc, contents, .none); + } + + pub fn putWithCancellation(self: *FsStore, alloc: Allocator, contents: []const u8, cancellation: CancellationToken) !artifact_store.ArtifactMetadata { + try cancellation.check(); + const checksum = try sha256StringWithCancellationAlloc(alloc, contents, cancellation); errdefer alloc.free(checksum); const artifact_id = try makeArtifactIdAlloc(alloc, checksum); errdefer alloc.free(artifact_id); @@ -54,9 +91,16 @@ pub const FsStore = struct { const path = try pathForArtifactAlloc(self.alloc, self.root_dir, checksum); defer self.alloc.free(path); - if (!fileExists(path)) { + const existing_valid = if (fileExists(path)) blk: { + verifyPathContent(path, @intCast(contents.len), checksum, cancellation) catch |err| switch (err) { + error.ArtifactIntegrityMismatch, error.FileNotFound => break :blk false, + else => return err, + }; + break :blk true; + } else false; + if (!existing_valid) { try ensureParentDir(path); - try writeFileAtomically(path, contents); + try writeFileAtomicallyWithCancellation(path, contents, cancellation); } return .{ @@ -102,6 +146,69 @@ pub const FsStore = struct { return try readFileRangeAllocWithCancellation(alloc, path, offset, len, cancellation); } + pub fn getVerifiedRangeAllocWithCancellation( + self: *FsStore, + alloc: Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + return self.getVerifiedRangeWithBudget(alloc, artifact_id, expected_byte_len, expected_checksum, offset, len, cancellation, null); + } + + fn getVerifiedRangeWithBudget( + self: *FsStore, + alloc: Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + remaining: ?*u64, + ) ![]u8 { + try cancellation.check(); + const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); + if (!std.mem.eql(u8, checksum, expected_checksum)) return error.ArtifactIntegrityMismatch; + const end = std.math.add(u64, offset, std.math.cast(u64, len) orelse return error.InvalidRange) catch return error.InvalidRange; + if (end > expected_byte_len) return error.InvalidRange; + const path = try pathForArtifactAlloc(self.alloc, self.root_dir, checksum); + defer self.alloc.free(path); + + var io_impl = threadedIo(); + defer io_impl.deinit(); + const io = io_impl.io(); + const file = if (std.fs.path.isAbsolute(path)) + try std.Io.Dir.openFileAbsolute(io, path, .{}) + else + try std.Io.Dir.cwd().openFile(io, path, .{}); + defer file.close(io); + const before = try file.stat(io); + if (before.size != expected_byte_len) return error.ArtifactIntegrityMismatch; + const verified = VerifiedFile.fromStat(before); + if (!self.isVerifiedFile(artifact_id, verified)) { + if (remaining) |budget| try artifact_store.chargeReadBudget(budget, expected_byte_len); + try verifyOpenFileContent(file, io, expected_byte_len, expected_checksum, cancellation); + const after = try file.stat(io); + if (!verified.matchesStat(after)) { + return error.ArtifactIntegrityMismatch; + } + try self.rememberVerifiedFile(artifact_id, verified); + } + const payload = try readOpenFileRangeAllocWithCancellation(alloc, file, io, offset, len, cancellation); + errdefer alloc.free(payload); + const after_read = try file.stat(io); + if (!verified.matchesStat(after_read)) { + self.forgetVerifiedFile(artifact_id); + return error.ArtifactIntegrityMismatch; + } + try cancellation.check(); + return payload; + } + pub fn stat(self: *FsStore, alloc: Allocator, artifact_id: []const u8) !artifact_store.ArtifactMetadata { return try self.statWithCancellation(alloc, artifact_id, .none); } @@ -128,22 +235,89 @@ pub const FsStore = struct { }; } + pub fn verifyContent( + self: *FsStore, + _: Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + cancellation: CancellationToken, + ) !void { + const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); + if (!std.mem.eql(u8, checksum, expected_checksum)) return error.ArtifactIntegrityMismatch; + const path = try pathForArtifactAlloc(self.alloc, self.root_dir, checksum); + defer self.alloc.free(path); + var io_impl = threadedIo(); + defer io_impl.deinit(); + const before = try std.Io.Dir.cwd().statFile(io_impl.io(), path, .{}); + if (before.size != expected_byte_len) return error.ArtifactIntegrityMismatch; + const verified = VerifiedFile.fromStat(before); + lockAtomic(&self.verified_mu); + if (self.verified_files.get(artifact_id)) |cached| { + if (std.meta.eql(cached, verified)) { + self.verified_mu.unlock(); + return; + } + } + self.verified_mu.unlock(); + + try verifyPathContent(path, expected_byte_len, expected_checksum, cancellation); + const after = try std.Io.Dir.cwd().statFile(io_impl.io(), path, .{}); + if (!verified.matchesStat(after)) return error.ArtifactIntegrityMismatch; + try self.rememberVerifiedFile(artifact_id, verified); + } + + fn isVerifiedFile(self: *FsStore, artifact_id: []const u8, verified: VerifiedFile) bool { + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + const cached = self.verified_files.get(artifact_id) orelse return false; + return std.meta.eql(cached, verified); + } + + fn rememberVerifiedFile(self: *FsStore, artifact_id: []const u8, verified: VerifiedFile) !void { + const owned_id = try self.alloc.dupe(u8, artifact_id); + errdefer self.alloc.free(owned_id); + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + if (!self.verified_files.contains(artifact_id) and self.verified_files.count() >= verified_file_cache_limit) { + var iterator = self.verified_files.keyIterator(); + if (iterator.next()) |victim| { + const removed = self.verified_files.fetchRemove(victim.*).?; + self.alloc.free(removed.key); + } + } + const gop = try self.verified_files.getOrPut(self.alloc, owned_id); + if (gop.found_existing) self.alloc.free(owned_id) else gop.key_ptr.* = owned_id; + gop.value_ptr.* = verified; + } + + fn forgetVerifiedFile(self: *FsStore, artifact_id: []const u8) void { + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + if (self.verified_files.fetchRemove(artifact_id)) |removed| self.alloc.free(removed.key); + } + pub fn delete(self: *FsStore, artifact_id: []const u8) !void { const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); const path = try pathForArtifactAlloc(self.alloc, self.root_dir, checksum); defer self.alloc.free(path); try deleteFile(path); + self.forgetVerifiedFile(artifact_id); } const vtable: artifact_store.ArtifactStore.VTable = .{ .deinit = erasedDeinit, .put = erasedPut, + .put_with_cancellation = erasedPutWithCancellation, .get_alloc = erasedGetAlloc, .get_alloc_with_cancellation = erasedGetAllocWithCancellation, .get_range_alloc = erasedGetRangeAlloc, .get_range_alloc_with_cancellation = erasedGetRangeAllocWithCancellation, + .get_verified_range_alloc_with_cancellation = erasedGetVerifiedRangeAllocWithCancellation, + .get_verified_range_alloc_with_budget = erasedGetVerifiedRangeWithBudget, .stat = erasedStat, .stat_with_cancellation = erasedStatWithCancellation, + .verify_content = erasedVerifyContent, .delete = erasedDelete, }; @@ -157,6 +331,11 @@ pub const FsStore = struct { return try self.put(alloc, contents); } + fn erasedPutWithCancellation(ptr: *anyopaque, alloc: Allocator, contents: []const u8, cancellation: CancellationToken) !artifact_store.ArtifactMetadata { + const self: *FsStore = @ptrCast(@alignCast(ptr)); + return try self.putWithCancellation(alloc, contents, cancellation); + } + fn erasedGetAlloc(ptr: *anyopaque, alloc: Allocator, artifact_id: []const u8) ![]u8 { const self: *FsStore = @ptrCast(@alignCast(ptr)); return try self.getAlloc(alloc, artifact_id); @@ -177,6 +356,11 @@ pub const FsStore = struct { return try self.getRangeAllocWithCancellation(alloc, artifact_id, offset, len, cancellation); } + fn erasedGetVerifiedRangeAllocWithCancellation(ptr: *anyopaque, alloc: Allocator, artifact_id: []const u8, expected_byte_len: u64, expected_checksum: []const u8, offset: u64, len: usize, cancellation: CancellationToken) ![]u8 { + const self: *FsStore = @ptrCast(@alignCast(ptr)); + return try self.getVerifiedRangeAllocWithCancellation(alloc, artifact_id, expected_byte_len, expected_checksum, offset, len, cancellation); + } + fn erasedStat(ptr: *anyopaque, alloc: Allocator, artifact_id: []const u8) !artifact_store.ArtifactMetadata { const self: *FsStore = @ptrCast(@alignCast(ptr)); return try self.stat(alloc, artifact_id); @@ -187,16 +371,63 @@ pub const FsStore = struct { return try self.statWithCancellation(alloc, artifact_id, cancellation); } + fn erasedVerifyContent(ptr: *anyopaque, alloc: Allocator, artifact_id: []const u8, expected_byte_len: u64, expected_checksum: []const u8, cancellation: CancellationToken) !void { + const self: *FsStore = @ptrCast(@alignCast(ptr)); + try self.verifyContent(alloc, artifact_id, expected_byte_len, expected_checksum, cancellation); + } + + fn erasedGetVerifiedRangeWithBudget(ptr: *anyopaque, alloc: Allocator, artifact_id: []const u8, byte_len: u64, checksum: []const u8, offset: u64, len: usize, cancellation: CancellationToken, remaining: *u64) ![]u8 { + const self: *FsStore = @ptrCast(@alignCast(ptr)); + return self.getVerifiedRangeWithBudget(alloc, artifact_id, byte_len, checksum, offset, len, cancellation, remaining); + } + fn erasedDelete(ptr: *anyopaque, artifact_id: []const u8) !void { const self: *FsStore = @ptrCast(@alignCast(ptr)); try self.delete(artifact_id); } }; +fn verifyPathContent(path: []const u8, expected_byte_len: u64, expected_checksum: []const u8, cancellation: CancellationToken) !void { + try cancellation.check(); + var io_impl = threadedIo(); + defer io_impl.deinit(); + var file = if (std.fs.path.isAbsolute(path)) + try std.Io.Dir.openFileAbsolute(io_impl.io(), path, .{}) + else + try std.Io.Dir.cwd().openFile(io_impl.io(), path, .{}); + defer file.close(io_impl.io()); + return try verifyOpenFileContent(file, io_impl.io(), expected_byte_len, expected_checksum, cancellation); +} + +fn verifyOpenFileContent(file: std.Io.File, io: std.Io, expected_byte_len: u64, expected_checksum: []const u8, cancellation: CancellationToken) !void { + var reader = file.reader(io, &.{}); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + var buffer: [1024 * 1024]u8 = undefined; + var total: u64 = 0; + while (true) { + try cancellation.check(); + const read = try reader.interface.readSliceShort(&buffer); + if (read == 0) break; + total = std.math.add(u64, total, read) catch return error.ArtifactIntegrityMismatch; + if (total > expected_byte_len) return error.ArtifactIntegrityMismatch; + hasher.update(buffer[0..read]); + } + if (total != expected_byte_len) return error.ArtifactIntegrityMismatch; + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + const actual = std.fmt.bytesToHex(digest, .lower); + if (!std.mem.eql(u8, &actual, expected_checksum)) return error.ArtifactIntegrityMismatch; + try cancellation.check(); +} + fn threadedIo() std.Io.Threaded { return std.Io.Threaded.init(std.heap.page_allocator, .{}); } +fn lockAtomic(mutex: *std.atomic.Mutex) void { + while (!mutex.tryLock()) std.atomic.spinLoopHint(); +} + fn fileExists(path: []const u8) bool { var io_impl = threadedIo(); defer io_impl.deinit(); @@ -247,6 +478,17 @@ fn readFileRangeAllocWithCancellation( else try std.Io.Dir.cwd().openFile(io, path, .{}); defer file.close(io); + return try readOpenFileRangeAllocWithCancellation(alloc, file, io, offset, len, cancellation); +} + +fn readOpenFileRangeAllocWithCancellation( + alloc: Allocator, + file: std.Io.File, + io: std.Io, + offset: u64, + len: usize, + cancellation: CancellationToken, +) ![]u8 { const stat = try file.stat(io); if (offset > stat.size) return error.InvalidRange; const available = stat.size - offset; @@ -281,12 +523,18 @@ fn ensureParentDir(path: []const u8) !void { } fn writeFileAtomically(path: []const u8, contents: []const u8) !void { + return writeFileAtomicallyWithCancellation(path, contents, .none); +} + +fn writeFileAtomicallyWithCancellation(path: []const u8, contents: []const u8, cancellation: CancellationToken) !void { + try cancellation.check(); const tmp_path = try std.fmt.allocPrint(std.heap.page_allocator, "{s}.tmp-{d}", .{ path, test_nonce.fetchAdd(1, .monotonic) }); defer std.heap.page_allocator.free(tmp_path); var io_impl = threadedIo(); defer io_impl.deinit(); const io = io_impl.io(); + errdefer std.Io.Dir.deleteFileAbsolute(io, tmp_path) catch {}; { var file = try std.Io.Dir.createFileAbsolute(io, tmp_path, .{ .truncate = true }); @@ -294,10 +542,19 @@ fn writeFileAtomically(path: []const u8, contents: []const u8) !void { var buf: [4096]u8 = undefined; var writer = file.writer(io, &buf); - try writer.interface.writeAll(contents); + const cancellation_chunk_bytes = 1024 * 1024; + var offset: usize = 0; + while (offset < contents.len) { + try cancellation.check(); + const len = @min(cancellation_chunk_bytes, contents.len - offset); + try writer.interface.writeAll(contents[offset..][0..len]); + offset += len; + } try writer.end(); } + try cancellation.check(); + if (std.fs.path.isAbsolute(path)) { std.Io.Dir.renameAbsolute(tmp_path, path, io) catch |err| { std.Io.Dir.deleteFileAbsolute(io, tmp_path) catch {}; @@ -312,8 +569,21 @@ fn writeFileAtomically(path: []const u8, contents: []const u8) !void { } fn sha256StringAlloc(alloc: Allocator, contents: []const u8) ![]u8 { + return sha256StringWithCancellationAlloc(alloc, contents, .none); +} + +fn sha256StringWithCancellationAlloc(alloc: Allocator, contents: []const u8, cancellation: CancellationToken) ![]u8 { var digest: [32]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(contents, &digest, .{}); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes = 1024 * 1024; + var offset: usize = 0; + while (offset < contents.len) { + try cancellation.check(); + const len = @min(chunk_bytes, contents.len - offset); + hasher.update(contents[offset..][0..len]); + offset += len; + } + hasher.final(&digest); const out = try alloc.alloc(u8, 64); for (digest, 0..) |byte, idx| { @@ -403,6 +673,121 @@ test "fs artifact store getRangeAlloc returns requested slice" { try std.testing.expectEqualStrings("cde", mid); } +test "serverless fs artifact store detects and repairs same-length content corruption" { + const alloc = std.testing.allocator; + var path_buf: [256]u8 = undefined; + const root = tmpPath(&path_buf, "repair-corruption"); + defer cleanupTmp(root); + + var store = try FsStore.init(alloc, std.mem.span(root)); + defer store.deinit(); + var meta = try store.put(alloc, "alpha"); + defer meta.deinit(alloc); + const path = try pathForArtifactAlloc(alloc, store.root_dir, meta.checksum); + defer alloc.free(path); + try writeFileAtomically(path, "omega"); + + var iface = store.artifactStore(); + try std.testing.expectError(error.ArtifactIntegrityMismatch, iface.verifyContentWithCancellationUsingAllocator( + alloc, + meta.artifact_id, + meta.byte_len, + meta.checksum, + .none, + )); + var repaired = try store.put(alloc, "alpha"); + defer repaired.deinit(alloc); + try iface.verifyContentWithCancellationUsingAllocator(alloc, repaired.artifact_id, repaired.byte_len, repaired.checksum, .none); + const payload = try store.getAlloc(alloc, repaired.artifact_id); + defer alloc.free(payload); + try std.testing.expectEqualStrings("alpha", payload); +} + +test "serverless fs artifact verified range budgets cold authentication and amortizes warm reads" { + const alloc = std.testing.allocator; + var path_buf: [256]u8 = undefined; + const root = tmpPath(&path_buf, "budgeted-verified-range"); + defer cleanupTmp(root); + var store = try FsStore.init(alloc, std.mem.span(root)); + defer store.deinit(); + var meta = try store.put(alloc, "alpha"); + defer meta.deinit(alloc); + store.forgetVerifiedFile(meta.artifact_id); + var iface = store.artifactStore(); + var remaining: u64 = 2; + try std.testing.expectError(error.ArtifactReadBudgetExceeded, iface.getVerifiedRangeAllocWithBudget(alloc, meta.artifact_id, meta.byte_len, meta.checksum, 0, 1, .none, &remaining)); + try std.testing.expectEqual(@as(u64, 1), remaining); + remaining = 6; + const cold = try iface.getVerifiedRangeAllocWithBudget(alloc, meta.artifact_id, meta.byte_len, meta.checksum, 0, 1, .none, &remaining); + defer alloc.free(cold); + try std.testing.expectEqualStrings("a", cold); + try std.testing.expectEqual(@as(u64, 0), remaining); + remaining = 1; + const warm = try iface.getVerifiedRangeAllocWithBudget(alloc, meta.artifact_id, meta.byte_len, meta.checksum, 0, 1, .none, &remaining); + defer alloc.free(warm); + try std.testing.expectEqualStrings("a", warm); + try std.testing.expectEqual(@as(u64, 0), remaining); + const path = try pathForArtifactAlloc(alloc, store.root_dir, meta.checksum); + defer alloc.free(path); + try writeFileAtomically(path, "omega"); + remaining = 6; + try std.testing.expectError(error.ArtifactIntegrityMismatch, iface.getVerifiedRangeAllocWithBudget(alloc, meta.artifact_id, meta.byte_len, meta.checksum, 0, 1, .none, &remaining)); +} + +test "serverless fs artifact verification cache detects in-place mutation with restored mtime" { + const alloc = std.testing.allocator; + var path_buf: [256]u8 = undefined; + const root = tmpPath(&path_buf, "cached-in-place-corruption"); + defer cleanupTmp(root); + + var store = try FsStore.init(alloc, std.mem.span(root)); + defer store.deinit(); + var meta = try store.put(alloc, "alpha"); + defer meta.deinit(alloc); + const path = try pathForArtifactAlloc(alloc, store.root_dir, meta.checksum); + defer alloc.free(path); + + var iface = store.artifactStore(); + try iface.verifyContentWithCancellationUsingAllocator(alloc, meta.artifact_id, meta.byte_len, meta.checksum, .none); + + var io_impl = threadedIo(); + defer io_impl.deinit(); + const io = io_impl.io(); + const before = try std.Io.Dir.cwd().statFile(io, path, .{}); + // Some CI filesystems expose ctime at a coarser resolution than their + // write path. Wait for that observable clock to advance so this test + // exercises cache invalidation instead of assuming nanosecond precision. + var after = before; + for (0..200) |attempt| { + if (attempt > 0) try io.sleep(std.Io.Duration.fromMilliseconds(10), .awake); + { + var file = try std.Io.Dir.createFileAbsolute(io, path, .{ .truncate = true }); + defer file.close(io); + var buffer: [32]u8 = undefined; + var writer = file.writer(io, &buffer); + try writer.interface.writeAll(if (attempt % 2 == 0) "omega" else "sigma"); + try writer.end(); + } + try std.Io.Dir.cwd().setTimestamps(io, path, .{ .modify_timestamp = .{ .new = before.mtime } }); + after = try std.Io.Dir.cwd().statFile(io, path, .{}); + if (!std.meta.eql(before.ctime, after.ctime)) break; + } + try std.testing.expectEqual(before.inode, after.inode); + try std.testing.expectEqual(before.size, after.size); + try std.testing.expect(std.meta.eql(before.mtime, after.mtime)); + try std.testing.expect(!std.meta.eql(before.ctime, after.ctime)); + + try std.testing.expectError(error.ArtifactIntegrityMismatch, iface.getVerifiedRangeAllocWithCancellationUsingAllocator( + alloc, + meta.artifact_id, + meta.byte_len, + meta.checksum, + 0, + 5, + .none, + )); +} + test "fs artifact store rejects malformed content addresses before lookup" { var path_buf: [256]u8 = undefined; const path = tmpPath(&path_buf, "invalid-id"); diff --git a/zig/pkg/antfly/src/serverless/artifacts/mod.zig b/zig/pkg/antfly/src/serverless/artifacts/mod.zig index 11c2c5779d..195c0f5885 100644 --- a/zig/pkg/antfly/src/serverless/artifacts/mod.zig +++ b/zig/pkg/antfly/src/serverless/artifacts/mod.zig @@ -21,6 +21,7 @@ pub const ArtifactStore = store.ArtifactStore; pub const sha256ChecksumFromArtifactId = store.sha256ChecksumFromArtifactId; pub const validateSha256ArtifactIdentity = store.validateSha256ArtifactIdentity; pub const validateSha256Checksum = store.validateSha256Checksum; +pub const sha256DigestFromChecksum = store.sha256DigestFromChecksum; pub const validatePayloadSha256WithCancellation = store.validatePayloadSha256WithCancellation; pub const FsStore = fs_store.FsStore; pub const RemoteStore = remote_store.RemoteStore; diff --git a/zig/pkg/antfly/src/serverless/artifacts/object_store.zig b/zig/pkg/antfly/src/serverless/artifacts/object_store.zig index 8fec527dc2..1d86c8eba8 100644 --- a/zig/pkg/antfly/src/serverless/artifacts/object_store.zig +++ b/zig/pkg/antfly/src/serverless/artifacts/object_store.zig @@ -20,6 +20,31 @@ const object_store_support = @import("../object_store_support.zig"); const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; pub const ObjectStore = struct { + const verified_object_cache_limit: usize = 4096; + const VerifiedObject = struct { + byte_len: u64, + identity: [std.crypto.hash.sha2.Sha256.digest_length]u8, + version_id: ?[]u8 = null, + etag: ?[]u8 = null, + + fn deinit(self: *VerifiedObject, alloc: std.mem.Allocator) void { + if (self.version_id) |value| alloc.free(value); + if (self.etag) |value| alloc.free(value); + self.* = undefined; + } + }; + + const VerifiedObjectPin = struct { + version_id: ?[]u8 = null, + etag: ?[]u8 = null, + + fn deinit(self: *VerifiedObjectPin, alloc: std.mem.Allocator) void { + if (self.version_id) |value| alloc.free(value); + if (self.etag) |value| alloc.free(value); + self.* = undefined; + } + }; + alloc: std.mem.Allocator, client: objectstore.Client, fs_client: ?*objectstore.FilesystemClient = null, @@ -28,6 +53,8 @@ pub const ObjectStore = struct { owns_client: bool = true, bucket: []u8, prefix: []u8, + verified_mu: std.atomic.Mutex = .unlocked, + verified_objects: std.StringHashMapUnmanaged(VerifiedObject) = .empty, pub fn initRemoteUri(alloc: std.mem.Allocator, uri: []const u8) !ObjectStore { return try initRemoteUriWithS3Options(alloc, uri, null); @@ -161,6 +188,14 @@ pub const ObjectStore = struct { } pub fn deinit(self: *ObjectStore) void { + lockAtomic(&self.verified_mu); + var verified_it = self.verified_objects.iterator(); + while (verified_it.next()) |entry| { + self.alloc.free(entry.key_ptr.*); + entry.value_ptr.deinit(self.alloc); + } + self.verified_objects.deinit(self.alloc); + self.verified_mu.unlock(); if (self.owns_client) self.client.deinit(); if (self.fs_client) |fs| self.alloc.destroy(fs); if (self.gcs_client) |gcs| self.alloc.destroy(gcs); @@ -179,16 +214,46 @@ pub const ObjectStore = struct { } pub fn put(self: *ObjectStore, alloc: std.mem.Allocator, contents: []const u8) !artifact_store.ArtifactMetadata { - const checksum = try sha256StringAlloc(alloc, contents); + return try self.putWithCancellation(alloc, contents, .none); + } + + pub fn putWithCancellation(self: *ObjectStore, alloc: std.mem.Allocator, contents: []const u8, cancellation: CancellationToken) !artifact_store.ArtifactMetadata { + try cancellation.check(); + const checksum = try sha256StringWithCancellationAlloc(alloc, contents, cancellation); errdefer alloc.free(checksum); const artifact_id = try makeArtifactIdAlloc(alloc, checksum); errdefer alloc.free(artifact_id); const key = try keyForChecksumAlloc(self.alloc, self.prefix, checksum); defer self.alloc.free(key); - var result = try self.client.putObject(self.bucket, key, contents, .{ + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + _ = std.fmt.hexToBytes(&digest, checksum) catch unreachable; + var checksum_base64_buf: [std.base64.standard.Encoder.calcSize(digest.len)]u8 = undefined; + const checksum_base64 = std.base64.standard.Encoder.encode(&checksum_base64_buf, &digest); + var result = self.client.putObject(self.bucket, key, contents, .{ .content_type = "application/octet-stream", - }); + .checksum_sha256_base64 = if (self.s3_client != null) checksum_base64 else null, + // GCS exposes provider-computed CRC32C/MD5 metadata. A caller-owned + // custom SHA value is not an authenticated object checksum and must + // not be used to bypass content verification. + .checksum_sha256_hex = null, + .if_none_match = true, + .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), + }) catch |err| switch (normalizeCancellationError(err, cancellation)) { + error.PreconditionFailed => { + // Content-addressed keys are immutable. Concurrent/idempotent + // writers may lose the create race, but the existing object is + // accepted only after authenticating it against the digest we + // computed from `contents`. + try self.verifyContent(alloc, artifact_id, contents.len, checksum, cancellation); + return .{ + .artifact_id = artifact_id, + .byte_len = @intCast(contents.len), + .checksum = checksum, + }; + }, + else => |normalized| return normalized, + }; defer result.deinit(self.client.allocator); return .{ @@ -214,7 +279,7 @@ pub const ObjectStore = struct { defer self.alloc.free(key); var result = self.client.getObject(self.bucket, key, .{ .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), - }) catch |err| return normalizeCancellationError(err); + }) catch |err| return normalizeCancellationError(err, cancellation); defer result.deinit(self.client.allocator); try cancellation.check(); return try dupeWithCancellationAlloc(alloc, result.body, cancellation); @@ -241,12 +306,68 @@ pub const ObjectStore = struct { .skip_metadata_probe = true, .max_response_bytes = len, .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), - }) catch |err| return normalizeCancellationError(err); + }) catch |err| return normalizeCancellationError(err, cancellation); defer result.deinit(self.client.allocator); try cancellation.check(); return try dupeWithCancellationAlloc(alloc, result.body, cancellation); } + pub fn getVerifiedRangeAllocWithCancellation( + self: *ObjectStore, + alloc: std.mem.Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + return self.getVerifiedRangeWithBudget(alloc, artifact_id, expected_byte_len, expected_checksum, offset, len, cancellation, null); + } + + fn getVerifiedRangeWithBudget( + self: *ObjectStore, + alloc: std.mem.Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + remaining: ?*u64, + ) ![]u8 { + try cancellation.check(); + const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); + if (!std.mem.eql(u8, checksum, expected_checksum)) return error.ArtifactIntegrityMismatch; + const end = std.math.add(u64, offset, std.math.cast(u64, len) orelse return error.InvalidRange) catch return error.InvalidRange; + if (end > expected_byte_len) return error.InvalidRange; + + var pin = (try self.verifiedObjectPinAlloc(alloc, artifact_id, expected_byte_len)) orelse blk: { + try self.verifyContentWithBudget(alloc, artifact_id, expected_byte_len, expected_checksum, cancellation, remaining); + break :blk (try self.verifiedObjectPinAlloc(alloc, artifact_id, expected_byte_len)) orelse + return error.ArtifactIdentityUnavailable; + }; + defer pin.deinit(alloc); + + const key = try keyForChecksumAlloc(self.alloc, self.prefix, checksum); + defer self.alloc.free(key); + var result = self.client.getObject(self.bucket, key, .{ + .range = .{ .offset = offset, .length = len }, + .version_id = pin.version_id, + .if_match_etag = pin.etag, + .skip_metadata_probe = true, + .max_response_bytes = len, + .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), + }) catch |err| switch (normalizeCancellationError(err, cancellation)) { + error.PreconditionFailed, error.FileNotFound => return error.ArtifactIntegrityMismatch, + else => |normalized| return normalized, + }; + defer result.deinit(self.client.allocator); + if (result.body.len != len) return error.ArtifactIntegrityMismatch; + try cancellation.check(); + return try dupeWithCancellationAlloc(alloc, result.body, cancellation); + } + pub fn stat(self: *ObjectStore, alloc: std.mem.Allocator, artifact_id: []const u8) !artifact_store.ArtifactMetadata { return try self.statWithCancellation(alloc, artifact_id, .none); } @@ -265,7 +386,7 @@ pub const ObjectStore = struct { defer self.alloc.free(key); var meta = self.client.statObjectWithOptions(self.bucket, key, .{ .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), - }) catch |err| return normalizeCancellationError(err); + }) catch |err| return normalizeCancellationError(err, cancellation); defer meta.deinit(self.client.allocator); try cancellation.check(); return .{ @@ -275,22 +396,186 @@ pub const ObjectStore = struct { }; } + pub fn verifyContent( + self: *ObjectStore, + alloc: std.mem.Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + cancellation: CancellationToken, + ) !void { + return self.verifyContentWithBudget(alloc, artifact_id, expected_byte_len, expected_checksum, cancellation, null); + } + + fn verifyContentWithBudget( + self: *ObjectStore, + _: std.mem.Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + cancellation: CancellationToken, + remaining: ?*u64, + ) !void { + const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); + if (!std.mem.eql(u8, checksum, expected_checksum)) return error.ArtifactIntegrityMismatch; + const key = try keyForChecksumAlloc(self.alloc, self.prefix, checksum); + defer self.alloc.free(key); + var meta = self.client.statObjectWithOptions(self.bucket, key, .{ + .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), + }) catch |err| return normalizeCancellationError(err, cancellation); + defer meta.deinit(self.client.allocator); + if (meta.content_length != expected_byte_len) return error.ArtifactIntegrityMismatch; + if (meta.checksum_scope == .object) { + if (meta.checksum) |native| { + if (native.checksum_type == .full_object) switch (native.algorithm) { + .sha256_hex => { + if (!std.ascii.eqlIgnoreCase(native.value, expected_checksum)) return error.ArtifactIntegrityMismatch; + const identity = metadataIdentity(meta) orelse return error.ArtifactIdentityUnavailable; + try self.rememberVerifiedObject(artifact_id, expected_byte_len, identity, meta.version_id, meta.etag); + return; + }, + .sha256_base64 => { + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + _ = std.fmt.hexToBytes(&digest, expected_checksum) catch return error.ArtifactIntegrityMismatch; + var encoded: [std.base64.standard.Encoder.calcSize(digest.len)]u8 = undefined; + const value = std.base64.standard.Encoder.encode(&encoded, &digest); + if (!std.mem.eql(u8, native.value, value)) return error.ArtifactIntegrityMismatch; + const identity = metadataIdentity(meta) orelse return error.ArtifactIdentityUnavailable; + try self.rememberVerifiedObject(artifact_id, expected_byte_len, identity, meta.version_id, meta.etag); + return; + }, + else => {}, + }; + } + } + + const identity = metadataIdentity(meta); + if (identity) |value| { + lockAtomic(&self.verified_mu); + if (self.verified_objects.get(artifact_id)) |cached| { + if (cached.byte_len == expected_byte_len and std.mem.eql(u8, &cached.identity, &value)) { + self.verified_mu.unlock(); + try cancellation.check(); + return; + } + } + self.verified_mu.unlock(); + } + + // Providers without a comparable SHA-256 metadata checksum are read in + // bounded ranges pinned to the provider identity when one is exposed. + // The final digest is authoritative and memory remains O(1). + if (remaining) |budget| try artifact_store.chargeReadBudget(budget, expected_byte_len); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes: u64 = 8 * 1024 * 1024; + var offset: u64 = 0; + while (offset < expected_byte_len) { + try cancellation.check(); + const len = @min(chunk_bytes, expected_byte_len - offset); + var part = self.client.getObject(self.bucket, key, .{ + .range = .{ .offset = offset, .length = len }, + .version_id = meta.version_id, + .if_match_etag = meta.etag, + .skip_metadata_probe = true, + .max_response_bytes = @intCast(len), + .cancellation = objectstore.CancellationToken.fromCallback(cancellation.ptr, cancellation.is_cancelled_fn), + }) catch |err| return normalizeCancellationError(err, cancellation); + defer part.deinit(self.client.allocator); + if (part.body.len != len) return error.ArtifactIntegrityMismatch; + hasher.update(part.body); + offset += len; + } + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + const actual = std.fmt.bytesToHex(digest, .lower); + if (!std.mem.eql(u8, &actual, expected_checksum)) return error.ArtifactIntegrityMismatch; + try cancellation.check(); + const value = identity orelse return error.ArtifactIdentityUnavailable; + try self.rememberVerifiedObject(artifact_id, expected_byte_len, value, meta.version_id, meta.etag); + } + pub fn delete(self: *ObjectStore, artifact_id: []const u8) !void { const checksum = try artifact_store.sha256ChecksumFromArtifactId(artifact_id); const key = try keyForChecksumAlloc(self.alloc, self.prefix, checksum); defer self.alloc.free(key); try self.client.deleteObject(self.bucket, key, .{}); + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + if (self.verified_objects.fetchRemove(artifact_id)) |removed| { + self.alloc.free(removed.key); + var value = removed.value; + value.deinit(self.alloc); + } + } + + fn rememberVerifiedObject( + self: *ObjectStore, + artifact_id: []const u8, + byte_len: u64, + identity: [32]u8, + version_id: ?[]const u8, + etag: ?[]const u8, + ) !void { + if (version_id == null and etag == null) return error.ArtifactIdentityUnavailable; + const owned_id = try self.alloc.dupe(u8, artifact_id); + errdefer self.alloc.free(owned_id); + const owned_version_id = if (version_id) |value| try self.alloc.dupe(u8, value) else null; + errdefer if (owned_version_id) |value| self.alloc.free(value); + const owned_etag = if (etag) |value| try self.alloc.dupe(u8, value) else null; + errdefer if (owned_etag) |value| self.alloc.free(value); + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + if (!self.verified_objects.contains(artifact_id) and self.verified_objects.count() >= verified_object_cache_limit) { + var iterator = self.verified_objects.keyIterator(); + if (iterator.next()) |victim| { + const removed = self.verified_objects.fetchRemove(victim.*).?; + self.alloc.free(removed.key); + var value = removed.value; + value.deinit(self.alloc); + } + } + const gop = try self.verified_objects.getOrPut(self.alloc, owned_id); + if (gop.found_existing) { + self.alloc.free(owned_id); + gop.value_ptr.deinit(self.alloc); + } else gop.key_ptr.* = owned_id; + gop.value_ptr.* = .{ + .byte_len = byte_len, + .identity = identity, + .version_id = owned_version_id, + .etag = owned_etag, + }; + } + + fn verifiedObjectPinAlloc( + self: *ObjectStore, + alloc: std.mem.Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + ) !?VerifiedObjectPin { + lockAtomic(&self.verified_mu); + defer self.verified_mu.unlock(); + const verified = self.verified_objects.get(artifact_id) orelse return null; + if (verified.byte_len != expected_byte_len) return null; + const version_id = if (verified.version_id) |value| try alloc.dupe(u8, value) else null; + errdefer if (version_id) |value| alloc.free(value); + const etag = if (verified.etag) |value| try alloc.dupe(u8, value) else null; + return .{ .version_id = version_id, .etag = etag }; } const vtable: artifact_store.ArtifactStore.VTable = .{ .deinit = erasedDeinit, .put = erasedPut, + .put_with_cancellation = erasedPutWithCancellation, .get_alloc = erasedGetAlloc, .get_alloc_with_cancellation = erasedGetAllocWithCancellation, .get_range_alloc = erasedGetRangeAlloc, .get_range_alloc_with_cancellation = erasedGetRangeAllocWithCancellation, + .get_verified_range_alloc_with_cancellation = erasedGetVerifiedRangeAllocWithCancellation, + .get_verified_range_alloc_with_budget = erasedGetVerifiedRangeWithBudget, .stat = erasedStat, .stat_with_cancellation = erasedStatWithCancellation, + .verify_content = erasedVerifyContent, .delete = erasedDelete, }; @@ -304,6 +589,11 @@ pub const ObjectStore = struct { return try self.put(alloc, contents); } + fn erasedPutWithCancellation(ptr: *anyopaque, alloc: std.mem.Allocator, contents: []const u8, cancellation: CancellationToken) !artifact_store.ArtifactMetadata { + const self: *ObjectStore = @ptrCast(@alignCast(ptr)); + return try self.putWithCancellation(alloc, contents, cancellation); + } + fn erasedGetAlloc(ptr: *anyopaque, alloc: std.mem.Allocator, artifact_id: []const u8) ![]u8 { const self: *ObjectStore = @ptrCast(@alignCast(ptr)); return try self.getAlloc(alloc, artifact_id); @@ -324,6 +614,11 @@ pub const ObjectStore = struct { return try self.getRangeAllocWithCancellation(alloc, artifact_id, offset, len, cancellation); } + fn erasedGetVerifiedRangeAllocWithCancellation(ptr: *anyopaque, alloc: std.mem.Allocator, artifact_id: []const u8, expected_byte_len: u64, expected_checksum: []const u8, offset: u64, len: usize, cancellation: CancellationToken) ![]u8 { + const self: *ObjectStore = @ptrCast(@alignCast(ptr)); + return try self.getVerifiedRangeAllocWithCancellation(alloc, artifact_id, expected_byte_len, expected_checksum, offset, len, cancellation); + } + fn erasedStat(ptr: *anyopaque, alloc: std.mem.Allocator, artifact_id: []const u8) !artifact_store.ArtifactMetadata { const self: *ObjectStore = @ptrCast(@alignCast(ptr)); return try self.stat(alloc, artifact_id); @@ -334,16 +629,43 @@ pub const ObjectStore = struct { return try self.statWithCancellation(alloc, artifact_id, cancellation); } + fn erasedVerifyContent(ptr: *anyopaque, alloc: std.mem.Allocator, artifact_id: []const u8, expected_byte_len: u64, expected_checksum: []const u8, cancellation: CancellationToken) !void { + const self: *ObjectStore = @ptrCast(@alignCast(ptr)); + try self.verifyContent(alloc, artifact_id, expected_byte_len, expected_checksum, cancellation); + } + + fn erasedGetVerifiedRangeWithBudget(ptr: *anyopaque, alloc: std.mem.Allocator, artifact_id: []const u8, byte_len: u64, checksum: []const u8, offset: u64, len: usize, cancellation: CancellationToken, remaining: *u64) ![]u8 { + const self: *ObjectStore = @ptrCast(@alignCast(ptr)); + return self.getVerifiedRangeWithBudget(alloc, artifact_id, byte_len, checksum, offset, len, cancellation, remaining); + } + fn erasedDelete(ptr: *anyopaque, artifact_id: []const u8) !void { const self: *ObjectStore = @ptrCast(@alignCast(ptr)); try self.delete(artifact_id); } }; -fn normalizeCancellationError(err: anyerror) anyerror { +fn normalizeCancellationError(err: anyerror, cancellation: CancellationToken) anyerror { + // Recover typed lease loss from the transport's boolean-only callback. + if (err == error.Cancelled or err == error.Canceled) { + cancellation.check() catch |reason| return reason; + } return if (err == error.Cancelled) error.Canceled else err; } +test "serverless objectstore cancellation preserves scoped lease failures" { + var state: u8 = 0; + const token = CancellationToken{ .ptr = &state, .check_fn = struct { + fn check(_: *const anyopaque) !void { + return error.WorkLeaseLost; + } + }.check }; + try std.testing.expectEqual(error.WorkLeaseLost, normalizeCancellationError(error.Cancelled, token)); + try std.testing.expectEqual(error.WorkLeaseLost, normalizeCancellationError(error.Canceled, token)); + try std.testing.expectEqual(error.Canceled, normalizeCancellationError(error.Cancelled, .none)); + try std.testing.expectEqual(error.Timeout, normalizeCancellationError(error.Timeout, token)); +} + fn dupeWithCancellationAlloc( alloc: std.mem.Allocator, source: []const u8, @@ -364,8 +686,21 @@ fn dupeWithCancellationAlloc( } fn sha256StringAlloc(alloc: std.mem.Allocator, contents: []const u8) ![]u8 { + return sha256StringWithCancellationAlloc(alloc, contents, .none); +} + +fn sha256StringWithCancellationAlloc(alloc: std.mem.Allocator, contents: []const u8, cancellation: CancellationToken) ![]u8 { var digest: [32]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(contents, &digest, .{}); + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes = 1024 * 1024; + var offset: usize = 0; + while (offset < contents.len) { + try cancellation.check(); + const len = @min(chunk_bytes, contents.len - offset); + hasher.update(contents[offset..][0..len]); + offset += len; + } + hasher.final(&digest); const out = try alloc.alloc(u8, 64); for (digest, 0..) |byte, idx| { @@ -379,6 +714,42 @@ fn makeArtifactIdAlloc(alloc: std.mem.Allocator, checksum: []const u8) ![]u8 { return try std.fmt.allocPrint(alloc, "sha256:{s}", .{checksum}); } +fn metadataIdentity(meta: objectstore.ObjectMetadata) ?[std.crypto.hash.sha2.Sha256.digest_length]u8 { + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + var has_identity = false; + if (meta.version_id) |value| { + hashIdentityField(&hasher, 1, value); + has_identity = true; + } + if (meta.etag) |value| { + hashIdentityField(&hasher, 2, value); + has_identity = true; + } + if (meta.checksum) |checksum| { + var algorithm: u64 = @intFromEnum(checksum.algorithm); + var checksum_type: u64 = @intFromEnum(checksum.checksum_type); + hashIdentityField(&hasher, 3, std.mem.asBytes(&algorithm)); + hashIdentityField(&hasher, 4, std.mem.asBytes(&checksum_type)); + hashIdentityField(&hasher, 5, checksum.value); + has_identity = true; + } + if (!has_identity) return null; + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + return digest; +} + +fn hashIdentityField(hasher: *std.crypto.hash.sha2.Sha256, tag: u8, value: []const u8) void { + hasher.update(&.{tag}); + var len: u64 = value.len; + hasher.update(std.mem.asBytes(&len)); + hasher.update(value); +} + +fn lockAtomic(mutex: *std.atomic.Mutex) void { + while (!mutex.tryLock()) std.atomic.spinLoopHint(); +} + fn keyForChecksumAlloc(alloc: std.mem.Allocator, prefix: []const u8, checksum: []const u8) ![]u8 { try artifact_store.validateSha256Checksum(checksum); if (prefix.len == 0) return try std.fmt.allocPrint(alloc, "sha256/{s}/{s}", .{ checksum[0..2], checksum[2..] }); @@ -403,6 +774,9 @@ test "objectstore-backed artifacts store round-trips over file uri" { var meta = try store.put("payload"); defer meta.deinit(std.testing.allocator); + var duplicate = try store.put("payload"); + defer duplicate.deinit(std.testing.allocator); + try std.testing.expectEqualStrings(meta.artifact_id, duplicate.artifact_id); const got = try store.getAlloc(meta.artifact_id); defer std.testing.allocator.free(got); try std.testing.expectEqualStrings("payload", got); @@ -456,6 +830,7 @@ test "serverless objectstore-backed artifacts preserve distinct client and resul var stat = try store.stat(meta.artifact_id); defer stat.deinit(result_alloc); try std.testing.expectEqual(@as(u64, "allocator-safe".len), stat.byte_len); + try store.verifyContentWithCancellationUsingAllocator(result_alloc, meta.artifact_id, meta.byte_len, meta.checksum, .none); var query_buffer: [1024]u8 = undefined; var query_fba = std.heap.FixedBufferAllocator.init(&query_buffer); @@ -470,6 +845,137 @@ test "serverless objectstore-backed artifacts preserve distinct client and resul try std.testing.expect(query_fba.ownsSlice(verified)); query_alloc.free(verified); try std.testing.expectEqual(@as(usize, 0), query_fba.end_index); + + const key = try keyForChecksumAlloc(result_alloc, "tenant/a", meta.checksum); + defer result_alloc.free(key); + var corrupt = try result_alloc.dupe(u8, "allocator-safe"); + defer result_alloc.free(corrupt); + corrupt[0] ^= 1; + var client = memory.client(); + var replaced = try client.putObject("bucket", key, corrupt, .{}); + defer replaced.deinit(client_alloc); + try std.testing.expectError( + error.ArtifactIntegrityMismatch, + store.getVerifiedRangeAllocWithCancellationUsingAllocator( + result_alloc, + meta.artifact_id, + meta.byte_len, + meta.checksum, + 0, + 4, + .none, + ), + ); + try std.testing.expectError( + error.ArtifactIntegrityMismatch, + store.verifyContentWithCancellationUsingAllocator(result_alloc, meta.artifact_id, meta.byte_len, meta.checksum, .none), + ); +} + +test "serverless objectstore verification caches immutable provider identities" { + const alloc = std.testing.allocator; + var memory = objectstore.MemoryClient.init(alloc); + defer memory.deinit(); + + const EtagOnlyClient = struct { + inner: objectstore.Client, + + fn client(self: *@This()) objectstore.Client { + return .{ .allocator = self.inner.allocator, .ptr = self, .vtable = &vtable }; + } + + const vtable: objectstore.Client.VTable = .{ + .deinit = deinit, + .bucket_exists = bucketExists, + .make_bucket = makeBucket, + .put_object = putObject, + .get_object = getObject, + .get_object_attributes = getObjectAttributes, + .stat_object = statObject, + .stat_object_with_options = statObjectWithOptions, + .delete_object = deleteObject, + .list_objects = listObjects, + }; + + fn deinit(_: std.mem.Allocator, _: *anyopaque) void {} + + fn from(ptr: *anyopaque) *@This() { + return @ptrCast(@alignCast(ptr)); + } + + fn bucketExists(ptr: *anyopaque, bucket: []const u8, options: objectstore.BucketOptions) !bool { + return try from(ptr).inner.bucketExistsWithOptions(bucket, options); + } + + fn makeBucket(ptr: *anyopaque, bucket: []const u8, options: objectstore.BucketOptions) !void { + try from(ptr).inner.makeBucketWithOptions(bucket, options); + } + + fn putObject(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, key: []const u8, body: []const u8, options: objectstore.PutOptions) !objectstore.PutResult { + return try from(ptr).inner.putObject(bucket, key, body, options); + } + + fn getObject(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, key: []const u8, options: objectstore.GetOptions) !objectstore.GetResult { + return try from(ptr).inner.getObject(bucket, key, options); + } + + fn getObjectAttributes(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, key: []const u8) !objectstore.ObjectAttributes { + return try from(ptr).inner.getObjectAttributes(bucket, key); + } + + fn etagOnly(metadata: *objectstore.ObjectMetadata, metadata_alloc: std.mem.Allocator) void { + if (metadata.checksum) |*checksum| checksum.deinit(metadata_alloc); + metadata.checksum = null; + } + + fn statObject(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, key: []const u8) !objectstore.ObjectMetadata { + const self = from(ptr); + var metadata = try self.inner.statObject(bucket, key); + etagOnly(&metadata, self.inner.allocator); + return metadata; + } + + fn statObjectWithOptions(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, key: []const u8, options: objectstore.StatOptions) !objectstore.ObjectMetadata { + const self = from(ptr); + var metadata = try self.inner.statObjectWithOptions(bucket, key, options); + etagOnly(&metadata, self.inner.allocator); + return metadata; + } + + fn deleteObject(ptr: *anyopaque, bucket: []const u8, key: []const u8, options: objectstore.DeleteOptions) !void { + try from(ptr).inner.deleteObject(bucket, key, options); + } + + fn listObjects(ptr: *anyopaque, _: std.mem.Allocator, bucket: []const u8, options: objectstore.ListOptions) !objectstore.ListResult { + return try from(ptr).inner.listObjects(bucket, options); + } + }; + + var etag_only = EtagOnlyClient{ .inner = memory.client() }; + var impl = try ObjectStore.initWithClient(alloc, etag_only.client(), "bucket", "tenant/cache"); + var store = impl.artifactStore(); + defer store.deinit(); + + var metadata = try store.put("verify-once"); + defer metadata.deinit(alloc); + memory.resetOperationCount(); + var cold_allowance: u64 = 1; + try std.testing.expectError(error.ArtifactReadBudgetExceeded, store.getVerifiedRangeAllocWithBudget(alloc, metadata.artifact_id, metadata.byte_len, metadata.checksum, 0, 1, .none, &cold_allowance)); + // HEAD is permitted, but no unaffordable full-object GET may start. + try std.testing.expectEqual(@as(u64, 1), memory.operationCount()); + try std.testing.expectEqual(@as(u64, 0), cold_allowance); + memory.resetOperationCount(); + try store.verifyContentWithCancellationUsingAllocator(alloc, metadata.artifact_id, metadata.byte_len, metadata.checksum, .none); + try std.testing.expectEqual(@as(u64, 2), memory.operationCount()); + try store.verifyContentWithCancellationUsingAllocator(alloc, metadata.artifact_id, metadata.byte_len, metadata.checksum, .none); + try std.testing.expectEqual(@as(u64, 3), memory.operationCount()); + var warm_allowance: u64 = 1; + const warm = try store.getVerifiedRangeAllocWithBudget(alloc, metadata.artifact_id, metadata.byte_len, metadata.checksum, 0, 1, .none, &warm_allowance); + defer alloc.free(warm); + try std.testing.expectEqualStrings("v", warm); + // The authenticated provider pin makes reuse one GET, without another HEAD. + try std.testing.expectEqual(@as(u64, 4), memory.operationCount()); + try std.testing.expectEqual(@as(u64, 0), warm_allowance); } test "serverless objectstore-backed artifact initialization cleans up every allocation failure" { diff --git a/zig/pkg/antfly/src/serverless/artifacts/store.zig b/zig/pkg/antfly/src/serverless/artifacts/store.zig index 2646c71066..759c328a6c 100644 --- a/zig/pkg/antfly/src/serverless/artifacts/store.zig +++ b/zig/pkg/antfly/src/serverless/artifacts/store.zig @@ -13,6 +13,11 @@ // limitations. const std = @import("std"); + +pub fn chargeReadBudget(remaining: *u64, amount: u64) !void { + if (amount > remaining.*) return error.ArtifactReadBudgetExceeded; + remaining.* -= amount; +} const Allocator = std.mem.Allocator; const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; @@ -39,6 +44,13 @@ pub fn validateSha256Checksum(checksum: []const u8) !void { } } +pub fn sha256DigestFromChecksum(checksum: []const u8) ![std.crypto.hash.sha2.Sha256.digest_length]u8 { + try validateSha256Checksum(checksum); + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + _ = std.fmt.hexToBytes(&digest, checksum) catch return error.InvalidArtifactId; + return digest; +} + pub fn validateSha256ArtifactIdentity(artifact_id: []const u8, checksum: []const u8) !void { try validateSha256Checksum(checksum); const id_checksum = try sha256ChecksumFromArtifactId(artifact_id); @@ -69,12 +81,16 @@ pub const ArtifactStore = struct { pub const VTable = struct { deinit: *const fn (Allocator, *anyopaque) void, put: *const fn (*anyopaque, Allocator, []const u8) anyerror!ArtifactMetadata, + put_with_cancellation: ?*const fn (*anyopaque, Allocator, []const u8, CancellationToken) anyerror!ArtifactMetadata = null, get_alloc: *const fn (*anyopaque, Allocator, []const u8) anyerror![]u8, get_alloc_with_cancellation: ?*const fn (*anyopaque, Allocator, []const u8, CancellationToken) anyerror![]u8 = null, get_range_alloc: *const fn (*anyopaque, Allocator, []const u8, u64, usize) anyerror![]u8, get_range_alloc_with_cancellation: ?*const fn (*anyopaque, Allocator, []const u8, u64, usize, CancellationToken) anyerror![]u8 = null, + get_verified_range_alloc_with_cancellation: ?*const fn (*anyopaque, Allocator, []const u8, u64, []const u8, u64, usize, CancellationToken) anyerror![]u8 = null, + get_verified_range_alloc_with_budget: ?*const fn (*anyopaque, Allocator, []const u8, u64, []const u8, u64, usize, CancellationToken, *u64) anyerror![]u8 = null, stat: *const fn (*anyopaque, Allocator, []const u8) anyerror!ArtifactMetadata, stat_with_cancellation: ?*const fn (*anyopaque, Allocator, []const u8, CancellationToken) anyerror!ArtifactMetadata = null, + verify_content: ?*const fn (*anyopaque, Allocator, []const u8, u64, []const u8, CancellationToken) anyerror!void = null, delete: *const fn (*anyopaque, []const u8) anyerror!void, }; @@ -84,7 +100,18 @@ pub const ArtifactStore = struct { } pub fn put(self: *ArtifactStore, contents: []const u8) !ArtifactMetadata { - return try self.vtable.put(self.ptr, self.allocator, contents); + return try self.putWithCancellation(contents, .none); + } + + pub fn putWithCancellation(self: *ArtifactStore, contents: []const u8, cancellation: CancellationToken) !ArtifactMetadata { + try cancellation.check(); + var metadata = if (self.vtable.put_with_cancellation) |put_with_cancellation| + try put_with_cancellation(self.ptr, self.allocator, contents, cancellation) + else + try self.vtable.put(self.ptr, self.allocator, contents); + errdefer metadata.deinit(self.allocator); + try cancellation.check(); + return metadata; } pub fn getAlloc(self: *ArtifactStore, artifact_id: []const u8) ![]u8 { @@ -222,6 +249,104 @@ pub const ArtifactStore = struct { return payload; } + /// Loads a range from the exact object or file identity authenticated by + /// `expected_checksum`. Native backends pin the provider generation, ETag, + /// or open file identity across verification and reading. The fallback is + /// retained for test and custom stores, but production stores should + /// implement the vtable entry so replacement cannot race a range read. + pub fn getVerifiedRangeAllocWithCancellationUsingAllocator( + self: *ArtifactStore, + result_alloc: Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + try cancellation.check(); + validateSha256ArtifactIdentity(artifact_id, expected_checksum) catch + return error.ArtifactIntegrityMismatch; + const end = std.math.add(u64, offset, std.math.cast(u64, len) orelse return error.InvalidRange) catch + return error.InvalidRange; + if (end > expected_byte_len) return error.InvalidRange; + if (len == 0) { + try self.verifyContentWithCancellationUsingAllocator( + result_alloc, + artifact_id, + expected_byte_len, + expected_checksum, + cancellation, + ); + const empty = try result_alloc.alloc(u8, 0); + errdefer result_alloc.free(empty); + try cancellation.check(); + return empty; + } + + const payload = if (self.vtable.get_verified_range_alloc_with_cancellation) |get_verified_range| + try get_verified_range( + self.ptr, + result_alloc, + artifact_id, + expected_byte_len, + expected_checksum, + offset, + len, + cancellation, + ) + else blk: { + try self.verifyContentWithCancellationUsingAllocator( + result_alloc, + artifact_id, + expected_byte_len, + expected_checksum, + cancellation, + ); + break :blk try self.getRangeAllocWithCancellationUsingAllocator( + result_alloc, + artifact_id, + offset, + len, + cancellation, + ); + }; + errdefer result_alloc.free(payload); + if (payload.len != len) return error.ArtifactIntegrityMismatch; + try cancellation.check(); + return payload; + } + + /// A shared read allowance covers both the requested range and any cold + /// full-content authentication. Native backends charge only cache misses. + pub fn getVerifiedRangeAllocWithBudget( + self: *ArtifactStore, + alloc: Allocator, + artifact_id: []const u8, + byte_len: u64, + checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + remaining: *u64, + ) ![]u8 { + try cancellation.check(); + try validateSha256ArtifactIdentity(artifact_id, checksum); + const end = std.math.add(u64, offset, len) catch return error.InvalidRange; + if (end > byte_len) return error.InvalidRange; + try chargeReadBudget(remaining, len); + const bytes = if (self.vtable.get_verified_range_alloc_with_budget) |read| + try read(self.ptr, alloc, artifact_id, byte_len, checksum, offset, len, cancellation, remaining) + else blk: { + try chargeReadBudget(remaining, byte_len); + break :blk try self.getVerifiedRangeAllocWithCancellationUsingAllocator(alloc, artifact_id, byte_len, checksum, offset, len, cancellation); + }; + errdefer alloc.free(bytes); + if (bytes.len != len) return error.ArtifactIntegrityMismatch; + try cancellation.check(); + return bytes; + } + pub fn stat(self: *ArtifactStore, artifact_id: []const u8) !ArtifactMetadata { return try self.statWithCancellation(artifact_id, .none); } @@ -249,6 +374,56 @@ pub const ArtifactStore = struct { pub fn delete(self: *ArtifactStore, artifact_id: []const u8) !void { try self.vtable.delete(self.ptr, artifact_id); } + + /// Verifies a content-addressed artifact with bounded memory. Backends may + /// provide a native streaming verifier; the portable fallback hashes + /// bounded ranges and therefore never allocates the full artifact. + pub fn verifyContentWithCancellationUsingAllocator( + self: *ArtifactStore, + result_alloc: Allocator, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + cancellation: CancellationToken, + ) !void { + try cancellation.check(); + validateSha256ArtifactIdentity(artifact_id, expected_checksum) catch return error.ArtifactIntegrityMismatch; + + // A backend verifier owns the complete identity, length, and content + // check. Calling stat here as well would duplicate provider HEADs on + // every verification and defeat backend identity caches. + if (self.vtable.verify_content) |verify_content| { + try verify_content(self.ptr, result_alloc, artifact_id, expected_byte_len, expected_checksum, cancellation); + return; + } + + // Portable backends without a native verifier first validate the + // declared metadata, then hash bounded ranges below. + var metadata = try self.statWithCancellationUsingAllocator(result_alloc, artifact_id, cancellation); + defer metadata.deinit(result_alloc); + if (metadata.byte_len != expected_byte_len or + !std.mem.eql(u8, metadata.artifact_id, artifact_id) or + !std.mem.eql(u8, metadata.checksum, expected_checksum)) return error.ArtifactIntegrityMismatch; + + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + const chunk_bytes: usize = 8 * 1024 * 1024; + var offset: u64 = 0; + while (offset < expected_byte_len) { + try cancellation.check(); + const remaining = expected_byte_len - offset; + const len: usize = @intCast(@min(remaining, chunk_bytes)); + const chunk = try self.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); + defer result_alloc.free(chunk); + if (chunk.len != len) return error.ArtifactIntegrityMismatch; + hasher.update(chunk); + offset += len; + } + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + const actual = std.fmt.bytesToHex(digest, .lower); + if (!std.mem.eql(u8, &actual, expected_checksum)) return error.ArtifactIntegrityMismatch; + try cancellation.check(); + } }; pub fn validatePayloadSha256WithCancellation( diff --git a/zig/pkg/antfly/src/serverless/build/builder.zig b/zig/pkg/antfly/src/serverless/build/builder.zig index acc76a1240..7b001e97b5 100644 --- a/zig/pkg/antfly/src/serverless/build/builder.zig +++ b/zig/pkg/antfly/src/serverless/build/builder.zig @@ -14,6 +14,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const artifacts_mod = @import("../artifacts/mod.zig"); const catalog_mod = @import("../catalog/mod.zig"); const manifest_mod = @import("../manifest/mod.zig"); @@ -26,11 +27,15 @@ const search_sources = @import("../search_sources.zig"); const catalog_types = @import("../catalog/types.zig"); const document_segment_mod = @import("../document_segment/mod.zig"); const graph_segment_mod = @import("../graph_segment/mod.zig"); +const graph_metric_segment_mod = @import("../graph_metric_segment/mod.zig"); const segment_mod = @import("../segment/mod.zig"); const text_segment_mod = @import("../text_segment/mod.zig"); const sparse_segment_mod = @import("../sparse_segment/mod.zig"); const vector_segment_mod = @import("../vector_segment/mod.zig"); const vector_index = @import("vector_index.zig"); +const graph_metric_config = @import("graph_metric_config.zig"); +const graph_metric_policy = @import("graph_metric_policy.zig"); +const lake_graph_metric = @import("lake_graph_metric.zig"); const publication_plan = @import("publication_plan.zig"); const work_lease = @import("work_lease.zig"); const maintenance_cancellation = @import("../maintenance_cancellation.zig"); @@ -117,6 +122,8 @@ pub const Builder = struct { manifests: *manifest_mod.ManifestStore, progress: *catalog_mod.ProgressStore, wal: *wal_mod.WalStore, + io: ?std.Io = null, + graph_metric_max_parallelism: usize = lake_graph_metric.default_compute_parallelism, publication_lifecycle_hook: ?PublicationLifecycleHook = null, const CurrentHeadManifest = struct { @@ -146,6 +153,22 @@ pub const Builder = struct { }; } + pub fn setIo(self: *Builder, io: ?std.Io) void { + self.io = io; + } + + pub fn setGraphMetricMaxParallelism(self: *Builder, max_parallelism: usize) !void { + if (max_parallelism == 0 or max_parallelism > lake_graph_metric.max_compute_parallelism) return error.InvalidGraphMetricBuildOptions; + self.graph_metric_max_parallelism = max_parallelism; + } + + fn realtimeNs(self: *const Builder) u64 { + if (self.io) |io| return @intCast(std.Io.Timestamp.now(io, .real).toNanoseconds()); + var io_impl = threadedIo(); + defer io_impl.deinit(); + return @intCast(std.Io.Timestamp.now(io_impl.io(), .real).toNanoseconds()); + } + pub fn setPublicationLifecycleHook( self: *Builder, hook: ?PublicationLifecycleHook, @@ -309,6 +332,21 @@ pub const Builder = struct { ); } + pub fn publishNamespaceWithMetricAndPlanWithCancellation( + self: *Builder, + namespace: []const u8, + vector_metric: shared_vector.DistanceMetric, + plan: publication_plan.TablePublicationPlan, + cancellation: CancellationToken, + ) !BuildResult { + var fallback: ?std.Io.Threaded = if (self.io == null) threadedIo() else null; + defer if (fallback) |*value| value.deinit(); + return self.publishNamespaceWithMetricAndPlanGuardedUntil(namespace, vector_metric, plan, null, .{ + .io = self.io orelse fallback.?.io(), + .cooperative = cancellation, + }); + } + pub fn publishNamespaceWithMetricAndPlanGuardedUntil( self: *Builder, namespace: []const u8, @@ -532,9 +570,22 @@ pub const Builder = struct { ); defer freeArtifactRefs(self.alloc, graph_refs); try maintenance_cancellation.check(cancellation); + const graph_metric_specs = if (targets.include_graph) + try graph_metric_config.parseIndexSpecsAlloc(self.alloc, plan.table_definition.indexes_json) + else + try self.alloc.alloc(graph_metric_config.IndexSpec, 0); + defer graph_metric_config.freeIndexSpecs(self.alloc, graph_metric_specs); + const built_at_ns = records[records.len - 1].timestamp_ns; + const graph_metric_refs = try buildGraphMetricArtifactRefsAlloc(self.alloc, self.artifacts, current_manifest, graph_refs, graph_metric_specs, cancellation, .{ + .published_generation = next_version, + .edge_generation = next_version, + .computed_at_ms = @divTrunc(self.realtimeNs(), std.time.ns_per_ms), + }, self.io, self.graph_metric_max_parallelism); + defer freeArtifactRefs(self.alloc, graph_metric_refs); + const published_graph_refs = try concatArtifactRefSlicesAlloc(self.alloc, graph_refs, graph_metric_refs); + defer self.alloc.free(published_graph_refs); const wal_end_lsn = records[records.len - 1].lsn; - const built_at_ns = records[records.len - 1].timestamp_ns; var derived_outputs = try detectMaterializedDerivedOutputsAlloc( self.alloc, materialized.documents, @@ -565,7 +616,7 @@ pub const Builder = struct { text_refs, sparse_refs, vector_refs, - graph_refs, + published_graph_refs, derived_outputs, plan.policy, plan.table_definition, @@ -586,7 +637,7 @@ pub const Builder = struct { text_refs, sparse_refs, vector_refs, - graph_refs, + published_graph_refs, derived_outputs, plan.policy, plan.table_definition, @@ -841,6 +892,20 @@ pub const Builder = struct { ); defer freeArtifactRefs(self.alloc, graph_refs); try maintenance_cancellation.check(cancellation); + const graph_metric_specs = if (targets.include_graph) + try graph_metric_config.parseIndexSpecsAlloc(self.alloc, plan.table_definition.indexes_json) + else + try self.alloc.alloc(graph_metric_config.IndexSpec, 0); + defer graph_metric_config.freeIndexSpecs(self.alloc, graph_metric_specs); + const next_version = current_head + 1; + const graph_metric_refs = try buildGraphMetricArtifactRefsAlloc(self.alloc, self.artifacts, current, graph_refs, graph_metric_specs, cancellation, .{ + .published_generation = next_version, + .edge_generation = next_version, + .computed_at_ms = @divTrunc(self.realtimeNs(), std.time.ns_per_ms), + }, self.io, self.graph_metric_max_parallelism); + defer freeArtifactRefs(self.alloc, graph_metric_refs); + const published_graph_refs = try concatArtifactRefSlicesAlloc(self.alloc, graph_refs, graph_metric_refs); + defer self.alloc.free(published_graph_refs); var scanned_derived_outputs: ?search_sources.MaterializedDerivedOutputs = null; defer if (scanned_derived_outputs) |*outputs| search_sources.deinitMaterializedDerivedOutputs(self.alloc, outputs); @@ -870,7 +935,6 @@ pub const Builder = struct { defer search_sources.deinitMaterializedDerivedOutputs(self.alloc, &derived_outputs); try maintenance_cancellation.check(cancellation); - const next_version = current_head + 1; const wal_end_lsn = if (consumed_record) |record| record.lsn else current.wal_end_lsn; var manifest = try buildCompactedManifestFromRefsAlloc( self.alloc, @@ -887,7 +951,7 @@ pub const Builder = struct { text_refs, sparse_refs, vector_refs, - graph_refs, + published_graph_refs, derived_outputs, plan.policy, plan.table_definition, @@ -1235,8 +1299,8 @@ fn buildManifestAlloc( const text_count: usize = text_refs.len; const sparse_count: usize = sparse_refs.len; const vector_count: usize = vector_refs.len; - const graph_count: usize = graph_refs.len; - const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 2 + text_count + sparse_count + vector_count + graph_count); + const graph_count: usize = countArtifactRefsByKind(graph_refs, .graph_segment); + const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 2 + text_count + sparse_count + vector_count + graph_refs.len); errdefer alloc.free(artifacts); artifacts[0] = .{ .kind = .mutation_segment, @@ -1414,8 +1478,8 @@ fn buildRebasedManifestFromRefsAlloc( const text_count: usize = text_refs.len; const sparse_count: usize = sparse_refs.len; const vector_count: usize = vector_refs.len; - const graph_count: usize = graph_refs.len; - const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 1 + text_count + sparse_count + vector_count + graph_count); + const graph_count: usize = countArtifactRefsByKind(graph_refs, .graph_segment); + const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 1 + text_count + sparse_count + vector_count + graph_refs.len); errdefer alloc.free(artifacts); artifacts[0] = try cloneArtifactRefAlloc(alloc, document_ref); var artifact_index: usize = 1; @@ -1499,8 +1563,8 @@ fn buildCompactedManifestFromRefsAlloc( const text_count: usize = text_refs.len; const sparse_count: usize = sparse_refs.len; const vector_count: usize = vector_refs.len; - const graph_count: usize = graph_refs.len; - const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 1 + text_count + sparse_count + vector_count + graph_count); + const graph_count: usize = countArtifactRefsByKind(graph_refs, .graph_segment); + const artifacts = try alloc.alloc(manifest_mod.ArtifactRef, 1 + text_count + sparse_count + vector_count + graph_refs.len); errdefer alloc.free(artifacts); artifacts[0] = try cloneArtifactRefAlloc(alloc, document_ref); var artifact_index: usize = 1; @@ -2722,6 +2786,14 @@ pub fn countArtifactsByKind(manifest: manifest_mod.Manifest, kind: manifest_mod. return count; } +fn countArtifactRefsByKind(refs: []const manifest_mod.ArtifactRef, kind: manifest_mod.ArtifactKind) usize { + var count: usize = 0; + for (refs) |artifact| if (artifact.kind == kind) { + count += 1; + }; + return count; +} + pub const VectorArtifactInfo = struct { metric: shared_vector.DistanceMetric = shared_vector.default_distance_metric, cluster_count: usize = 0, @@ -2882,6 +2954,22 @@ pub fn cloneArtifactRefAlloc(alloc: Allocator, artifact: manifest_mod.ArtifactRe .artifact_id = try alloc.dupe(u8, artifact.artifact_id), .byte_len = artifact.byte_len, .checksum = try alloc.dupe(u8, artifact.checksum), + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }; } @@ -3781,6 +3869,8 @@ pub fn buildGraphArtifactRefsForMaterializedDocsAllocUntil( const refs = try alloc.alloc(manifest_mod.ArtifactRef, 1); errdefer alloc.free(refs); refs[0] = try artifactRefFromMetadataAlloc(alloc, .graph_segment, artifact); + errdefer freeArtifactRef(alloc, refs[0]); + try graph_segment_mod.codec.compact.bindTopologyControl(&refs[0], payload); return refs; } return try alloc.alloc(manifest_mod.ArtifactRef, 0); @@ -3813,6 +3903,7 @@ pub fn buildGraphArtifactRefsForMaterializedDocsAllocUntil( for (graph_index_names, 0..) |index_name, idx| { refs[idx] = try artifactRefFromMetadataNamedAlloc(alloc, .graph_segment, index_name, artifact); initialized += 1; + try graph_segment_mod.codec.compact.bindTopologyControl(&refs[idx], payload); } return refs; } @@ -3849,6 +3940,8 @@ fn buildGraphArtifactRefsForRepublishAlloc( const refs = try alloc.alloc(manifest_mod.ArtifactRef, 1); errdefer alloc.free(refs); refs[0] = try artifactRefFromMetadataAlloc(alloc, .graph_segment, artifact); + errdefer freeArtifactRef(alloc, refs[0]); + try graph_segment_mod.codec.compact.bindTopologyControl(&refs[0], payload); return refs; } return try alloc.alloc(manifest_mod.ArtifactRef, 0); @@ -3879,6 +3972,7 @@ fn buildGraphArtifactRefsForRepublishAlloc( for (graph_index_names, 0..) |index_name, idx| { refs[idx] = try artifactRefFromMetadataNamedAlloc(alloc, .graph_segment, index_name, artifact); initialized += 1; + try graph_segment_mod.codec.compact.bindTopologyControl(&refs[idx], payload); } return refs; } @@ -3934,6 +4028,109 @@ fn cloneNamedArtifactRefAlloc( return null; } +fn concatArtifactRefSlicesAlloc( + alloc: Allocator, + first: []const manifest_mod.ArtifactRef, + second: []const manifest_mod.ArtifactRef, +) ![]manifest_mod.ArtifactRef { + const total = std.math.add(usize, first.len, second.len) catch return error.OutOfMemory; + const refs = try alloc.alloc(manifest_mod.ArtifactRef, total); + @memcpy(refs[0..first.len], first); + @memcpy(refs[first.len..], second); + return refs; +} + +fn findArtifactRefByName( + refs: []const manifest_mod.ArtifactRef, + kind: manifest_mod.ArtifactKind, + name: []const u8, +) ?manifest_mod.ArtifactRef { + for (refs) |ref| if (ref.kind == kind and std.mem.eql(u8, ref.name, name)) return ref; + if (refs.len == 1 and refs[0].kind == kind) return refs[0]; + return null; +} + +fn stampGraphTopologyGenerations( + graph_refs: []manifest_mod.ArtifactRef, + current: ?manifest_mod.Manifest, + next_generation: u64, +) void { + for (graph_refs) |*graph_ref| { + graph_ref.edge_generation = next_generation; + const manifest = current orelse continue; + const previous_graph = findArtifactRefByName(manifest.artifacts, .graph_segment, graph_ref.name) orelse continue; + if (!artifactRefsIdentifySamePayload(previous_graph, graph_ref.*)) continue; + + if (previous_graph.edge_generation != 0) { + graph_ref.edge_generation = previous_graph.edge_generation; + continue; + } + + // An unstamped graph first acquires metric provenance now. + } +} + +fn buildGraphMetricArtifactRefsAlloc( + alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + current: ?manifest_mod.Manifest, + graph_refs: []manifest_mod.ArtifactRef, + specs: []const graph_metric_config.IndexSpec, + maintenance: ?maintenance_cancellation.Token, + provenance: lake_graph_metric.Provenance, + io: ?std.Io, + max_parallelism: usize, +) ![]manifest_mod.ArtifactRef { + var bridge = maintenance_cancellation.GraphBridge{ .maintenance = maintenance }; + const cancellation = bridge.token(); + var requests = std.ArrayListUnmanaged(lake_graph_metric.PublicationRequest).empty; + defer requests.deinit(alloc); + + // Metric-bearing publications pin a shared topology generation. + if (specs.len > 0) { + stampGraphTopologyGenerations(graph_refs, current, provenance.edge_generation); + } + + const graph_metric_limits = lake_graph_metric.Limits{}; + var graph_metric_budget = graph_metric_policy.Budget{ .limits = graph_metric_limits }; + for (specs) |spec| { + try cancellation.check(); + const graph_ref = findArtifactRefByName(graph_refs, .graph_segment, spec.index_name) orelse continue; + var effective_provenance = provenance; + effective_provenance.edge_generation = graph_ref.edge_generation; + + // Submit the complete desired plan. Shared prior-artifact resolution + // happens before any dirty computation consumes the build budget. + for (spec.configs) |config| { + var request = lake_graph_metric.PublicationRequest{ + .graph_index_name = spec.index_name, + .source_graph = graph_ref, + .config = config, + .provenance = effective_provenance, + }; + if (current) |manifest| { + const name = try graph_metric_segment_mod.artifactNameAlloc(alloc, spec.index_name, config.name); + defer alloc.free(name); + if (findNamedArtifactIndex(manifest, .graph_metric_segment, name)) |metric_index| { + const prior = manifest.artifacts[metric_index]; + request.prior_artifact = prior; + } + } + try requests.append(alloc, request); + } + } + return try lake_graph_metric.publishRequestsWithPriorAlloc(alloc, artifacts, requests.items, if (current) |manifest| manifest.artifacts else &.{}, cancellation, graph_metric_limits, &graph_metric_budget, .{ + .io = io, + .max_parallelism = if (io == null) 1 else max_parallelism, + }); +} + +fn artifactRefsIdentifySamePayload(lhs: manifest_mod.ArtifactRef, rhs: manifest_mod.ArtifactRef) bool { + return lhs.byte_len == rhs.byte_len and + std.mem.eql(u8, lhs.artifact_id, rhs.artifact_id) and + std.mem.eql(u8, lhs.checksum, rhs.checksum); +} + fn namedArtifactActionForName( planned_actions: []const publication_plan.NamedArtifactAction, name: []const u8, @@ -4092,21 +4289,19 @@ fn predictDerivedOutputAction( }; } -pub const GraphSegmentBuildResult = struct { - payload: ?[]u8, - edge_count: usize, -}; +pub const GraphSegmentBuildResult = struct { payload: ?[]u8, edge_count: usize }; -pub fn buildGraphSegmentAlloc( +/// Benchmark oracle only: the former string-expanded graph construction. +pub fn benchmarkReferenceGraphSegmentAlloc( alloc: Allocator, source_table: []const u8, docs: []const query_mod.QueryMaterializedDocument, include_graph: bool, ) !GraphSegmentBuildResult { - return try buildGraphSegmentAllocUntil(alloc, source_table, docs, include_graph, null); + return try benchmarkReferenceGraphSegmentAllocUntil(alloc, source_table, docs, include_graph, null); } -pub fn buildGraphSegmentAllocUntil( +fn benchmarkReferenceGraphSegmentAllocUntil( alloc: Allocator, source_table: []const u8, docs: []const query_mod.QueryMaterializedDocument, @@ -4166,6 +4361,51 @@ pub fn buildGraphSegmentAllocUntil( }; } +pub fn buildGraphSegmentAlloc( + alloc: Allocator, + source_table: []const u8, + docs: []const query_mod.QueryMaterializedDocument, + include_graph: bool, +) !GraphSegmentBuildResult { + return buildGraphSegmentAllocUntil(alloc, source_table, docs, include_graph, null); +} + +pub fn buildGraphSegmentAllocUntil( + alloc: Allocator, + source_table: []const u8, + docs: []const query_mod.QueryMaterializedDocument, + include_graph: bool, + maintenance: ?maintenance_cancellation.Token, +) !GraphSegmentBuildResult { + var bridge = maintenance_cancellation.GraphBridge{ .maintenance = maintenance }; + const cancellation = bridge.token(); + try cancellation.check(); + if (!include_graph) return .{ .payload = null, .edge_count = 0 }; + var builder = graph_segment_mod.Builder{ .alloc = alloc }; + defer builder.deinit(); + + for (docs) |doc| { + try cancellation.check(); + try builder.addNode(doc.doc_id); + const parsed_edges = try parseGraphEdgesAlloc(alloc, doc.body); + defer freeParsedGraphEdges(alloc, parsed_edges); + for (parsed_edges, 0..) |edge, i| { + if (i % 4096 == 0) try cancellation.check(); + const target_table = if (edge.target_table) |table| + if (std.mem.eql(u8, table, source_table)) null else table + else + null; + try builder.addEdge(doc.doc_id, edge.target, edge.edge_type, edge.weight, target_table); + } + } + + if (builder.edges.items.len == 0) return .{ .payload = null, .edge_count = 0 }; + return .{ + .payload = try builder.encodeAlloc(std.math.maxInt(usize), cancellation), + .edge_count = builder.edges.items.len, + }; +} + const ParsedGraphEdge = struct { target: []u8, edge_type: []u8, @@ -4188,7 +4428,10 @@ fn ensureNode(alloc: Allocator, node_map: *std.StringArrayHashMapUnmanaged(NodeE } fn parseGraphEdgesAlloc(alloc: Allocator, body: []const u8) ![]ParsedGraphEdge { - var parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch return try alloc.alloc(ParsedGraphEdge, 0); + var parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return try alloc.alloc(ParsedGraphEdge, 0), + }; defer parsed.deinit(); if (parsed.value != .object) return try alloc.alloc(ParsedGraphEdge, 0); const raw_edges = parsed.value.object.get("graph_edges") orelse return try alloc.alloc(ParsedGraphEdge, 0); @@ -4211,21 +4454,25 @@ fn parseGraphEdgesAlloc(alloc: Allocator, body: []const u8) ![]ParsedGraphEdge { const edge_type_value = item.object.get("edge_type"); const weight_value = item.object.get("weight"); const target_table_value = item.object.get("target_table"); + const owned_target = try alloc.dupe(u8, target_value.string); + errdefer alloc.free(owned_target); + const owned_type = try alloc.dupe(u8, if (edge_type_value != null and edge_type_value.? == .string) edge_type_value.?.string else ""); + errdefer alloc.free(owned_type); + const owned_table = if (target_table_value != null and target_table_value.? == .string and target_table_value.?.string.len > 0) + try alloc.dupe(u8, target_table_value.?.string) + else + null; + errdefer if (owned_table) |table| alloc.free(table); try out.append(alloc, .{ - .target = try alloc.dupe(u8, target_value.string), - .edge_type = if (edge_type_value != null and edge_type_value.? == .string) try alloc.dupe(u8, edge_type_value.?.string) else try alloc.dupe(u8, ""), + .target = owned_target, + .edge_type = owned_type, .weight = if (weight_value) |weight| switch (weight) { .float => @floatCast(weight.float), .integer => @floatFromInt(weight.integer), .number_string => std.fmt.parseFloat(f32, weight.number_string) catch 1.0, else => 1.0, } else 1.0, - .target_table = if (target_table_value != null and - target_table_value.? == .string and - target_table_value.?.string.len > 0) - try alloc.dupe(u8, target_table_value.?.string) - else - null, + .target_table = owned_table, }); } return try out.toOwnedSlice(alloc); @@ -4240,6 +4487,16 @@ fn freeParsedGraphEdges(alloc: Allocator, edges: []ParsedGraphEdge) void { alloc.free(edges); } +test "serverless graph builder parser propagates allocation failure without losing edges" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, struct { + fn run(alloc: Allocator) !void { + const edges = try parseGraphEdgesAlloc(alloc, "{\"graph_edges\":[{\"target\":\"b\",\"edge_type\":\"link\",\"target_table\":\"other\"}]}"); + defer freeParsedGraphEdges(alloc, edges); + try std.testing.expectEqual(@as(usize, 1), edges.len); + } + }.run, .{}); +} + fn sortParsedGraphEdges(edges: []ParsedGraphEdge) void { std.mem.sort(ParsedGraphEdge, edges, {}, lessParsedGraphEdge); } @@ -5842,6 +6099,277 @@ test "builder publishes named graph segments for graph indexes" { ); } +test "serverless builder warm starts changed graphs and cold starts when prior scores disappear" { + const alloc = std.testing.allocator; + var artifact_root_buf: [256]u8 = undefined; + var manifest_root_buf: [256]u8 = undefined; + var wal_root_buf: [256]u8 = undefined; + const artifact_root = tmpPath(&artifact_root_buf, "artifacts-graph-metric-seed"); + const manifest_root = tmpPath(&manifest_root_buf, "manifests-graph-metric-seed"); + const wal_root = tmpPath(&wal_root_buf, "wal-graph-metric-seed"); + defer cleanupTmp(artifact_root); + defer cleanupTmp(manifest_root); + defer cleanupTmp(wal_root); + + var fs_artifacts = try artifacts_mod.FsStore.init(alloc, std.mem.span(artifact_root)); + var artifact_store = fs_artifacts.artifactStore(); + defer artifact_store.deinit(); + var fs_manifests = try manifest_mod.FsStore.init(alloc, std.mem.span(manifest_root)); + var manifest_store = fs_manifests.manifestStore(); + defer manifest_store.deinit(); + var fs_wal = try wal_mod.FsStore.init(alloc, std.mem.span(wal_root)); + var wal_store = fs_wal.walStore(); + defer wal_store.deinit(); + var fs_progress = try catalog_mod.FsProgressStore.init(alloc, std.mem.span(manifest_root)); + var progress_store = fs_progress.progressStore(); + defer progress_store.deinit(); + + const plan = publication_plan.TablePublicationPlan{ + .targets = .{ + .published_search_sources = search_sources.defaultPublishedSearchSources(), + .include_graph = true, + }, + .table_definition = .{ .indexes_json = @constCast( + \\{"graph_idx":{"type":"graph","metrics":{"rank":{"kind":"pagerank","max_iterations":1}}}} + ) }, + }; + var builder = Builder.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store); + const metric_name = try graph_metric_segment_mod.artifactNameAlloc(alloc, "graph_idx", "rank"); + defer alloc.free(metric_name); + for ([_][]const u8{ "doc-a", "doc-c", "doc-d" }, 0..) |node, round| { + const mutation = try api_codec.encodeMutationAlloc(alloc, .{ + .kind = .upsert, + .doc_id = node, + .body = + \\{"graph_edges":[{"target":"doc-b","edge_type":"cites"}]} + , + }); + defer alloc.free(mutation); + _ = try wal_store.append("docs", @intCast(100 * (round + 1)), mutation); + var result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, plan); + defer result.deinit(alloc); + try std.testing.expect(result.published); + var runtime = query_mod.QueryRuntime.init(alloc, &artifact_store, &manifest_store, &progress_store); + defer runtime.deinit(); + var session = try runtime.openHeadSession("docs"); + defer session.deinit(); + var top = try query_mod.graphMetricTopAlloc(alloc, &session, "graph_idx", "rank", 10); + defer top.deinit(alloc); + try std.testing.expectEqual(round + 2, top.scores.len); + try std.testing.expectEqualStrings("doc-b", top.scores[0].node_id); + // One iteration makes use of the prior seed observable. Round 1 maps + // [0.2875, 0.7125] onto [a, b, c], giving c zero initial mass. + const expected_top: f64 = switch (round) { + 0 => 0.7125, + 1 => 0.49625, + else => 0.728125, + }; + try std.testing.expectApproxEqAbs(expected_top, top.scores[0].value, 0.0000001); + if (round == 1) { + var manifest = try manifest_store.getAlloc("docs", 2); + defer manifest.deinit(alloc); + const prior = manifest.artifacts[findNamedArtifactIndex(manifest, .graph_metric_segment, metric_name).?]; + // Missing optional acceleration cannot block the next authoritative + // graph publication; round 2 must use the four-node cold seed. + try artifact_store.delete(prior.artifact_id); + } + } +} + +test "serverless builder publishes and lifecycle-binds configured graph metrics" { + const alloc = std.testing.allocator; + var artifact_root_buf: [256]u8 = undefined; + var manifest_root_buf: [256]u8 = undefined; + var wal_root_buf: [256]u8 = undefined; + const artifact_root = tmpPath(&artifact_root_buf, "artifacts-graph-metric-lifecycle"); + const manifest_root = tmpPath(&manifest_root_buf, "manifests-graph-metric-lifecycle"); + const wal_root = tmpPath(&wal_root_buf, "wal-graph-metric-lifecycle"); + defer cleanupTmp(artifact_root); + defer cleanupTmp(manifest_root); + defer cleanupTmp(wal_root); + + var fs_artifacts = try artifacts_mod.FsStore.init(alloc, std.mem.span(artifact_root)); + var artifact_store = fs_artifacts.artifactStore(); + defer artifact_store.deinit(); + var fs_manifests = try manifest_mod.FsStore.init(alloc, std.mem.span(manifest_root)); + var manifest_store = fs_manifests.manifestStore(); + defer manifest_store.deinit(); + var fs_wal = try wal_mod.FsStore.init(alloc, std.mem.span(wal_root)); + var wal_store = fs_wal.walStore(); + defer wal_store.deinit(); + var fs_progress = try catalog_mod.FsProgressStore.init(alloc, std.mem.span(manifest_root)); + var progress_store = fs_progress.progressStore(); + defer progress_store.deinit(); + + const indexes_json = try alloc.dupe(u8, + \\{"graph_idx":{"type":"graph","metrics":{"degree":{"kind":"degree"},"rank":{"kind":"pagerank","max_iterations":20}}}} + ); + defer alloc.free(indexes_json); + const plan = publication_plan.TablePublicationPlan{ + .targets = .{ + .published_search_sources = search_sources.defaultPublishedSearchSources(), + .include_graph = true, + }, + .table_definition = .{ .indexes_json = indexes_json }, + }; + var builder = Builder.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store); + + const first_mutation = try api_codec.encodeMutationAlloc(alloc, .{ + .kind = .upsert, + .doc_id = "doc-a", + .body = "{\"text\":\"one\",\"graph_edges\":[{\"target\":\"doc-b\",\"edge_type\":\"cites\"}]}", + }); + defer alloc.free(first_mutation); + _ = try wal_store.append("docs", 100, first_mutation); + var first_result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, plan); + defer first_result.deinit(alloc); + var first = try manifest_store.getAlloc("docs", 1); + defer first.deinit(alloc); + try std.testing.expectEqual(@as(u32, 1), first.stats.graph_segment_count); + const metric_name = try graph_metric_segment_mod.artifactNameAlloc(alloc, "graph_idx", "rank"); + defer alloc.free(metric_name); + const degree_metric_name = try graph_metric_segment_mod.artifactNameAlloc(alloc, "graph_idx", "degree"); + defer alloc.free(degree_metric_name); + const first_graph = first.artifacts[findNamedArtifactIndex(first, .graph_segment, "graph_idx").?]; + try std.testing.expect(!std.mem.eql(u8, &first_graph.graph_topology_control_checksum, &@as([32]u8, @splat(0)))); + const first_metric = first.artifacts[findNamedArtifactIndex(first, .graph_metric_segment, metric_name).?]; + try std.testing.expectEqual(@as(u64, 1), first_graph.edge_generation); + try std.testing.expectEqual(graph_metric_segment_mod.wire_version, first_metric.metadata_version); + try std.testing.expectEqual(@as(u64, 1), first_metric.published_generation); + try std.testing.expectEqual(@as(u64, 1), first_metric.edge_generation); + try std.testing.expectEqual(lake_graph_metric.materializerFingerprint(.{}), first_metric.materializer_fingerprint); + try std.testing.expect(findNamedArtifactIndex(first, .graph_metric_segment, degree_metric_name) != null); + try artifact_store.delete(first_metric.artifact_id); + + const unchanged_graph_mutation = try api_codec.encodeMutationAlloc(alloc, .{ + .kind = .upsert, + .doc_id = "doc-a", + .body = "{\"text\":\"two\",\"graph_edges\":[{\"target\":\"doc-b\",\"edge_type\":\"cites\"}]}", + }); + defer alloc.free(unchanged_graph_mutation); + _ = try wal_store.append("docs", 200, unchanged_graph_mutation); + var second_result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, plan); + defer second_result.deinit(alloc); + var second = try manifest_store.getAlloc("docs", 2); + defer second.deinit(alloc); + const second_graph = second.artifacts[findNamedArtifactIndex(second, .graph_segment, "graph_idx").?]; + try std.testing.expectEqualSlices(u8, &first_graph.graph_topology_control_checksum, &second_graph.graph_topology_control_checksum); + const second_metric = second.artifacts[findNamedArtifactIndex(second, .graph_metric_segment, metric_name).?]; + try std.testing.expectEqualStrings(first_graph.artifact_id, second_graph.artifact_id); + try std.testing.expectEqual(first_graph.edge_generation, second_graph.edge_generation); + try std.testing.expectEqualStrings(first_metric.artifact_id, second_metric.artifact_id); + try std.testing.expectEqual(first_metric.metadata_version, second_metric.metadata_version); + // The missing payload was recomputed; only unchanged, reusable artifacts + // retain their previous publication/computation provenance. + try std.testing.expectEqual(@as(u64, 2), second_metric.published_generation); + try std.testing.expectEqual(first_metric.edge_generation, second_metric.edge_generation); + try std.testing.expect(second_metric.computed_at_ms >= first_metric.computed_at_ms); + const first_degree = first.artifacts[findNamedArtifactIndex(first, .graph_metric_segment, degree_metric_name).?]; + const second_degree = second.artifacts[findNamedArtifactIndex(second, .graph_metric_segment, degree_metric_name).?]; + try std.testing.expectEqualStrings(first_degree.artifact_id, second_degree.artifact_id); + try std.testing.expectEqual(first_degree.published_generation, second_degree.published_generation); + try std.testing.expectEqual(first_degree.computed_at_ms, second_degree.computed_at_ms); + var restored_metric = try artifact_store.stat(second_metric.artifact_id); + restored_metric.deinit(alloc); + + const changed_graph_mutation = try api_codec.encodeMutationAlloc(alloc, .{ + .kind = .upsert, + .doc_id = "doc-a", + .body = "{\"text\":\"three\",\"graph_edges\":[{\"target\":\"doc-c\",\"edge_type\":\"cites\"}]}", + }); + defer alloc.free(changed_graph_mutation); + _ = try wal_store.append("docs", 300, changed_graph_mutation); + var third_result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, plan); + defer third_result.deinit(alloc); + var third = try manifest_store.getAlloc("docs", 3); + defer third.deinit(alloc); + const third_graph = third.artifacts[findNamedArtifactIndex(third, .graph_segment, "graph_idx").?]; + try std.testing.expect(!std.mem.eql(u8, &third_graph.graph_topology_control_checksum, &@as([32]u8, @splat(0)))); + const third_metric = third.artifacts[findNamedArtifactIndex(third, .graph_metric_segment, metric_name).?]; + const third_degree_metric = third.artifacts[findNamedArtifactIndex(third, .graph_metric_segment, degree_metric_name).?]; + try std.testing.expect(!std.mem.eql(u8, second_graph.artifact_id, third_graph.artifact_id)); + try std.testing.expectEqual(@as(u64, 3), third_graph.edge_generation); + try std.testing.expect(!std.mem.eql(u8, second_metric.artifact_id, third_metric.artifact_id)); + try std.testing.expectEqual(@as(u64, 3), third_metric.published_generation); + try std.testing.expectEqual(@as(u64, 3), third_metric.edge_generation); + + var runtime = query_mod.QueryRuntime.init(alloc, &artifact_store, &manifest_store, &progress_store); + defer runtime.deinit(); + var session = try runtime.openHeadSession("docs"); + defer session.deinit(); + var top = try query_mod.graphMetricTopAlloc(alloc, &session, "graph_idx", "rank", 2); + defer top.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), top.scores.len); + try std.testing.expect(top.scores[0].value >= top.scores[1].value); + try std.testing.expectEqual(@as(u64, 3), top.published_generation); + + const payload = try artifact_store.getVerifiedAllocWithCancellationUsingAllocator(alloc, third_metric.artifact_id, third_metric.byte_len, third_metric.checksum, .none); + defer alloc.free(payload); + var decoded = try graph_metric_segment_mod.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqualStrings(third_graph.artifact_id, decoded.source_graph_artifact_id); + try std.testing.expectEqualStrings(third_graph.checksum, decoded.source_graph_checksum); + + const updated_indexes_json = try alloc.dupe(u8, + \\{"graph_idx":{"type":"graph","metrics":{"degree":{"kind":"degree"},"rank":{"kind":"pagerank","max_iterations":40}}}} + ); + defer alloc.free(updated_indexes_json); + var fourth_result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, .{ + .targets = plan.targets, + .table_definition = .{ .indexes_json = updated_indexes_json }, + .metadata_republish = .{ .artifact_families_changed = true }, + .artifact_actions = .{ .document_segment = .reuse, .full_text = .reuse, .dense_vector = .reuse, .sparse_vector = .reuse, .graph = .reuse }, + }); + defer fourth_result.deinit(alloc); + var fourth = try manifest_store.getAlloc("docs", 4); + defer fourth.deinit(alloc); + const fourth_graph = fourth.artifacts[findNamedArtifactIndex(fourth, .graph_segment, "graph_idx").?]; + const fourth_metric = fourth.artifacts[findNamedArtifactIndex(fourth, .graph_metric_segment, metric_name).?]; + const fourth_degree_metric = fourth.artifacts[findNamedArtifactIndex(fourth, .graph_metric_segment, degree_metric_name).?]; + try std.testing.expectEqualStrings(third_graph.artifact_id, fourth_graph.artifact_id); + try std.testing.expectEqual(third_graph.edge_generation, fourth_graph.edge_generation); + try std.testing.expect(!std.mem.eql(u8, third_metric.artifact_id, fourth_metric.artifact_id)); + try std.testing.expectEqual(@as(u64, 4), fourth_metric.published_generation); + try std.testing.expectEqual(third_metric.edge_generation, fourth_metric.edge_generation); + try std.testing.expectEqualStrings(third_degree_metric.artifact_id, fourth_degree_metric.artifact_id); + try std.testing.expectEqual(third_degree_metric.published_generation, fourth_degree_metric.published_generation); + try std.testing.expectEqual(third_degree_metric.edge_generation, fourth_degree_metric.edge_generation); + try std.testing.expectEqual(third_degree_metric.computed_at_ms, fourth_degree_metric.computed_at_ms); + + const expanded_indexes_json = try alloc.dupe(u8, + \\{"graph_idx":{"type":"graph","metrics":{"degree":{"kind":"degree"},"rank":{"kind":"pagerank","max_iterations":40},"centrality":{"kind":"eigenvector"},"degree_alias":{"kind":"degree"}}}} + ); + defer alloc.free(expanded_indexes_json); + var fifth_result = try builder.publishNamespaceWithMetricAndPlan("docs", .cosine, .{ + .targets = plan.targets, + .table_definition = .{ .indexes_json = expanded_indexes_json }, + .metadata_republish = .{ .artifact_families_changed = true }, + .artifact_actions = .{ .document_segment = .reuse, .full_text = .reuse, .dense_vector = .reuse, .sparse_vector = .reuse, .graph = .reuse }, + }); + defer fifth_result.deinit(alloc); + var fifth = try manifest_store.getAlloc("docs", 5); + defer fifth.deinit(alloc); + const centrality_metric_name = try graph_metric_segment_mod.artifactNameAlloc(alloc, "graph_idx", "centrality"); + defer alloc.free(centrality_metric_name); + const fifth_graph = fifth.artifacts[findNamedArtifactIndex(fifth, .graph_segment, "graph_idx").?]; + const fifth_metric = fifth.artifacts[findNamedArtifactIndex(fifth, .graph_metric_segment, metric_name).?]; + const fifth_degree_metric = fifth.artifacts[findNamedArtifactIndex(fifth, .graph_metric_segment, degree_metric_name).?]; + const fifth_centrality_metric = fifth.artifacts[findNamedArtifactIndex(fifth, .graph_metric_segment, centrality_metric_name).?]; + try std.testing.expectEqualStrings(fourth_graph.artifact_id, fifth_graph.artifact_id); + try std.testing.expectEqual(fourth_graph.edge_generation, fifth_graph.edge_generation); + try std.testing.expectEqualStrings(fourth_metric.artifact_id, fifth_metric.artifact_id); + try std.testing.expectEqualStrings(fourth_degree_metric.artifact_id, fifth_degree_metric.artifact_id); + try std.testing.expectEqual(@as(u64, 5), fifth_centrality_metric.published_generation); + try std.testing.expectEqual(fourth_graph.edge_generation, fifth_centrality_metric.edge_generation); + const alias_name = try graph_metric_segment_mod.artifactNameAlloc(alloc, "graph_idx", "degree_alias"); + defer alloc.free(alias_name); + const alias = fifth.artifacts[findNamedArtifactIndex(fifth, .graph_metric_segment, alias_name).?]; + try std.testing.expectEqualStrings(fourth_degree_metric.artifact_id, alias.artifact_id); + try std.testing.expectEqual(fourth_degree_metric.computed_at_ms, alias.computed_at_ms); + try std.testing.expectEqual(@as(u64, 5), alias.published_generation); + try std.testing.expectEqual(fifth_graph.edge_generation, alias.edge_generation); +} + test "builder reuses graph artifact when wal updates do not change graph projection" { const alloc = std.testing.allocator; diff --git a/zig/pkg/antfly/src/serverless/build/coordinator.zig b/zig/pkg/antfly/src/serverless/build/coordinator.zig index 49fa3de80c..34b558066a 100644 --- a/zig/pkg/antfly/src/serverless/build/coordinator.zig +++ b/zig/pkg/antfly/src/serverless/build/coordinator.zig @@ -15,6 +15,7 @@ const std = @import("std"); const platform_sync = @import("antfly_platform").sync; const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const catalog_service = @import("../catalog/service.zig"); const work_lease = @import("work_lease.zig"); const maintenance_cancellation = @import("../maintenance_cancellation.zig"); @@ -127,6 +128,14 @@ pub const BackgroundPublisher = struct { .io = self.io, .requested = cancel_requested, }; + return self.runOnceWithToken(cancellation); + } + + pub fn runOnceWithCancellation(self: *BackgroundPublisher, cancellation: CancellationToken) !PublishRunStats { + return self.runOnceWithToken(.{ .io = self.io, .requested = &self.stop_requested, .cooperative = cancellation }); + } + + fn runOnceWithToken(self: *BackgroundPublisher, cancellation: maintenance_cancellation.Token) !PublishRunStats { const namespaces = try self.catalog.listNamespacesAlloc(self.alloc); defer self.catalog.freeNamespaces(self.alloc, namespaces); diff --git a/zig/pkg/antfly/src/serverless/build/external_source_manifest.zig b/zig/pkg/antfly/src/serverless/build/external_source_manifest.zig index 4fd0a516ec..343541f783 100644 --- a/zig/pkg/antfly/src/serverless/build/external_source_manifest.zig +++ b/zig/pkg/antfly/src/serverless/build/external_source_manifest.zig @@ -27,6 +27,22 @@ pub const PublishedArtifact = struct { byte_len: u64, checksum: []const u8, name: []const u8 = &.{}, + metadata_version: u16 = 0, + published_generation: u64 = 0, + edge_generation: u64 = 0, + computed_at_ms: u64 = 0, + materializer_fingerprint: u64 = 0, + graph_topology_control_checksum: [32]u8 = @splat(0), + graph_metric_control_len: u32 = 0, + graph_metric_routing_footer_len: u32 = 0, + graph_metric_control_checksum: [32]u8 = @splat(0), + graph_metric_routing_checksum: [32]u8 = @splat(0), + graph_metric_point_index_checksum: [32]u8 = @splat(0), + graph_metric_config_fingerprint: u64 = 0, + graph_metric_source_checksum: [32]u8 = @splat(0), + graph_metric_topology_checksum: [32]u8 = @splat(0), + graph_metric_materialization_state: manifest_artifact.GraphMetricMaterializationState = .ready, + graph_metric_rejection_reason: manifest_artifact.GraphMetricRejectionReason = .none, }; pub const Plan = struct { @@ -189,6 +205,22 @@ fn cloneArtifactRef( .artifact_id = try alloc.dupe(u8, artifact.artifact_id), .byte_len = artifact.byte_len, .checksum = try alloc.dupe(u8, artifact.checksum), + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }; } @@ -210,6 +242,22 @@ fn cloneAppendedArtifactsAlloc( .byte_len = artifact.byte_len, .checksum = artifact.checksum, .name = artifact.name, + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }); initialized += 1; } @@ -219,6 +267,22 @@ fn cloneAppendedArtifactsAlloc( .byte_len = artifact.byte_len, .checksum = artifact.checksum, .name = artifact.name, + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }); initialized += 1; } diff --git a/zig/pkg/antfly/src/serverless/build/graph_metric_config.zig b/zig/pkg/antfly/src/serverless/build/graph_metric_config.zig new file mode 100644 index 0000000000..680e9616b1 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/build/graph_metric_config.zig @@ -0,0 +1,239 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the Elastic License 2.0 is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See +// the Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Minimal, storage-independent graph metric configuration parsing for the +//! serverless publication path. Keeping this beside the builder avoids pulling +//! LMDB-backed catalog code into official serverless builds. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const graph_mod = @import("../../graph/graph.zig"); +const graph_metric_policy = @import("graph_metric_policy.zig"); + +pub const IndexSpec = struct { + index_name: []u8, + configs: []graph_mod.GraphMetricConfig, + + pub fn deinit(self: *IndexSpec, alloc: Allocator) void { + alloc.free(self.index_name); + graph_mod.freeGraphMetricConfigs(alloc, self.configs); + self.* = undefined; + } +}; + +pub fn freeIndexSpecs(alloc: Allocator, specs: []IndexSpec) void { + for (specs) |*spec| spec.deinit(alloc); + if (specs.len > 0) alloc.free(specs); +} + +pub fn parseIndexSpecsAlloc(alloc: Allocator, indexes_json: []const u8) ![]IndexSpec { + if (indexes_json.len == 0) return try alloc.alloc(IndexSpec, 0); + var parsed = std.json.parseFromSlice(std.json.Value, alloc, indexes_json, .{}) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.InvalidIndexConfig, + }; + defer parsed.deinit(); + if (parsed.value != .object) return error.InvalidIndexConfig; + + var specs = std.ArrayListUnmanaged(IndexSpec).empty; + var total_metric_count: usize = 0; + errdefer { + for (specs.items) |*spec| spec.deinit(alloc); + specs.deinit(alloc); + } + var it = parsed.value.object.iterator(); + while (it.next()) |entry| { + const value = entry.value_ptr.*; + if (value != .object) return error.InvalidIndexConfig; + const type_value = value.object.get("type") orelse continue; + if (type_value != .string) return error.InvalidIndexConfig; + if (!std.mem.eql(u8, type_value.string, "graph")) continue; + const configs = try parseMetricConfigsAlloc(alloc, value); + var configs_moved = false; + errdefer if (!configs_moved) graph_mod.freeGraphMetricConfigs(alloc, configs); + if (configs.len == 0) { + graph_mod.freeGraphMetricConfigs(alloc, configs); + configs_moved = true; + continue; + } + const limits = graph_metric_policy.Limits{}; + if (entry.key_ptr.*.len == 0 or entry.key_ptr.*.len > limits.max_graph_index_name_bytes) { + return error.GraphMetricConfigurationLimitExceeded; + } + total_metric_count = std.math.add(usize, total_metric_count, configs.len) catch + return error.GraphMetricConfigurationLimitExceeded; + try graph_metric_policy.validateCatalogFanout(specs.items.len + 1, total_metric_count, limits); + const index_name = try alloc.dupe(u8, entry.key_ptr.*); + var spec = IndexSpec{ .index_name = index_name, .configs = configs }; + configs_moved = true; + var moved = false; + errdefer if (!moved) spec.deinit(alloc); + try specs.append(alloc, spec); + moved = true; + } + std.mem.sort(IndexSpec, specs.items, {}, struct { + fn lessThan(_: void, a: IndexSpec, b: IndexSpec) bool { + return std.mem.lessThan(u8, a.index_name, b.index_name); + } + }.lessThan); + return try specs.toOwnedSlice(alloc); +} + +pub fn parseMetricConfigsAlloc(alloc: Allocator, index: std.json.Value) ![]graph_mod.GraphMetricConfig { + const metrics = index.object.get("metrics") orelse return try alloc.alloc(graph_mod.GraphMetricConfig, 0); + if (metrics != .object) return error.InvalidIndexConfig; + var configs = std.ArrayListUnmanaged(graph_mod.GraphMetricConfig).empty; + errdefer { + for (configs.items) |*cfg| { + alloc.free(cfg.name); + cfg.edge_filter.deinit(alloc); + } + configs.deinit(alloc); + } + var it = metrics.object.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.* != .object) return error.InvalidIndexConfig; + const object = entry.value_ptr.*.object; + const enabled = if (object.get("enabled")) |value| blk: { + if (value != .bool) return error.InvalidIndexConfig; + break :blk value.bool; + } else true; + if (!enabled) continue; + + const name = entry.key_ptr.*; + const kind = if (object.get("kind")) |value| try parseKind(value) else try parseKind(.{ .string = name }); + const refresh = if (object.get("refresh")) |value| blk: { + if (value != .string) return error.InvalidIndexConfig; + if (std.mem.eql(u8, value.string, "background")) break :blk graph_mod.GraphMetricRefreshMode.background; + // Serverless publication has no synchronous refresh endpoint. Do + // not accept a mode whose operational contract cannot be honored. + if (std.mem.eql(u8, value.string, "manual")) return error.UnsupportedGraphMetricRefreshMode; + return error.InvalidIndexConfig; + } else .background; + const damping = if (object.get("damping")) |value| try numberAsF64(value) else 0.85; + const tolerance = if (object.get("tolerance")) |value| try numberAsF64(value) else 0.000001; + const max_iterations = if (object.get("max_iterations")) |value| try numberAsU32(value) else 50; + if (!(damping > 0 and damping < 1) or !(tolerance > 0) or max_iterations == 0 or max_iterations > graph_mod.graph_metric_max_iterations) return error.InvalidIndexConfig; + const owned_name = try alloc.dupe(u8, name); + var owned_name_moved = false; + errdefer if (!owned_name_moved) alloc.free(owned_name); + var edge_filter = try parseEdgeFilterAlloc(alloc, object.get("edge_filter")); + var edge_filter_moved = false; + errdefer if (!edge_filter_moved) edge_filter.deinit(alloc); + var config = graph_mod.GraphMetricConfig{ + .name = owned_name, + .kind = kind, + .damping = damping, + .tolerance = tolerance, + .max_iterations = max_iterations, + .refresh = refresh, + .edge_filter = edge_filter, + }; + var moved = false; + errdefer if (!moved) { + alloc.free(config.name); + config.edge_filter.deinit(alloc); + }; + owned_name_moved = true; + edge_filter_moved = true; + try configs.append(alloc, config); + moved = true; + } + std.mem.sort(graph_mod.GraphMetricConfig, configs.items, {}, struct { + fn lessThan(_: void, a: graph_mod.GraphMetricConfig, b: graph_mod.GraphMetricConfig) bool { + return std.mem.lessThan(u8, a.name, b.name); + } + }.lessThan); + try graph_mod.validateGraphMetricEdgeFilters(&.{}, configs.items); + try graph_metric_policy.validateConfigs(configs.items, .{}); + return try configs.toOwnedSlice(alloc); +} + +fn parseKind(value: std.json.Value) !graph_mod.GraphMetricKind { + if (value != .string) return error.InvalidIndexConfig; + inline for (std.meta.fields(graph_mod.GraphMetricKind)) |field| { + if (std.mem.eql(u8, value.string, field.name)) return @enumFromInt(field.value); + } + return error.InvalidIndexConfig; +} + +fn parseEdgeFilterAlloc(alloc: Allocator, maybe_value: ?std.json.Value) !graph_mod.GraphMetricEdgeFilter { + const value = maybe_value orelse return .{}; + if (value != .object) return error.InvalidIndexConfig; + if (value.object.get("types")) |types| { + if (types != .array or types.array.items.len == 0) return error.InvalidIndexConfig; + const out = try alloc.alloc([]const u8, types.array.items.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |item| alloc.free(item); + alloc.free(out); + } + for (types.array.items, 0..) |item, i| { + if (item != .string or item.string.len == 0) return error.InvalidIndexConfig; + out[i] = try alloc.dupe(u8, item.string); + initialized += 1; + } + return .{ .mode = .types, .types = out }; + } + if (value.object.get("mode")) |mode| { + if (mode != .string or !std.mem.eql(u8, mode.string, "all")) return error.InvalidIndexConfig; + } + return .{}; +} + +fn numberAsF64(value: std.json.Value) !f64 { + const result: f64 = switch (value) { + .integer => |number| @floatFromInt(number), + .float => |number| number, + else => return error.InvalidIndexConfig, + }; + if (!std.math.isFinite(result)) return error.InvalidIndexConfig; + return result; +} + +fn numberAsU32(value: std.json.Value) !u32 { + return switch (value) { + .integer => |number| if (number > 0 and number <= std.math.maxInt(u32)) @intCast(number) else error.InvalidIndexConfig, + .float => |number| if (std.math.isFinite(number) and number > 0 and number <= std.math.maxInt(u32) and @floor(number) == number) @intFromFloat(number) else error.InvalidIndexConfig, + else => error.InvalidIndexConfig, + }; +} + +test "serverless graph metric configs are deterministic and honor disabled metrics" { + const specs = try parseIndexSpecsAlloc(std.testing.allocator, + \\{"z":{"type":"graph","metrics":{"degree":{"enabled":false},"rank":{"kind":"pagerank","edge_filter":{"types":["cites"]}}}},"a":{"type":"text"}} + ); + defer freeIndexSpecs(std.testing.allocator, specs); + try std.testing.expectEqual(@as(usize, 1), specs.len); + try std.testing.expectEqualStrings("z", specs[0].index_name); + try std.testing.expectEqual(@as(usize, 1), specs[0].configs.len); + try std.testing.expectEqualStrings("rank", specs[0].configs[0].name); + try std.testing.expect(specs[0].configs[0].edge_filter.includesType("cites")); + + const AllocationRunner = struct { + fn run(alloc: Allocator, json: []const u8) !void { + const parsed_specs = try parseIndexSpecsAlloc(alloc, json); + defer freeIndexSpecs(alloc, parsed_specs); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, AllocationRunner.run, .{ + "{\"graph\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"edge_filter\":{\"types\":[\"cites\",\"mentions\"]}}}}}", + }); +} + +test "serverless graph metric configs reject manual refresh" { + try std.testing.expectError(error.UnsupportedGraphMetricRefreshMode, parseIndexSpecsAlloc(std.testing.allocator, + \\{"graph":{"type":"graph","metrics":{"rank":{"kind":"pagerank","refresh":"manual"}}}} + )); +} diff --git a/zig/pkg/antfly/src/serverless/build/graph_metric_policy.zig b/zig/pkg/antfly/src/serverless/build/graph_metric_policy.zig new file mode 100644 index 0000000000..f0ac9ca256 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/build/graph_metric_policy.zig @@ -0,0 +1,306 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Serverless graph-metric admission and materializer compatibility policy. +//! Keep this independent of persistence so configuration validation, builders, +//! and reuse checks all enforce exactly the same bounded contract. + +const std = @import("std"); +const graph_mod = @import("../../graph/graph.zig"); +const metric_cost = @import("../../graph/metric_cost.zig"); +const bounded_decode = @import("../bounded_decode.zig"); + +/// Increment whenever an implementation change can alter admission or output +/// without changing the user-visible metric configuration. +// Addressable topology changes cold read/work admission, not metric semantics. +pub const materializer_epoch: u32 = 23; +const max_tracked_graph_indexes: usize = 16; + +pub const Limits = struct { + // Keep graph admission aligned with the decoder and query-runtime + // artifact contract. Larger topology artifacts cannot be served safely by + // this runtime and are represented by durable rejected metric sidecars. + max_graph_payload_bytes: usize = (bounded_decode.Limits{}).max_artifact_bytes, + // Publication charges actual topology ranges, including authenticated block + // alignment. Unbound controls require full-content authentication; explicit + // source-wide preparation reserves each unique complete payload once. + // Excess work receives durable rejection sidecars, not unbounded I/O. + max_total_graph_payload_bytes: usize = 512 * 1024 * 1024, + max_metric_payload_bytes: usize = 256 * 1024 * 1024, + max_total_metric_payload_bytes: usize = 512 * 1024 * 1024, + // Optional acceleration must not starve a later cold materialization. + // Zero disables warm starts. Charge every read, not just unique identities: + // seeds are not retained across materializations. + max_total_seed_payload_bytes: u64 = 64 * 1024 * 1024, + max_total_seed_work_items: u64 = 64 * 1024 * 1024, + // Reuse authentication is independent of optional warm-start inputs. + // Includes cold full-content verification and requested control ranges. + max_total_reuse_read_bytes: u64 = 512 * 1024 * 1024, + /// Optional semantic-reuse hashing is byte-accounted independently of + /// cold projection/kernel work, so acceleration cannot starve a rebuild. + max_total_identity_work_bytes: u64 = 1024 * 1024 * 1024, + // Bounds the decoded topology, compiled projection, dense kernel vectors, + // borrowed sortable score views, and encoded output that may coexist for one + // materialization. Work and payload limits alone do not bound this peak. + max_peak_memory_bytes: usize = 1024 * 1024 * 1024, + max_nodes: usize = 1_000_000, + max_edges: usize = 10_000_000, + max_work_items: u64 = 500_000_000, + max_total_work_items: u64 = 1_000_000_000, + max_graph_indexes: usize = 16, + max_metrics: usize = 16, + max_total_metrics: usize = 64, + max_graph_index_name_bytes: usize = 256, + max_edge_filter_types: usize = 64, + max_metric_name_bytes: usize = 128, + max_edge_type_bytes: usize = 256, +}; + +pub const Budget = struct { + limits: Limits, + work_items: u64 = 0, + metric_payload_bytes: usize = 0, + graph_payload_bytes: usize = 0, + seed_payload_bytes: u64 = 0, + seed_work_items: u64 = 0, + reuse_read_bytes: u64 = 0, + identity_work_bytes: u64 = 0, + graph_identity_count: usize = 0, + graph_identities: [max_tracked_graph_indexes][32]u8 = undefined, + + /// Reserve optional work atomically. One byte of prior input is a + /// conservative decode-work unit, including authentication and ID parsing; + /// mapping also visits the current vector. Rejection leaves the cold budget + /// and both seed counters untouched. + pub fn admitSeed(self: *Budget, byte_len: u64, node_count: usize) bool { + const bytes = std.math.add(u64, self.seed_payload_bytes, byte_len) catch return false; + const work = std.math.add(u64, byte_len, node_count) catch return false; + const total = std.math.add(u64, self.seed_work_items, work) catch return false; + if (bytes > self.limits.max_total_seed_payload_bytes or total > self.limits.max_total_seed_work_items) return false; + self.seed_payload_bytes = bytes; + self.seed_work_items = total; + return true; + } + + /// Charges one immutable topology identity at most once. The digest is + /// bookkeeping only: artifact authentication remains the responsibility + /// of ArtifactStore before decoding. + pub fn chargeGraphPayload(self: *Budget, artifact_id: []const u8, checksum: []const u8, byte_len: u64) !void { + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + hasher.update(artifact_id); + hasher.update(&.{0}); + hasher.update(checksum); + var encoded_len: [8]u8 = undefined; + std.mem.writeInt(u64, &encoded_len, byte_len, .little); + hasher.update(&encoded_len); + var identity: [32]u8 = undefined; + hasher.final(&identity); + for (self.graph_identities[0..self.graph_identity_count]) |existing| { + if (std.mem.eql(u8, &existing, &identity)) return; + } + if (self.graph_identity_count >= self.limits.max_graph_indexes or + self.graph_identity_count >= self.graph_identities.len) + { + return error.GraphMetricBuildBudgetExceeded; + } + const admitted_bytes = std.math.cast(usize, byte_len) orelse return error.GraphMetricBuildBudgetExceeded; + const next = std.math.add(usize, self.graph_payload_bytes, admitted_bytes) catch + return error.GraphMetricBuildBudgetExceeded; + if (next > self.limits.max_total_graph_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + self.graph_identities[self.graph_identity_count] = identity; + self.graph_identity_count += 1; + self.graph_payload_bytes = next; + } + + pub fn chargeWork(self: *Budget, amount: u64) !void { + const next = std.math.add(u64, self.work_items, amount) catch + return error.GraphMetricBuildBudgetExceeded; + if (next > self.limits.max_total_work_items) return error.GraphMetricBuildBudgetExceeded; + self.work_items = next; + } + + pub fn chargePayload(self: *Budget, amount: usize) !void { + const next = std.math.add(usize, self.metric_payload_bytes, amount) catch + return error.GraphMetricBuildBudgetExceeded; + if (next > self.limits.max_total_metric_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + self.metric_payload_bytes = next; + } +}; + +pub fn validateLimits(limits: Limits) !void { + if (limits.max_graph_payload_bytes == 0 or + limits.max_graph_payload_bytes > (bounded_decode.Limits{}).max_artifact_bytes or + limits.max_total_graph_payload_bytes == 0 or + limits.max_metric_payload_bytes == 0 or + limits.max_total_metric_payload_bytes == 0 or + limits.max_peak_memory_bytes == 0 or + limits.max_nodes == 0 or + limits.max_edges == 0 or + limits.max_work_items == 0 or + limits.max_total_work_items == 0 or + limits.max_graph_indexes == 0 or limits.max_graph_indexes > max_tracked_graph_indexes or + limits.max_metrics == 0 or + limits.max_total_metrics == 0 or + limits.max_graph_index_name_bytes == 0 or + limits.max_edge_filter_types == 0 or + limits.max_metric_name_bytes == 0 or + limits.max_edge_type_bytes == 0) + { + return error.InvalidGraphMetricBuildOptions; + } +} + +pub fn validateCatalogFanout(graph_index_count: usize, metric_count: usize, limits: Limits) !void { + try validateLimits(limits); + if (graph_index_count > limits.max_graph_indexes or metric_count > limits.max_total_metrics) { + return error.GraphMetricConfigurationLimitExceeded; + } +} + +pub fn validateConfigs(configs: []const graph_mod.GraphMetricConfig, limits: Limits) !void { + try validateLimits(limits); + if (configs.len > limits.max_metrics) return error.GraphMetricConfigurationLimitExceeded; + for (configs) |config| { + if (config.name.len == 0 or config.name.len > limits.max_metric_name_bytes or + config.edge_filter.types.len > limits.max_edge_filter_types) + { + return error.GraphMetricConfigurationLimitExceeded; + } + for (config.edge_filter.types) |edge_type| { + if (edge_type.len == 0 or edge_type.len > limits.max_edge_type_bytes) { + return error.GraphMetricConfigurationLimitExceeded; + } + } + } +} + +pub fn workItems(node_count: usize, edge_count: usize, iterations: u32, passes: u64) !u64 { + const per_iteration = std.math.add(u64, @intCast(node_count), @intCast(edge_count)) catch + return error.GraphMetricBuildBudgetExceeded; + const iteration_work = std.math.mul(u64, per_iteration, passes) catch + return error.GraphMetricBuildBudgetExceeded; + return std.math.mul(u64, iteration_work, iterations) catch + return error.GraphMetricBuildBudgetExceeded; +} + +pub fn metricWorkItems(kind: graph_mod.GraphMetricKind, node_count: usize, edge_count: usize, max_iterations: u32) !u64 { + const kernel_kind: metric_cost.Kind = switch (kind) { + .degree => .degree, + .pagerank => .pagerank, + .eigenvector => .eigenvector, + .hits_authority, .hits_hub => .hits, + }; + return try metric_cost.kernelWorkItems(kernel_kind, node_count, edge_count, max_iterations); +} + +pub fn projectionWorkItems( + source_node_count: usize, + source_edge_count: usize, + projected_node_count: usize, + projected_edge_count: usize, + projected_passes: u64, +) !u64 { + if (projected_passes == 0 or projected_passes > 4) return error.InvalidGraphMetricBuildOptions; + const source_scan = try workItems(source_node_count, source_edge_count, 1, 1); + const indexed_projection = try workItems(projected_node_count, projected_edge_count, 1, projected_passes); + return std.math.add(u64, source_scan, indexed_projection) catch error.GraphMetricBuildBudgetExceeded; +} + +/// Total work charged for one materialization. Projection scans every source +/// node and outbound edge once before the metric kernel sees the filtered +/// graph, so those passes must be included in admission as well. +pub fn materializationWorkItems( + kind: graph_mod.GraphMetricKind, + source_node_count: usize, + source_edge_count: usize, + projected_node_count: usize, + projected_edge_count: usize, + max_iterations: u32, +) !u64 { + // Active-set discovery, exact projection fill, and degree counts require + // three passes. Neighbor-bearing kernels add one adjacency fill pass. + const projection_passes: u64 = if (kind == .degree) 3 else 4; + const projection = try projectionWorkItems(source_node_count, source_edge_count, projected_node_count, projected_edge_count, projection_passes); + const kernel = try metricWorkItems(kind, projected_node_count, projected_edge_count, max_iterations); + return std.math.add(u64, projection, kernel) catch error.GraphMetricBuildBudgetExceeded; +} + +pub fn materializerFingerprint(limits: Limits) u64 { + var hasher = std.hash.Wyhash.init(0); + hash(&hasher, materializer_epoch); + inline for (std.meta.fields(Limits)) |field| hash(&hasher, @field(limits, field.name)); + const value = hasher.final() & std.math.maxInt(i64); + return if (value == 0) 1 else value; +} + +fn hash(hasher: *std.hash.Wyhash, value: anytype) void { + const normalized: u64 = @intCast(value); + var encoded: [@sizeOf(u64)]u8 = undefined; + std.mem.writeInt(u64, &encoded, normalized, .little); + hasher.update(&encoded); +} + +test "serverless graph metric policy bounds aggregate work and configuration fanout" { + var seeds = Budget{ .limits = .{ .max_total_seed_payload_bytes = 100, .max_total_seed_work_items = 110 } }; + try std.testing.expect(seeds.admitSeed(50, 5)); + try std.testing.expect(!seeds.admitSeed(51, 5)); + try std.testing.expect(!seeds.admitSeed(50, 6)); + try std.testing.expect(seeds.admitSeed(50, 5)); + try std.testing.expectEqual(@as(u64, 100), seeds.seed_payload_bytes); + try std.testing.expectEqual(@as(u64, 110), seeds.seed_work_items); + try std.testing.expectEqual(@as(u64, 0), seeds.work_items); + var budget = Budget{ .limits = .{ .max_work_items = 10, .max_total_work_items = 12 } }; + try budget.chargeWork(10); + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, budget.chargeWork(3)); + + const configs = [_]graph_mod.GraphMetricConfig{ + .{ .name = "a" }, + .{ .name = "b" }, + }; + try std.testing.expectError( + error.GraphMetricConfigurationLimitExceeded, + validateConfigs(&configs, .{ .max_metrics = 1 }), + ); + try std.testing.expectError( + error.GraphMetricConfigurationLimitExceeded, + validateCatalogFanout(2, 2, .{ .max_graph_indexes = 1 }), + ); + try std.testing.expectError( + error.GraphMetricConfigurationLimitExceeded, + validateCatalogFanout(1, 2, .{ .max_total_metrics = 1 }), + ); +} + +test "serverless graph metric policy charges each unique topology once" { + var budget = Budget{ .limits = .{ .max_graph_payload_bytes = 10, .max_total_graph_payload_bytes = 12 } }; + try budget.chargeGraphPayload("sha256:a", "a", 10); + try budget.chargeGraphPayload("sha256:a", "a", 10); + try std.testing.expectEqual(@as(usize, 10), budget.graph_payload_bytes); + try std.testing.expectEqual(@as(usize, 1), budget.graph_identity_count); + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, budget.chargeGraphPayload("sha256:b", "b", 3)); +} + +test "serverless graph metric policy fingerprint changes with materialization limits" { + const baseline = materializerFingerprint(.{}); + try std.testing.expect(baseline != materializerFingerprint(.{ .max_nodes = 999_999 })); +} + +test "serverless graph metric graph admission cannot exceed decoder capacity" { + const decoder_limit = (bounded_decode.Limits{}).max_artifact_bytes; + try std.testing.expectEqual(decoder_limit, (Limits{}).max_graph_payload_bytes); + try std.testing.expectError( + error.InvalidGraphMetricBuildOptions, + validateLimits(.{ .max_graph_payload_bytes = decoder_limit + 1 }), + ); +} diff --git a/zig/pkg/antfly/src/serverless/build/impact_planner.zig b/zig/pkg/antfly/src/serverless/build/impact_planner.zig index e56ccded14..ec1640bfa2 100644 --- a/zig/pkg/antfly/src/serverless/build/impact_planner.zig +++ b/zig/pkg/antfly/src/serverless/build/impact_planner.zig @@ -14,6 +14,8 @@ const std = @import("std"); const catalog_types = @import("../catalog/types.zig"); +const graph_metric_config = @import("graph_metric_config.zig"); +const lake_graph_metric = @import("lake_graph_metric.zig"); pub const ArtifactFamily = enum { document_segment, @@ -43,6 +45,7 @@ pub const ArtifactImpactPlan = struct { rebuild_dense_vector: bool = false, rebuild_sparse_vector: bool = false, rebuild_graph: bool = false, + rebuild_graph_metrics: bool = false, republish_full_text_from_head: bool = false, republish_dense_vector_from_head: bool = false, republish_sparse_vector_from_head: bool = false, @@ -58,6 +61,7 @@ pub const ArtifactImpactPlan = struct { self.rebuild_dense_vector or self.rebuild_sparse_vector or self.rebuild_graph or + self.rebuild_graph_metrics or self.rebuild_chunk_preview or self.rebuild_chunk_embeddings or self.rebuild_rerank_terms or @@ -70,6 +74,7 @@ pub const ArtifactImpactPlan = struct { self.republish_dense_vector_from_head or self.republish_sparse_vector_from_head or self.republish_graph_from_head or + self.rebuild_graph_metrics or self.rebuild_chunk_preview or self.rebuild_rerank_terms; } @@ -107,6 +112,9 @@ pub fn planAlloc(alloc: std.mem.Allocator, input: PlanInput) !ArtifactImpactPlan plan.rebuild_graph = true; plan.republish_graph_from_head = true; } + if (try graphMetricsChanged(alloc, before_indexes.value, after_indexes.value)) { + plan.rebuild_graph_metrics = true; + } const lexical_sparse_changed = input.before_policy.enrichment_enabled != input.after_policy.enrichment_enabled or @@ -179,7 +187,7 @@ fn familyChanged(before: std.json.Value, after: std.json.Value, family: Artifact if (classifyIndexFamily(entry.value_ptr.*) != family) continue; const after_value = after_object.get(entry.key_ptr.*) orelse return true; if (classifyIndexFamily(after_value) != family) return true; - if (!jsonValueEql(entry.value_ptr.*, after_value)) return true; + if (!indexConfigEql(entry.value_ptr.*, after_value, family)) return true; } var after_it = after_object.iterator(); @@ -192,6 +200,64 @@ fn familyChanged(before: std.json.Value, after: std.json.Value, family: Artifact return false; } +fn graphMetricsChanged(alloc: std.mem.Allocator, before: std.json.Value, after: std.json.Value) !bool { + const before_object = switch (before) { + .object => |value| value, + else => return error.InvalidTableIndexMetadata, + }; + const after_object = switch (after) { + .object => |value| value, + else => return error.InvalidTableIndexMetadata, + }; + var it = before_object.iterator(); + while (it.next()) |entry| { + if (classifyIndexFamily(entry.value_ptr.*) != .graph) continue; + const after_value = after_object.get(entry.key_ptr.*) orelse continue; + if (classifyIndexFamily(after_value) != .graph) continue; + const before_metrics = try graph_metric_config.parseMetricConfigsAlloc(alloc, entry.value_ptr.*); + defer @import("../../graph/graph.zig").freeGraphMetricConfigs(alloc, before_metrics); + const after_metrics = try graph_metric_config.parseMetricConfigsAlloc(alloc, after_value); + defer @import("../../graph/graph.zig").freeGraphMetricConfigs(alloc, after_metrics); + if (!canonicalGraphMetricsEql(before_metrics, after_metrics)) return true; + } + return false; +} + +fn canonicalGraphMetricsEql( + lhs: []const @import("../../graph/graph.zig").GraphMetricConfig, + rhs: []const @import("../../graph/graph.zig").GraphMetricConfig, +) bool { + if (lhs.len != rhs.len) return false; + for (lhs, rhs) |left, right| { + if (!std.mem.eql(u8, left.name, right.name) or + lake_graph_metric.configFingerprint(left) != lake_graph_metric.configFingerprint(right)) return false; + } + return true; +} + +fn indexConfigEql(lhs: std.json.Value, rhs: std.json.Value, family: ArtifactFamily) bool { + if (family != .graph) return jsonValueEql(lhs, rhs); + return jsonObjectEqlIgnoringField(lhs, rhs, "metrics"); +} + +fn jsonObjectEqlIgnoringField(lhs: std.json.Value, rhs: std.json.Value, ignored: []const u8) bool { + if (lhs != .object or rhs != .object) return false; + var lhs_count: usize = 0; + var lhs_it = lhs.object.iterator(); + while (lhs_it.next()) |entry| { + if (std.mem.eql(u8, entry.key_ptr.*, ignored)) continue; + lhs_count += 1; + const other = rhs.object.get(entry.key_ptr.*) orelse return false; + if (!jsonValueEql(entry.value_ptr.*, other)) return false; + } + var rhs_count: usize = 0; + var rhs_it = rhs.object.iterator(); + while (rhs_it.next()) |entry| { + if (!std.mem.eql(u8, entry.key_ptr.*, ignored)) rhs_count += 1; + } + return lhs_count == rhs_count; +} + fn classifyIndexFamily(value: std.json.Value) ?ArtifactFamily { const object = switch (value) { .object => |map| map, @@ -306,3 +372,21 @@ test "impact planner is stable for semantically equal index json with reordered }); try std.testing.expect(!plan.any()); } + +test "serverless impact planner rebuilds graph metrics without rebuilding graph topology" { + const plan = try planAlloc(std.testing.allocator, .{ + .before_indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"edge_types\":[\"cites\"],\"metrics\":{\"pagerank\":{\"kind\":\"pagerank\",\"max_iterations\":20}}}}", + .after_indexes_json = "{\"graph_idx\":{\"metrics\":{\"pagerank\":{\"max_iterations\":40,\"kind\":\"pagerank\"}},\"edge_types\":[\"cites\"],\"type\":\"graph\"}}", + }); + try std.testing.expect(!plan.rebuild_graph); + try std.testing.expect(plan.rebuild_graph_metrics); + try std.testing.expect(plan.requiresHeadRepublish()); +} + +test "serverless impact planner ignores explicit graph metric defaults" { + const plan = try planAlloc(std.testing.allocator, .{ + .before_indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\"}}}}", + .after_indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"refresh\":\"background\",\"damping\":0.85,\"tolerance\":0.000001,\"max_iterations\":50}}}}", + }); + try std.testing.expect(!plan.any()); +} diff --git a/zig/pkg/antfly/src/serverless/build/lake_gc.zig b/zig/pkg/antfly/src/serverless/build/lake_gc.zig index a3cfd9e57e..4c5e7bacbc 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_gc.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_gc.zig @@ -185,7 +185,7 @@ fn findArtifact( fn isLakeArtifact(kind: artifact_ref.ArtifactKind) bool { return switch (kind) { .row_fragment, .row_fragment_stats, .algebraic_segment, .external_base_source => true, - .text_segment, .vector_segment, .sparse_segment, .graph_segment => true, + .text_segment, .vector_segment, .sparse_segment, .graph_segment, .graph_metric_segment => true, .doc_values, .stored_fields, .mutation_segment, .document_segment => false, }; } diff --git a/zig/pkg/antfly/src/serverless/build/lake_graph_metric.zig b/zig/pkg/antfly/src/serverless/build/lake_graph_metric.zig new file mode 100644 index 0000000000..da219c085d --- /dev/null +++ b/zig/pkg/antfly/src/serverless/build/lake_graph_metric.zig @@ -0,0 +1,4261 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Bounded materialization of immutable graph metric vectors. This code is +//! synchronous at its public boundary: callers schedule it through their +//! existing runtime, while large target-owned kernel ranges use that same +//! std.Io instance for bounded deterministic fan-out. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +const graph_mod = @import("../../graph/graph.zig"); +const metrics = @import("../../graph/metrics.zig"); +const artifact_ref = @import("../manifest/artifact_ref.zig"); +const artifact_store = @import("../artifacts/store.zig"); +const fs_artifact_store = @import("../artifacts/fs_store.zig"); +const bounded_decode = @import("../bounded_decode.zig"); +const graph_segment = @import("../graph_segment/mod.zig"); +const metric_segment = @import("../graph_metric_segment/mod.zig"); +const graph_metric_policy = @import("graph_metric_policy.zig"); + +pub const Limits = graph_metric_policy.Limits; +pub const default_compute_parallelism: usize = 4; +pub const max_compute_parallelism: usize = 16; + +pub const Provenance = struct { + published_generation: u64 = 0, + edge_generation: u64 = 0, + computed_at_ms: u64 = 0, + + pub fn validate(self: Provenance) !void { + if (self.published_generation == 0 or self.edge_generation == 0 or self.computed_at_ms == 0 or + self.edge_generation > self.published_generation) + { + return error.InvalidGraphMetricProvenance; + } + } +}; + +pub const BuildOptions = struct { + graph_index_name: []const u8, + config: graph_mod.GraphMetricConfig, + source_graph: artifact_ref.ArtifactRef, + cancellation: CancellationToken = .none, + limits: Limits = .{}, + batch_budget: ?*graph_metric_policy.Budget = null, + provenance: Provenance = .{}, + io: ?std.Io = null, + max_parallelism: usize = 1, + topology_requirements: ?metrics.TopologyRequirements = null, + topology_checksum: [32]u8 = @splat(0), + /// Optional current-ordinal vectors recovered from the last compatible + /// publication. They are borrowed for the duration of the build and are + /// normalized/validated by the storage-independent kernel. + initial_scores: ?[]const f64 = null, + initial_authorities: ?[]const f64 = null, + initial_hubs: ?[]const f64 = null, +}; + +pub const ComputeRuntime = struct { + io: ?std.Io = null, + max_parallelism: usize = 1, + + fn validate(self: ComputeRuntime) !void { + if (self.max_parallelism == 0 or self.max_parallelism > max_compute_parallelism) return error.InvalidGraphMetricBuildOptions; + if (self.io == null and self.max_parallelism != 1) return error.InvalidGraphMetricBuildOptions; + } +}; + +fn topologyRequirementsForKind(kind: graph_mod.GraphMetricKind) metrics.TopologyRequirements { + return switch (kind) { + .degree => .degree, + .pagerank => .pagerank, + .eigenvector => .eigenvector, + .hits_authority, .hits_hub => .hits, + }; +} + +pub const BuildResult = struct { + payload: []u8, + artifact: artifact_ref.ArtifactRef, + + pub fn deinit(self: *BuildResult, alloc: Allocator) void { + alloc.free(self.payload); + freeArtifactRef(alloc, self.artifact); + self.* = undefined; + } +}; + +const indexed_topology = @import("../graph_segment/topology_reader.zig"); +const CompiledEdge = indexed_topology.Edge; +const CompiledTopology = indexed_topology.Topology; + +/// One authenticated and decoded topology artifact. Publication orchestrators +/// may reuse this across graph-index aliases that identify the same immutable +/// payload, avoiding repeated object-store reads and decode allocations while +/// retaining an exact provenance check at every use. +pub const PreparedGraphArtifact = struct { + source_artifact_id: []u8, + source_checksum: []u8, + source_byte_len: u64, + topology: CompiledTopology, + /// Bounded, request-scoped projection reuse. Publication visits graph + /// aliases sequentially, so retaining only the most recent filter keeps + /// peak memory O(one projection) while eliminating repeated O(V+E) work + /// for aliases over the same immutable graph. + cached_projection: ?Projection = null, + cached_projection_filter: ?graph_mod.GraphMetricEdgeFilter = null, + cached_projection_requirements: metrics.TopologyRequirements = .{}, + + pub fn deinit(self: *PreparedGraphArtifact, alloc: Allocator) void { + if (self.cached_projection) |*projection| projection.deinit(alloc); + if (self.cached_projection_filter) |*filter| filter.deinit(alloc); + alloc.free(self.source_artifact_id); + alloc.free(self.source_checksum); + self.topology.deinit(alloc); + self.* = undefined; + } + + pub fn identifies(self: PreparedGraphArtifact, source_graph: artifact_ref.ArtifactRef) bool { + return source_graph.kind == .graph_segment and + source_graph.byte_len == self.source_byte_len and + std.mem.eql(u8, source_graph.artifact_id, self.source_artifact_id) and + std.mem.eql(u8, source_graph.checksum, self.source_checksum); + } +}; + +// Hash node strings once, then fixed-width endpoint digests once per edge. +// Type runs and endpoints are canonical; global ordinal renumbering, weights, +// isolated documents, and unrelated types cannot change a selected digest. +fn typeChecksumsAlloc(alloc: Allocator, topology: CompiledTopology, limits: Limits, cancellation: CancellationToken) ![][32]u8 { + if (try topologyIdentityWorkBytes(topology) > limits.max_total_identity_work_bytes) return error.GraphMetricBuildBudgetExceeded; + if (topology.retained_bytes >= limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes - topology.retained_bytes); + return typeChecksumsBoundedAlloc(limiter.allocator(), topology, cancellation) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }; +} + +fn topologyIdentityWorkBytes(topology: CompiledTopology) !u64 { + var bytes = std.math.mul(u64, topology.edges.len, 64) catch return error.GraphMetricBuildBudgetExceeded; + bytes = std.math.add(u64, bytes, std.math.mul(u64, topology.node_ids.len, 64) catch return error.GraphMetricBuildBudgetExceeded) catch return error.GraphMetricBuildBudgetExceeded; + bytes = std.math.add(u64, bytes, std.math.mul(u64, topology.edge_types.len, 128) catch return error.GraphMetricBuildBudgetExceeded) catch return error.GraphMetricBuildBudgetExceeded; + for (topology.node_ids) |id| bytes = std.math.add(u64, bytes, id.len) catch return error.GraphMetricBuildBudgetExceeded; + for (topology.edge_types) |kind| bytes = std.math.add(u64, bytes, kind.len) catch return error.GraphMetricBuildBudgetExceeded; + return bytes; +} + +fn typeChecksumsBoundedAlloc(alloc: Allocator, topology: CompiledTopology, cancellation: CancellationToken) ![][32]u8 { + const nodes = try alloc.alloc([32]u8, topology.node_ids.len); + defer alloc.free(nodes); + for (topology.node_ids, nodes, 0..) |id, *digest, i| { + if (i % 256 == 0) try cancellation.check(); + std.crypto.hash.sha2.Sha256.hash(id, digest, .{}); + } + const result = try alloc.alloc([32]u8, topology.edge_types.len); + errdefer alloc.free(result); + for (topology.edge_types, result, 0..) |kind, *digest, i| { + try cancellation.check(); + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update("antfly:unweighted-type:v1"); + var count: [8]u8 = undefined; + std.mem.writeInt(u64, &count, kind.len, .little); + hash.update(&count); + hash.update(kind); + const edges = topology.edges[topology.edge_type_offsets[i]..topology.edge_type_offsets[i + 1]]; + std.mem.writeInt(u64, &count, edges.len, .little); + hash.update(&count); + for (edges, 0..) |edge, j| { + if (j % 4096 == 0) try cancellation.check(); + hash.update(&nodes[edge.source]); + hash.update(&nodes[edge.target]); + } + hash.final(digest); + } + return result; +} + +fn selectedTopologyChecksum(topology: CompiledTopology, checksums: []const [32]u8, filter: graph_mod.GraphMetricEdgeFilter) [32]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update("antfly:selected-unweighted-topology:v1"); + for (topology.edge_types, checksums, 0..) |kind, digest, i| { + if (topology.edge_type_offsets[i] == topology.edge_type_offsets[i + 1]) continue; + if (filter.mode != .all and for (filter.types) |selected| { + if (std.mem.eql(u8, selected, kind)) break false; + } else true) continue; + hash.update(&digest); + } + var result: [32]u8 = undefined; + hash.final(&result); + return result; +} + +pub fn artifactNameAlloc(alloc: Allocator, graph_index_name: []const u8, metric_name: []const u8) ![]u8 { + return metric_segment.artifactNameAlloc(alloc, graph_index_name, metric_name) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.InvalidGraphMetricBuildOptions, + }; +} + +fn admitGraphDecodePeak(payload_bytes: usize, decoded_retained_bytes: usize, max_peak_memory_bytes: usize) !void { + const decode_peak_bytes = std.math.add(usize, payload_bytes, decoded_retained_bytes) catch + return error.GraphMetricBuildBudgetExceeded; + if (decode_peak_bytes > max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; +} + +/// Metric materialization is an outbound-edge computation. Drop the decoded +/// reverse adjacency immediately after artifact validation so it does not +/// coexist with compiled topology, dense vectors, sorting, and output bytes. +fn discardInboundEdges(alloc: Allocator, graph: *graph_segment.Segment) !usize { + var released: usize = 0; + for (graph.adjacencies) |*adjacency| { + released = std.math.add( + usize, + released, + std.math.mul(usize, adjacency.in_edges.len, @sizeOf(graph_segment.Edge)) catch + return error.GraphMetricBuildBudgetExceeded, + ) catch return error.GraphMetricBuildBudgetExceeded; + for (adjacency.in_edges) |*edge| { + released = std.math.add(usize, released, edge.neighbor_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + released = std.math.add(usize, released, edge.edge_type.len) catch + return error.GraphMetricBuildBudgetExceeded; + edge.deinit(alloc); + } + alloc.free(adjacency.in_edges); + adjacency.in_edges = &.{}; + } + return released; +} + +pub fn buildFromGraphPayloadAlloc(alloc: Allocator, graph_payload: []const u8, options: BuildOptions) !BuildResult { + try validateOptions(graph_payload, options); + var topology = try prepareTopologyFromPackedAlloc(alloc, graph_payload, options.cancellation, options.limits); + defer topology.deinit(alloc); + // This entry point borrows the payload throughout kernel execution. + topology.retained_bytes = try std.math.add(usize, topology.retained_bytes, graph_payload.len); + return buildFromTopologyAlloc(alloc, topology, options); +} + +pub fn publishFromGraphPayloadAlloc(alloc: Allocator, artifacts: *artifact_store.ArtifactStore, graph_payload: []const u8, options: BuildOptions) !artifact_ref.ArtifactRef { + try options.provenance.validate(); + var built = try buildFromGraphPayloadAlloc(alloc, graph_payload, options); + defer built.deinit(alloc); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); + defer metadata.deinit(alloc); + const name = try alloc.dupe(u8, built.artifact.name); + errdefer alloc.free(name); + const artifact_id = try alloc.dupe(u8, metadata.artifact_id); + errdefer alloc.free(artifact_id); + const checksum = try alloc.dupe(u8, metadata.checksum); + return .{ + .kind = .graph_metric_segment, + .name = name, + .artifact_id = artifact_id, + .byte_len = metadata.byte_len, + .checksum = checksum, + .metadata_version = built.artifact.metadata_version, + .published_generation = built.artifact.published_generation, + .edge_generation = built.artifact.edge_generation, + .computed_at_ms = built.artifact.computed_at_ms, + .materializer_fingerprint = built.artifact.materializer_fingerprint, + .graph_metric_control_len = built.artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = built.artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = built.artifact.graph_metric_control_checksum, + .graph_metric_routing_checksum = built.artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = built.artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = built.artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = built.artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = built.artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = built.artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = built.artifact.graph_metric_rejection_reason, + }; +} + +/// Builds a metric only from bytes verified against the immutable graph +/// artifact reference. Production callers should prefer this entry point so a +/// caller cannot accidentally publish a metric whose provenance names bytes +/// other than those that were actually evaluated. +pub fn publishFromGraphArtifactAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + options: BuildOptions, +) !artifact_ref.ArtifactRef { + if (options.source_graph.byte_len > options.limits.max_graph_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + const graph_payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator( + alloc, + options.source_graph.artifact_id, + options.source_graph.byte_len, + options.source_graph.checksum, + options.cancellation, + ); + defer alloc.free(graph_payload); + return try publishFromGraphPayloadAlloc(alloc, artifacts, graph_payload, options); +} + +/// Publishes several metric configurations from one verified fetch and one +/// graph decode. This is the normal lifecycle entry point for a graph index; +/// its peak graph memory is constant and its graph verification/decoding cost +/// does not grow with the number of configured metrics. +pub fn publishManyFromGraphArtifactAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + limits: Limits, + provenance: Provenance, +) ![]artifact_ref.ArtifactRef { + var batch_budget = graph_metric_policy.Budget{ .limits = limits }; + return publishManyFromGraphArtifactWithBudgetAlloc( + alloc, + artifacts, + graph_index_name, + source_graph, + configs, + cancellation, + limits, + &batch_budget, + provenance, + .{}, + ); +} + +/// Variant used by table publication so work and output admission span every +/// graph index rebuilt in the same publication, rather than resetting for each +/// index. +pub fn publishManyFromGraphArtifactWithBudgetAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + limits: Limits, + batch_budget: *graph_metric_policy.Budget, + provenance: Provenance, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + if (configs.len == 0) return try alloc.alloc(artifact_ref.ArtifactRef, 0); + try validatePublicationOptions(graph_index_name, source_graph, configs, cancellation, limits, batch_budget); + try batch_budget.chargeGraphPayload(source_graph.artifact_id, source_graph.checksum, source_graph.byte_len); + var prepared = prepareGraphArtifactAlloc(alloc, artifacts, source_graph, cancellation, limits) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => return publishRejectedManyAlloc(alloc, artifacts, graph_index_name, source_graph, configs, cancellation, .build_budget_exceeded, limits, provenance), + else => return err, + }; + defer prepared.deinit(alloc); + return try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + artifacts, + graph_index_name, + source_graph, + configs, + cancellation, + limits, + batch_budget, + &prepared, + provenance, + runtime, + ); +} + +fn prepareSelectedGraphArtifactAlloc(alloc: Allocator, artifacts: *artifact_store.ArtifactStore, source: artifact_ref.ArtifactRef, configs: []const graph_mod.GraphMetricConfig, cancellation: CancellationToken, limits: Limits, budget: *graph_metric_policy.Budget) !?PreparedGraphArtifact { + if (source.byte_len > limits.max_graph_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes); + var remaining: u64 = limits.max_total_graph_payload_bytes -| budget.graph_payload_bytes; + const before = remaining; + defer budget.graph_payload_bytes += @intCast(before - remaining); + var topology = (indexed_topology.readAlloc(limiter.allocator(), artifacts, source, configs, limits, cancellation, &remaining) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }) orelse return null; + errdefer topology.deinit(alloc); + const id = limiter.allocator().dupe(u8, source.artifact_id) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }; + errdefer alloc.free(id); + const checksum = limiter.allocator().dupe(u8, source.checksum) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }; + return .{ .source_artifact_id = id, .source_checksum = checksum, .source_byte_len = source.byte_len, .topology = topology }; +} + +fn prepareContextGraphAlloc(alloc: Allocator, context: *indexed_topology.Context, source: artifact_ref.ArtifactRef, configs: []const graph_mod.GraphMetricConfig, cancellation: CancellationToken, limits: Limits) !?PreparedGraphArtifact { + if (source.byte_len > limits.max_graph_payload_bytes or context.retainedBytes() >= limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes - context.retainedBytes()); + var topology = (indexed_topology.readPreparedAlloc(limiter.allocator(), context, configs, limits, cancellation) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }) orelse return null; + errdefer topology.deinit(alloc); + topology.retained_bytes += context.retainedBytes() + source.artifact_id.len + source.checksum.len; + const id = limiter.allocator().dupe(u8, source.artifact_id) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }; + errdefer alloc.free(id); + const checksum = limiter.allocator().dupe(u8, source.checksum) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.GraphMetricBuildBudgetExceeded; + return err; + }; + return .{ .source_artifact_id = id, .source_checksum = checksum, .source_byte_len = source.byte_len, .topology = topology }; +} + +pub fn prepareGraphArtifactAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + source_graph: artifact_ref.ArtifactRef, + cancellation: CancellationToken, + limits: Limits, +) !PreparedGraphArtifact { + try cancellation.check(); + try graph_metric_policy.validateLimits(limits); + if (source_graph.kind != .graph_segment or source_graph.byte_len == 0) return error.InvalidGraphMetricBuildOptions; + // Payload residency is part of the peak, not merely an independent wire + // limit. Reject impossible builds before any object-store read/allocation. + if (source_graph.byte_len > limits.max_graph_payload_bytes or + source_graph.byte_len >= limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + const graph_payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator( + alloc, + source_graph.artifact_id, + source_graph.byte_len, + source_graph.checksum, + cancellation, + ); + defer alloc.free(graph_payload); + var topology = try prepareTopologyFromPackedAlloc(alloc, graph_payload, cancellation, limits); + errdefer topology.deinit(alloc); + const source_artifact_id = try alloc.dupe(u8, source_graph.artifact_id); + errdefer alloc.free(source_artifact_id); + const source_checksum = try alloc.dupe(u8, source_graph.checksum); + return .{ + .source_artifact_id = source_artifact_id, + .source_checksum = source_checksum, + .source_byte_len = source_graph.byte_len, + .topology = topology, + }; +} + +pub fn publishManyFromPreparedGraphWithBudgetAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + limits: Limits, + batch_budget: *graph_metric_policy.Budget, + prepared: *PreparedGraphArtifact, + provenance: Provenance, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + return publishManyFromPreparedGraphWithWarmStartsAlloc( + alloc, + artifacts, + graph_index_name, + source_graph, + configs, + &.{}, + cancellation, + limits, + batch_budget, + prepared, + provenance, + runtime, + ); +} + +/// Rebuild variant that may seed iterative kernels from previous immutable +/// metric artifacts. `prior_artifacts`, when non-empty, is aligned with +/// `configs`; every candidate is independently authenticated and rejected as a +/// seed (not as a build) when its format or configuration is incompatible. +pub fn publishManyFromPreparedGraphWithWarmStartsAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + prior_artifacts: []const ?artifact_ref.ArtifactRef, + cancellation: CancellationToken, + limits: Limits, + batch_budget: *graph_metric_policy.Budget, + prepared: *PreparedGraphArtifact, + provenance: Provenance, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + try validatePublicationOptions(graph_index_name, source_graph, configs, cancellation, limits, batch_budget); + return publishPreparedComputationsAlloc(alloc, artifacts, graph_index_name, source_graph, configs, prior_artifacts, cancellation, limits, batch_budget, prepared, provenance, runtime); +} + +fn inProjectionGroup(candidate: graph_mod.GraphMetricConfig, representative: graph_mod.GraphMetricConfig, share: bool) bool { + return candidate.edge_filter.equivalent(representative.edge_filter) and + (share or std.meta.eql(topologyRequirementsForKind(candidate.kind), topologyRequirementsForKind(representative.kind))); +} + +/// A no-allocation upper-bound admission pass. Share the union only when all +/// consumers fit; otherwise process cheapest exact requirement groups first. +/// Overestimation can forgo sharing, but cannot reject an affordable metric or +/// spend work building a union which is immediately discarded under pressure. +fn projectionGroupFits(topology: CompiledTopology, configs: []const graph_mod.GraphMetricConfig, processed: []const bool, filter: graph_mod.GraphMetricEdgeFilter, limits: Limits, budget: graph_metric_policy.Budget) bool { + var e: usize = 0; + for (topology.edge_types, 0..) |edge_type, i| { + if (filter.mode != .all and !filter.includesType(edge_type)) continue; + e += topology.edge_type_offsets[i + 1] - topology.edge_type_offsets[i]; + } + // Every active vertex is an endpoint of a selected local edge. This + // allocation-free bound is valid even before constructing the exact census + // and, unlike source V, scales with sparse edge-type selections. + const n = @min(topology.source_node_count, std.math.mul(usize, e, 2) catch return false); + // Covers packed source residency, active/mapping/CSR construction, every + // kernel's vectors, borrowed score views and one encoded output. Paired + // HITS outputs are encoded sequentially and share the same peak bound. + var peak = std.math.add(usize, topology.retained_bytes, 1024 * 1024) catch return false; + peak = std.math.add(usize, peak, std.math.mul(usize, topology.string_bytes.len, 2) catch return false) catch return false; + peak = std.math.add(usize, peak, std.math.mul(usize, n, 192) catch return false) catch return false; + peak = std.math.add(usize, peak, std.math.mul(usize, e, 24) catch return false) catch return false; + // Dense preparation still owns a source-wide ordinal map and bitset; + // sparse preparation owns at most 2E endpoint ordinals instead. + const mapping = if (useSparseProjection(topology.source_node_count, e)) + std.math.mul(usize, e, 8) catch return false + else + std.math.mul(usize, topology.source_node_count, 5) catch return false; + peak = std.math.add(usize, peak, mapping) catch return false; + if (peak > limits.max_peak_memory_bytes) return false; + const census = projectionCensusWork(topology, filter, e) catch return false; + const projection = std.math.add(u64, census, graph_metric_policy.workItems(n, e, 1, 4) catch return false) catch return false; + var work = projection; + for (configs, processed, 0..) |config, done, i| { + if (done or !config.edge_filter.equivalent(filter)) continue; + const paired = for (configs[0..i], processed[0..i]) |prior, prior_done| { + if (!prior_done and (graph_mod.graphMetricHitsPairCompatible(prior, config) or sameComputation(prior, config))) break true; + } else false; + if (paired) continue; + const kernel = graph_metric_policy.metricWorkItems(config.kind, n, e, config.max_iterations) catch return false; + const individual = std.math.add(u64, projection, kernel) catch return false; + if (individual > limits.max_work_items) return false; + work = std.math.add(u64, work, kernel) catch return false; + } + return work <= limits.max_total_work_items -| budget.work_items; +} + +/// Internal computation groups can span independently validated index aliases. +/// Their names and refresh policies are not a catalog configuration. +fn publishPreparedComputationsAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + prior_artifacts: []const ?artifact_ref.ArtifactRef, + cancellation: CancellationToken, + limits: Limits, + batch_budget: *graph_metric_policy.Budget, + prepared: *PreparedGraphArtifact, + provenance: Provenance, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + if (configs.len == 0) return try alloc.alloc(artifact_ref.ArtifactRef, 0); + if (prior_artifacts.len != 0 and prior_artifacts.len != configs.len) return error.InvalidGraphMetricBuildOptions; + try provenance.validate(); + try runtime.validate(); + if (!prepared.identifies(source_graph)) return error.ArtifactIntegrityMismatch; + + const refs = try alloc.alloc(artifact_ref.ArtifactRef, configs.len); + errdefer alloc.free(refs); + const initialized = try alloc.alloc(bool, configs.len); + defer alloc.free(initialized); + @memset(initialized, false); + const processed = try alloc.alloc(bool, configs.len); + defer alloc.free(processed); + @memset(processed, false); + errdefer { + for (refs, initialized) |ref, ready| if (ready) freeArtifactRef(alloc, ref); + } + const topology_checksums = try alloc.alloc([32]u8, configs.len); + defer alloc.free(topology_checksums); + @memset(topology_checksums, @splat(0)); + if (prepared.topology.type_checksums.len == prepared.topology.edge_types.len) { + for (configs, topology_checksums) |config, *digest| digest.* = selectedTopologyChecksum(prepared.topology, prepared.topology.type_checksums, config.edge_filter); + } else { + const work = topologyIdentityWorkBytes(prepared.topology) catch std.math.maxInt(u64); + if (work <= limits.max_total_identity_work_bytes -| batch_budget.identity_work_bytes) { + batch_budget.identity_work_bytes += work; + // Fingerprinting may overlap a cached projection from an earlier + // call. Account for it, and make the optional allocation a no-op + // when it would crowd out already retained work. + var hashing_topology = prepared.topology; + if (prepared.cached_projection) |projection| hashing_topology.retained_bytes = try projectionResidentMemoryBytes(projection); + const type_checksums = typeChecksumsAlloc(alloc, hashing_topology, limits, cancellation) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => null, + else => return err, + }; + if (type_checksums) |checksums| { + defer alloc.free(checksums); + for (configs, topology_checksums) |config, *digest| digest.* = selectedTopologyChecksum(prepared.topology, checksums, config.edge_filter); + } + } + } + var inventory = PriorInventory{ .budget = batch_budget }; + defer inventory.deinit(alloc); + for (configs, topology_checksums, 0..) |config, digest, i| { + if (prior_artifacts.len == 0) break; + const prior = prior_artifacts[i] orelse continue; + const zero: [32]u8 = @splat(0); + if (std.mem.eql(u8, &digest, &zero)) continue; + const request = PublicationRequest{ .graph_index_name = graph_index_name, .source_graph = source_graph, .config = config, .provenance = provenance }; + if (prior.metadata_version != metric_segment.wire_version or prior.graph_metric_materialization_state != .ready or + prior.graph_metric_config_fingerprint != configFingerprint(config) or prior.materializer_fingerprint != materializerFingerprint(limits) or + prior.byte_len == 0 or prior.byte_len > limits.max_metric_payload_bytes or + !std.mem.eql(u8, &prior.graph_metric_topology_checksum, &digest)) continue; + const header = (try inventory.header(alloc, artifacts, prior, request, cancellation)) orelse continue; + if (header.version != metric_segment.wire_version or header.kind != config.kind or + header.materialization_state != .ready or header.config_fingerprint != configFingerprint(config) or + header.materializer_fingerprint != materializerFingerprint(limits) or !std.mem.eql(u8, &header.topology_checksum, &digest)) continue; + refs[i] = try aliasRefAlloc(alloc, prior, graph_index_name, config.name, provenance); + initialized[i] = true; + processed[i] = true; + refs[i].computed_at_ms = prior.computed_at_ms; + refs[i].graph_metric_source_checksum = try artifact_store.sha256DigestFromChecksum(source_graph.checksum); + } + while (true) { + const first = for (processed, 0..) |done, i| { + if (!done) break i; + } else break; + try cancellation.check(); + var config = configs[first]; + const share = projectionGroupFits(prepared.topology, configs, processed, config.edge_filter, limits, batch_budget.*); + if (!share) { + var cheapest: u64 = std.math.maxInt(u64); + for (configs, processed) |candidate, done| { + if (done or !candidate.edge_filter.equivalent(config.edge_filter)) continue; + const selected_edges = try selectedEdgeCount(prepared.topology, candidate.edge_filter, cancellation); + const selected_nodes = @min(prepared.topology.source_node_count, selected_edges * 2); + const cost = graph_metric_policy.metricWorkItems(candidate.kind, selected_nodes, selected_edges, candidate.max_iterations) catch std.math.maxInt(u64); + if (cost < cheapest) { + cheapest = cost; + config = candidate; + } + } + } + var group_requirements = topologyRequirementsForKind(config.kind); + if (share) for (configs, processed) |candidate, done| { + if (done or !candidate.edge_filter.equivalent(config.edge_filter)) continue; + group_requirements = group_requirements.merge(topologyRequirementsForKind(candidate.kind)); + }; + const group_options = BuildOptions{ + .graph_index_name = graph_index_name, + .config = config, + .source_graph = source_graph, + .cancellation = cancellation, + .limits = limits, + .batch_budget = batch_budget, + .provenance = provenance, + .io = runtime.io, + .max_parallelism = runtime.max_parallelism, + .topology_requirements = group_requirements, + .topology_checksum = topology_checksums[first], + }; + const projection_result = preparedProjectionAlloc(alloc, prepared, group_options) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => { + for (configs, 0..) |candidate, candidate_index| { + if (processed[candidate_index] or !inProjectionGroup(candidate, config, share)) continue; + refs[candidate_index] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, candidate, cancellation, .build_budget_exceeded, limits, provenance); + initialized[candidate_index] = true; + processed[candidate_index] = true; + } + continue; + }, + else => return err, + }; + const projection = projection_result.projection; + + for (configs, 0..) |candidate, candidate_index| { + if (processed[candidate_index] or !inProjectionGroup(candidate, config, share)) continue; + try cancellation.check(); + const reused = for (configs, initialized, 0..) |prior, ready, prior_index| { + if (ready and sameComputation(prior, candidate)) break prior_index; + } else null; + if (reused) |prior_index| { + refs[candidate_index] = try aliasRefAlloc(alloc, refs[prior_index], graph_index_name, candidate.name, provenance); + initialized[candidate_index] = true; + processed[candidate_index] = true; + continue; + } + var options = group_options; + options.config = candidate; + const projection_resident_bytes = try projectionResidentMemoryBytes(projection.*); + const cold_peak_bytes = try estimatedPeakMemoryBytes(projection.*, options, 1); + if (findCompatibleHitsPairIndex(configs, processed, candidate_index)) |pair_index| { + // The pair still shares one topology and kernel execution, + // but spectral vectors always use canonical cold seeds. + const pair_refs = publishHitsPairFromProjectionAlloc( + alloc, + artifacts, + projection.*, + options, + configs[pair_index], + cancellation, + ) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => { + refs[candidate_index] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, candidate, cancellation, .build_budget_exceeded, limits, provenance); + initialized[candidate_index] = true; + refs[pair_index] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, configs[pair_index], cancellation, .build_budget_exceeded, limits, provenance); + initialized[pair_index] = true; + processed[candidate_index] = true; + processed[pair_index] = true; + continue; + }, + else => return err, + }; + refs[candidate_index] = pair_refs[0]; + initialized[candidate_index] = true; + refs[pair_index] = pair_refs[1]; + initialized[pair_index] = true; + processed[candidate_index] = true; + processed[pair_index] = true; + continue; + } + admitProjectionKernel(projection.*, options) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => { + refs[candidate_index] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, candidate, cancellation, .build_budget_exceeded, limits, provenance); + initialized[candidate_index] = true; + processed[candidate_index] = true; + continue; + }, + else => return err, + }; + const warm_seed: ?[]f64 = if (prior_artifacts.len == 0) + null + else + try warmStartVectorAlloc(alloc, artifacts, prior_artifacts[candidate_index], projection.node_ids.items, candidate, cancellation, limits, projection_resident_bytes, cold_peak_bytes, options.batch_budget); + defer if (warm_seed) |seed| alloc.free(seed); + switch (candidate.kind) { + .pagerank, .eigenvector => options.initial_scores = warm_seed, + .hits_authority => options.initial_authorities = warm_seed, + .hits_hub => options.initial_hubs = warm_seed, + .degree => {}, + } + var built = buildAdmittedProjectionAlloc(alloc, projection.*, options) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => { + refs[candidate_index] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, candidate, cancellation, .build_budget_exceeded, limits, provenance); + initialized[candidate_index] = true; + processed[candidate_index] = true; + continue; + }, + else => return err, + }; + defer built.deinit(alloc); + refs[candidate_index] = try putBuildResultAlloc(alloc, artifacts, &built, cancellation); + initialized[candidate_index] = true; + processed[candidate_index] = true; + } + } + return refs; +} + +pub const PublicationRequest = struct { + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + config: graph_mod.GraphMetricConfig, + prior_artifact: ?artifact_ref.ArtifactRef = null, + provenance: Provenance, +}; + +fn sameSource(a: artifact_ref.ArtifactRef, b: artifact_ref.ArtifactRef) bool { + return a.kind == b.kind and a.byte_len == b.byte_len and + std.mem.eql(u8, &a.graph_topology_control_checksum, &b.graph_topology_control_checksum) and + std.mem.eql(u8, a.artifact_id, b.artifact_id) and std.mem.eql(u8, a.checksum, b.checksum); +} + +pub fn sameComputation(a: graph_mod.GraphMetricConfig, b: graph_mod.GraphMetricConfig) bool { + // Exact equality, not a truncated storage fingerprint. Names and refresh + // scheduling do not affect immutable metric computation. + return a.kind == b.kind and @as(u64, @bitCast(a.damping)) == @as(u64, @bitCast(b.damping)) and + @as(u64, @bitCast(a.tolerance)) == @as(u64, @bitCast(b.tolerance)) and + a.max_iterations == b.max_iterations and a.edge_filter.equivalent(b.edge_filter); +} + +fn aliasRefAlloc(alloc: Allocator, original: artifact_ref.ArtifactRef, index_name: []const u8, metric_name: []const u8, provenance: Provenance) !artifact_ref.ArtifactRef { + var ref = original; + ref.name = try metric_segment.artifactNameAlloc(alloc, index_name, metric_name); + errdefer alloc.free(ref.name); + ref.artifact_id = try alloc.dupe(u8, original.artifact_id); + errdefer alloc.free(ref.artifact_id); + ref.checksum = try alloc.dupe(u8, original.checksum); + ref.published_generation = provenance.published_generation; + ref.edge_generation = provenance.edge_generation; + ref.computed_at_ms = if (original.computed_at_ms != 0) original.computed_at_ms else provenance.computed_at_ms; + return ref; +} + +/// Plan a whole publication, not one named index at a time. Stable source and +/// filter groups keep exactly one decoded source and one projection resident. +/// Each distinct kernel is encoded/uploaded once; only lightweight, independently +/// named/provenanced references fan out. Results retain the caller's order. +pub fn publishRequestsAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + requests: []const PublicationRequest, + cancellation: CancellationToken, + limits: Limits, + budget: *graph_metric_policy.Budget, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + return publishRequestsWithPriorAlloc(alloc, artifacts, requests, &.{}, cancellation, limits, budget, runtime); +} + +/// The previous manifest is the durable admission-plan witness. Compare the +/// complete ordered plan, not just dirty metrics: removing/changing a sibling +/// can make a previously rejected computation affordable. Provenance alone is +/// not a plan change, and stable rejections must not turn into retry hot loops. +fn admissionPlanUnchanged(alloc: Allocator, requests: []const PublicationRequest, previous: []const artifact_ref.ArtifactRef, limits: Limits) !bool { + var index: usize = 0; + for (previous) |prior| { + if (prior.kind != .graph_metric_segment) continue; + if (index == requests.len) return false; + const request = requests[index]; + const name = try artifactNameAlloc(alloc, request.graph_index_name, request.config.name); + defer alloc.free(name); + if (!std.mem.eql(u8, name, prior.name) or !priorIdentifiesComputation(prior, request, limits)) return false; + index += 1; + } + return index == requests.len; +} + +/// Publication provenance and authenticated computation identity are separate. +/// A zero identity disables cross-source reuse, never source authentication. +pub fn metricSourceMatches(header: anytype, graph: artifact_ref.ArtifactRef, metric: artifact_ref.ArtifactRef) bool { + const zero: [32]u8 = @splat(0); + if (!std.mem.eql(u8, &metric.graph_metric_topology_checksum, &zero)) { + const source = artifact_store.sha256DigestFromChecksum(graph.checksum) catch return false; + return std.mem.eql(u8, &header.topology_checksum, &metric.graph_metric_topology_checksum) and + std.mem.eql(u8, &source, &metric.graph_metric_source_checksum); + } + return std.mem.eql(u8, &header.topology_checksum, &zero) and + std.mem.eql(u8, header.source_graph_artifact_id, graph.artifact_id) and + std.mem.eql(u8, header.source_graph_checksum, graph.checksum); +} + +fn priorIdentifiesComputation(prior: artifact_ref.ArtifactRef, request: PublicationRequest, limits: Limits) bool { + const source_checksum = artifact_store.sha256DigestFromChecksum(request.source_graph.checksum) catch return false; + return prior.kind == .graph_metric_segment and prior.metadata_version == metric_segment.wire_version and + prior.byte_len > 0 and prior.byte_len <= limits.max_metric_payload_bytes and + prior.materializer_fingerprint == materializerFingerprint(limits) and + prior.graph_metric_config_fingerprint == configFingerprint(request.config) and + std.mem.eql(u8, &prior.graph_metric_source_checksum, &source_checksum); +} + +/// One authenticated header per immutable prior payload, shared by aliases. +/// Failed verification is memoized too, so missing/corrupt aliases cannot +/// multiply origin requests before the canonical rebuild plan takes over. +const PriorInventory = struct { + const Entry = struct { ref: artifact_ref.ArtifactRef, prefix: ?[]u8 }; + budget: *graph_metric_policy.Budget, + entries: std.ArrayListUnmanaged(Entry) = .empty, + + fn deinit(self: *PriorInventory, alloc: Allocator) void { + for (self.entries.items) |entry| if (entry.prefix) |prefix| alloc.free(prefix); + self.entries.deinit(alloc); + } + + fn header(self: *PriorInventory, alloc: Allocator, artifacts: *artifact_store.ArtifactStore, prior: artifact_ref.ArtifactRef, request: PublicationRequest, cancellation: CancellationToken) !?metric_segment.codec.Header { + for (self.entries.items) |entry| if (sameSource(entry.ref, prior)) { + return if (entry.prefix) |prefix| metric_segment.decodeHeader(prefix) catch null else null; + }; + try self.entries.ensureUnusedCapacity(alloc, 1); + var remaining = self.budget.limits.max_total_reuse_read_bytes -| self.budget.reuse_read_bytes; + const before = remaining; + defer self.budget.reuse_read_bytes += before - remaining; + const prefix = read: { + _ = request; + // A reused artifact can retain original-source strings of a + // different length. Its authenticated control length is authoritative. + const len = @min(prior.byte_len, prior.graph_metric_control_len); + // The range verifier authenticates and pins the full object on a + // cold miss. A separate verify call would duplicate provider HEADs. + break :read artifacts.getVerifiedRangeAllocWithBudget(alloc, prior.artifact_id, prior.byte_len, prior.checksum, 0, len, cancellation, &remaining) catch |err| switch (err) { + error.ArtifactReadBudgetExceeded, error.FileNotFound, error.InvalidArtifactId, error.InvalidRange, error.ArtifactIntegrityMismatch, error.ArtifactIdentityUnavailable => null, + else => return err, + }; + }; + self.entries.appendAssumeCapacity(.{ .ref = prior, .prefix = prefix }); + return if (prefix) |bytes| metric_segment.decodeHeader(bytes) catch null else null; + } + + fn find(self: *PriorInventory, alloc: Allocator, artifacts: *artifact_store.ArtifactStore, previous: []const artifact_ref.ArtifactRef, request: PublicationRequest, reuse_rejections: bool, limits: Limits, cancellation: CancellationToken) !?artifact_ref.ArtifactRef { + const name = try artifactNameAlloc(alloc, request.graph_index_name, request.config.name); + defer alloc.free(name); + // A valid ready computation always wins over a terminal rejection. + for ([_]artifact_ref.GraphMetricMaterializationState{ .ready, .rejected }) |state| { + if (state == .rejected and !reuse_rejections) continue; + for ([_]bool{ true, false }) |same_name| { + for (previous) |prior| { + try cancellation.check(); + if (std.mem.eql(u8, name, prior.name) != same_name) continue; + if (prior.graph_metric_materialization_state != state or !priorIdentifiesComputation(prior, request, limits)) continue; + const decoded = (try self.header(alloc, artifacts, prior, request, cancellation)) orelse continue; + if (decoded.version != metric_segment.wire_version or decoded.kind != request.config.kind or + decoded.config_fingerprint != configFingerprint(request.config) or + decoded.materializer_fingerprint != materializerFingerprint(limits) or + @intFromEnum(decoded.materialization_state) != @intFromEnum(state) or + @intFromEnum(decoded.rejection_reason) != @intFromEnum(prior.graph_metric_rejection_reason) or + !metricSourceMatches(decoded, request.source_graph, prior)) continue; + return prior; + } + } + } + return null; + } + + fn findSemantic(self: *PriorInventory, alloc: Allocator, artifacts: *artifact_store.ArtifactStore, previous: []const artifact_ref.ArtifactRef, request: PublicationRequest, digest: [32]u8, limits: Limits, cancellation: CancellationToken) !?artifact_ref.ArtifactRef { + for (previous) |prior| { + try cancellation.check(); + if (prior.kind != .graph_metric_segment or prior.metadata_version != metric_segment.wire_version or + prior.graph_metric_materialization_state != .ready or + prior.graph_metric_config_fingerprint != configFingerprint(request.config) or + prior.materializer_fingerprint != materializerFingerprint(limits) or prior.byte_len == 0 or + prior.byte_len > limits.max_metric_payload_bytes or !std.mem.eql(u8, &prior.graph_metric_topology_checksum, &digest)) continue; + const decoded = (try self.header(alloc, artifacts, prior, request, cancellation)) orelse continue; + if (decoded.version != metric_segment.wire_version or decoded.kind != request.config.kind or + decoded.materialization_state != .ready or decoded.config_fingerprint != configFingerprint(request.config) or + decoded.materializer_fingerprint != materializerFingerprint(limits) or + !std.mem.eql(u8, &decoded.topology_checksum, &digest)) continue; + return prior; + } + return null; + } +}; + +const TopologyDirectory = struct { bytes: []u8, checksum: [32]u8 }; + +/// Authenticate the small directory before source-wide preparation. The range +/// store pins the exact SHA-256 object identity; cold full authentication is +/// charged to the same reuse-read allowance as prior metric control reads. +fn readTopologyDirectoryAlloc(alloc: Allocator, artifacts: *artifact_store.ArtifactStore, source: artifact_ref.ArtifactRef, budget: *graph_metric_policy.Budget, cancellation: CancellationToken) !?TopologyDirectory { + const wire = graph_segment.codec.compact; + if (source.byte_len < wire.topology_trailer_len or source.byte_len > budget.limits.max_graph_payload_bytes or + budget.limits.max_peak_memory_bytes < wire.topology_trailer_len or + budget.identity_work_bytes >= budget.limits.max_total_identity_work_bytes) return null; + var remaining = budget.limits.max_total_reuse_read_bytes -| budget.reuse_read_bytes; + const before = remaining; + defer budget.reuse_read_bytes += before - remaining; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, budget.limits.max_peak_memory_bytes); + var context = indexed_topology.Context.init(limiter.allocator(), artifacts, source, cancellation, &remaining) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded, error.FileNotFound, error.InvalidArtifactId, error.InvalidRange, error.InvalidGraphSegment, error.ArtifactIntegrityMismatch, error.ArtifactIdentityUnavailable => return null, + error.OutOfMemory => if (limiter.limit_exceeded) return null else return err, + else => return err, + }; + if (context.trailer.source_nodes > budget.limits.max_nodes or context.trailer.source_edges > budget.limits.max_edges or context.directory == null) { + context.deinit(); + return null; + } + alloc.free(context.block_bytes); + return .{ .bytes = context.bytes, .checksum = context.trailer.checksum }; +} + +pub fn publishRequestsWithPriorAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + requests: []const PublicationRequest, + previous: []const artifact_ref.ArtifactRef, + cancellation: CancellationToken, + limits: Limits, + budget: *graph_metric_policy.Budget, + runtime: ComputeRuntime, +) ![]artifact_ref.ArtifactRef { + try graph_metric_policy.validateCatalogFanout(0, requests.len, limits); + if (!std.meta.eql(budget.limits, limits)) return error.InvalidGraphMetricBuildOptions; + try runtime.validate(); + for (requests) |request| { + try request.provenance.validate(); + try graph_metric_policy.validateConfigs(&.{request.config}, limits); + try graph_mod.validateGraphMetricEdgeFilters(&.{}, &.{request.config}); + if (request.graph_index_name.len == 0 or request.graph_index_name.len > limits.max_graph_index_name_bytes or + request.source_graph.kind != .graph_segment or request.source_graph.byte_len == 0) + return error.InvalidGraphMetricBuildOptions; + } + const refs = try alloc.alloc(artifact_ref.ArtifactRef, requests.len); + errdefer alloc.free(refs); + const ready = try alloc.alloc(bool, requests.len); + defer alloc.free(ready); + @memset(ready, false); + errdefer for (refs, ready) |ref, initialized| if (initialized) freeArtifactRef(alloc, ref); + const reuse_rejections = try admissionPlanUnchanged(alloc, requests, previous, limits); + var inventory = PriorInventory{ .budget = budget }; + defer inventory.deinit(alloc); + for (requests, refs, ready) |request, *ref, *initialized| { + if (try inventory.find(alloc, artifacts, previous, request, reuse_rejections, limits, cancellation)) |prior| { + ref.* = try aliasRefAlloc(alloc, prior, request.graph_index_name, request.config.name, request.provenance); + // A new alias publishes now but did not recompute the existing + // vector. Keep its real computation timestamp. + if (prior.computed_at_ms != 0) ref.computed_at_ms = prior.computed_at_ms; + if (std.mem.eql(u8, prior.name, ref.name) and prior.published_generation >= request.provenance.edge_generation) + ref.published_generation = prior.published_generation; + initialized.* = true; + } + } + var configs = std.ArrayListUnmanaged(graph_mod.GraphMetricConfig).empty; + defer configs.deinit(alloc); + var priors = std.ArrayListUnmanaged(?artifact_ref.ArtifactRef).empty; + defer priors.deinit(alloc); + var mapping = std.ArrayListUnmanaged(usize).empty; + defer mapping.deinit(alloc); + var source_context: ?indexed_topology.Context = null; + defer if (source_context) |*context| context.deinit(); + var source_remaining: u64 = 0; + while (true) { + const held_source: ?PublicationRequest = if (source_context) |context| for (requests, ready) |request, initialized| { + if (!initialized and sameSource(context.reader.source, request.source_graph)) break request; + } else null else null; + var first = held_source orelse for (requests, ready) |request, initialized| { + if (!initialized) break request; + } else break; + try cancellation.check(); + // A source is prepared only if at least one computation still needs + // it after exact-source and selected-topology reuse. Keep the directory + // request-local: no O(number of source artifacts) metadata cache. + if ((previous.len > 0 or first.prior_artifact != null) and + (source_context == null or !sameSource(source_context.?.reader.source, first.source_graph))) + { + if (try readTopologyDirectoryAlloc(alloc, artifacts, first.source_graph, budget, cancellation)) |directory| { + defer alloc.free(directory.bytes); + for (requests, refs, ready) |request, *ref, *initialized| { + if (initialized.* or !sameSource(first.source_graph, request.source_graph)) continue; + const comparisons: u64 = if (request.config.edge_filter.mode == .all) 1 else @as(u64, request.config.edge_filter.types.len) + 1; + const identity_work = std.math.mul(u64, directory.bytes.len, comparisons) catch continue; + if (identity_work > limits.max_total_identity_work_bytes -| budget.identity_work_bytes) continue; + budget.identity_work_bytes += identity_work; + const digest = (try graph_segment.codec.compact.selectedDirectoryChecksum(directory.bytes, directory.checksum, request.config.edge_filter)) orelse continue; + const prior = (try inventory.findSemantic(alloc, artifacts, previous, request, digest, limits, cancellation)) orelse + (if (request.prior_artifact) |candidate| try inventory.findSemantic(alloc, artifacts, &.{candidate}, request, digest, limits, cancellation) else null) orelse continue; + ref.* = try aliasRefAlloc(alloc, prior, request.graph_index_name, request.config.name, request.provenance); + initialized.* = true; + ref.computed_at_ms = prior.computed_at_ms; + ref.graph_metric_source_checksum = try artifact_store.sha256DigestFromChecksum(first.source_graph.checksum); + } + } + } + const pending_source = for (requests, ready) |request, initialized| { + if (!initialized and sameSource(first.source_graph, request.source_graph)) break true; + } else false; + if (!pending_source) continue; + // Admission is per filter, not the union of unrelated computations. + // Retain only one bounded authenticated source control, and prioritize + // the smallest selected topology before spending shared data-read IO. + if (source_context) |*context| if (!sameSource(context.reader.source, first.source_graph)) { + context.deinit(); + source_context = null; + }; + if (source_context == null and first.source_graph.byte_len <= limits.max_graph_payload_bytes) { + source_remaining = limits.max_total_graph_payload_bytes -| budget.graph_payload_bytes; + const before = source_remaining; + defer budget.graph_payload_bytes += @intCast(before - source_remaining); + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes); + source_context = indexed_topology.Context.init(limiter.allocator(), artifacts, first.source_graph, cancellation, &source_remaining) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => null, + error.OutOfMemory => if (limiter.limit_exceeded) null else return err, + else => return err, + }; + if (source_context) |*context| context.reader.alloc = alloc; + } + const split_filters = source_context != null and source_context.?.directory != null; + var cheapest: u64 = std.math.maxInt(u64); + for (requests, ready) |request, initialized| { + if (initialized or !sameSource(first.source_graph, request.source_graph)) continue; + var cost: u64 = 0; + if (source_context) |context| if (context.directory) |directory| { + var entries = directory.iterator(); + while (try entries.next()) |entry| { + if (request.config.edge_filter.mode == .all or request.config.edge_filter.includesType(entry.kind)) cost += entry.edges; + } + }; + if (cost < cheapest) { + cheapest = cost; + first = request; + } + } + configs.clearRetainingCapacity(); + priors.clearRetainingCapacity(); + mapping.clearRetainingCapacity(); + for (requests, ready) |request, initialized| { + if (initialized or !sameSource(first.source_graph, request.source_graph) or + (split_filters and !request.config.edge_filter.equivalent(first.config.edge_filter))) continue; + const existing = for (configs.items, 0..) |config, i| { + if (sameComputation(config, request.config)) break i; + } else null; + const index = existing orelse configs.items.len; + if (existing == null) { + var config = request.config; + config.refresh = .background; + try configs.append(alloc, config); + try priors.append(alloc, request.prior_artifact); + } else if (priors.items[index] == null) { + // The first available seed in stable publication order wins. + // Seed authentication/compatibility is still mandatory. + priors.items[index] = request.prior_artifact; + } + try mapping.append(alloc, index); + } + if (configs.items.len == 0) continue; + const built = build: { + var prepared = prepare: { + if (first.source_graph.byte_len > limits.max_graph_payload_bytes) break :prepare null; + const context = if (source_context) |*value| value else break :prepare null; + source_remaining = limits.max_total_graph_payload_bytes -| budget.graph_payload_bytes; + const before = source_remaining; + defer budget.graph_payload_bytes += @intCast(before - source_remaining); + if (prepareContextGraphAlloc(alloc, context, first.source_graph, configs.items, cancellation, limits) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => break :prepare null, + else => return err, + }) |selected| break :prepare selected; + budget.chargeGraphPayload(first.source_graph.artifact_id, first.source_graph.checksum, first.source_graph.byte_len) catch break :prepare null; + var fallback_limits = limits; + fallback_limits.max_peak_memory_bytes -= context.retainedBytes(); + var fallback: ?PreparedGraphArtifact = prepareGraphArtifactAlloc(alloc, artifacts, first.source_graph, cancellation, fallback_limits) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => null, + else => return err, + }; + if (fallback) |*graph| graph.topology.retained_bytes += context.retainedBytes(); + break :prepare fallback; + }; + if (prepared) |*graph| { + defer graph.deinit(alloc); + break :build try publishPreparedComputationsAlloc(alloc, artifacts, first.graph_index_name, first.source_graph, configs.items, priors.items, cancellation, limits, budget, graph, first.provenance, runtime); + } + const rejected = try alloc.alloc(artifact_ref.ArtifactRef, configs.items.len); + errdefer alloc.free(rejected); + var count: usize = 0; + errdefer for (rejected[0..count]) |ref| freeArtifactRef(alloc, ref); + for (configs.items, rejected) |config, *ref| { + ref.* = try publishRejectedAlloc(alloc, artifacts, first.graph_index_name, first.source_graph, config, cancellation, .build_budget_exceeded, limits, first.provenance); + count += 1; + } + break :build rejected; + }; + defer { + for (built) |ref| freeArtifactRef(alloc, ref); + alloc.free(built); + } + var mapped: usize = 0; + for (requests, ready, refs) |request, *initialized, *ref| { + if (initialized.* or !sameSource(first.source_graph, request.source_graph) or + (split_filters and !request.config.edge_filter.equivalent(first.config.edge_filter))) continue; + ref.* = try aliasRefAlloc(alloc, built[mapping.items[mapped]], request.graph_index_name, request.config.name, request.provenance); + initialized.* = true; + mapped += 1; + } + } + return refs; +} + +fn validatePublicationOptions( + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + limits: Limits, + batch_budget: *graph_metric_policy.Budget, +) !void { + try cancellation.check(); + try graph_metric_policy.validateConfigs(configs, limits); + if (!std.meta.eql(batch_budget.limits, limits)) return error.InvalidGraphMetricBuildOptions; + if (graph_index_name.len == 0 or source_graph.kind != .graph_segment or source_graph.byte_len == 0) return error.InvalidGraphMetricBuildOptions; + if (source_graph.byte_len > limits.max_graph_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + try graph_mod.validateGraphMetricEdgeFilters(&.{}, configs); +} + +fn putBuildResultAlloc(alloc: Allocator, artifacts: *artifact_store.ArtifactStore, built: *const BuildResult, cancellation: CancellationToken) !artifact_ref.ArtifactRef { + var metadata = try artifacts.putWithCancellation(built.payload, cancellation); + defer metadata.deinit(alloc); + const name = try alloc.dupe(u8, built.artifact.name); + errdefer alloc.free(name); + const artifact_id = try alloc.dupe(u8, metadata.artifact_id); + errdefer alloc.free(artifact_id); + const ref = artifact_ref.ArtifactRef{ + .kind = .graph_metric_segment, + .name = name, + .artifact_id = artifact_id, + .byte_len = metadata.byte_len, + .checksum = try alloc.dupe(u8, metadata.checksum), + .metadata_version = built.artifact.metadata_version, + .published_generation = built.artifact.published_generation, + .edge_generation = built.artifact.edge_generation, + .computed_at_ms = built.artifact.computed_at_ms, + .materializer_fingerprint = built.artifact.materializer_fingerprint, + .graph_metric_control_len = built.artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = built.artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = built.artifact.graph_metric_control_checksum, + .graph_metric_routing_checksum = built.artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = built.artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = built.artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = built.artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = built.artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = built.artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = built.artifact.graph_metric_rejection_reason, + }; + return ref; +} + +fn findCompatibleHitsPairIndex(configs: []const graph_mod.GraphMetricConfig, processed: []const bool, index: usize) ?usize { + if (configs.len != processed.len) return null; + if (graph_mod.graphMetricOppositeHitsKind(configs[index].kind) == null) return null; + for (configs, 0..) |candidate, candidate_index| { + if (candidate_index == index or candidate_index < index or processed[candidate_index]) continue; + if (graph_mod.graphMetricHitsPairCompatible(configs[index], candidate)) return candidate_index; + } + return null; +} + +/// Publishes durable terminal status sidecars when the source graph itself is +/// outside the materializer's admission budget. These sidecars preserve base +/// publication, prevent retry hot loops, and let queries return an actionable +/// rejection instead of pretending the configured metric does not exist. +pub fn publishRejectedManyAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + configs: []const graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + reason: metric_segment.RejectionReason, + limits: Limits, + provenance: Provenance, +) ![]artifact_ref.ArtifactRef { + try provenance.validate(); + try graph_metric_policy.validateConfigs(configs, limits); + const refs = try alloc.alloc(artifact_ref.ArtifactRef, configs.len); + var initialized: usize = 0; + errdefer { + for (refs[0..initialized]) |ref| freeArtifactRef(alloc, ref); + alloc.free(refs); + } + for (configs, 0..) |config, i| { + try cancellation.check(); + refs[i] = try publishRejectedAlloc(alloc, artifacts, graph_index_name, source_graph, config, cancellation, reason, limits, provenance); + initialized += 1; + } + return refs; +} + +fn publishRejectedAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + graph_index_name: []const u8, + source_graph: artifact_ref.ArtifactRef, + config: graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + reason: metric_segment.RejectionReason, + limits: Limits, + provenance: Provenance, +) !artifact_ref.ArtifactRef { + try cancellation.check(); + var segment = try rejectedSegmentAlloc(alloc, source_graph, config, reason, limits, provenance); + defer segment.deinit(alloc); + const payload = try metric_segment.encodeAllocWithCancellation(alloc, segment, cancellation); + defer alloc.free(payload); + var metadata = try artifacts.putWithCancellation(payload, cancellation); + defer metadata.deinit(alloc); + const name = try metric_segment.artifactNameAlloc(alloc, graph_index_name, config.name); + errdefer alloc.free(name); + const artifact_id = try alloc.dupe(u8, metadata.artifact_id); + errdefer alloc.free(artifact_id); + const checksum = try alloc.dupe(u8, metadata.checksum); + var ref = artifact_ref.ArtifactRef{ + .kind = .graph_metric_segment, + .name = name, + .artifact_id = artifact_id, + .byte_len = metadata.byte_len, + .checksum = checksum, + .metadata_version = metric_segment.wire_version, + .published_generation = provenance.published_generation, + .edge_generation = provenance.edge_generation, + .computed_at_ms = provenance.computed_at_ms, + .materializer_fingerprint = segment.materializer_fingerprint, + }; + try populateGraphMetricIntegrity(&ref, segment, payload); + return ref; +} + +fn rejectedSegmentAlloc( + alloc: Allocator, + source_graph: artifact_ref.ArtifactRef, + config: graph_mod.GraphMetricConfig, + reason: metric_segment.RejectionReason, + limits: Limits, + provenance: Provenance, +) !metric_segment.Segment { + const owned_source_graph_artifact_id = try alloc.dupe(u8, source_graph.artifact_id); + errdefer alloc.free(owned_source_graph_artifact_id); + const owned_source_graph_checksum = try alloc.dupe(u8, source_graph.checksum); + errdefer alloc.free(owned_source_graph_checksum); + var edge_filter = try cloneSortedEdgeFilterAlloc(alloc, config.edge_filter); + errdefer edge_filter.deinit(alloc); + const scores = try alloc.alloc(metric_segment.Score, 0); + errdefer alloc.free(scores); + const segment = metric_segment.Segment{ + .kind = config.kind, + .source_graph_artifact_id = owned_source_graph_artifact_id, + .source_graph_checksum = owned_source_graph_checksum, + .config_fingerprint = configFingerprint(config), + .materializer_fingerprint = graph_metric_policy.materializerFingerprint(limits), + .published_generation = provenance.published_generation, + .edge_generation = provenance.edge_generation, + .computed_at_ms = provenance.computed_at_ms, + .materialization_state = .rejected, + .rejection_reason = reason, + .edge_filter = edge_filter, + .converged = false, + .iterations_completed = 0, + .delta = 0, + .scores = scores, + }; + return segment; +} + +const Projection = struct { + ordinals: std.StringHashMapUnmanaged(usize) = .empty, + node_ids: std.ArrayListUnmanaged([]const u8) = .empty, + topology: ?metrics.Topology = null, + source_node_count: usize = 0, + source_edge_count: usize = 0, + census_work: ?u64 = null, + node_id_bytes: usize = 0, + decoded_retained_bytes: usize = 0, + + fn deinit(self: *Projection, alloc: Allocator) void { + self.ordinals.deinit(alloc); + self.node_ids.deinit(alloc); + if (self.topology) |*topology| topology.deinit(alloc); + self.* = .{}; + } + + fn edgeCount(self: Projection) usize { + return if (self.topology) |topology| topology.edgeCount() else 0; + } +}; + +const EdgeFilterIndex = struct { + all: bool, + types: std.StringHashMapUnmanaged(void) = .empty, + + fn init(alloc: Allocator, filter: graph_mod.GraphMetricEdgeFilter) !EdgeFilterIndex { + var self = EdgeFilterIndex{ .all = filter.mode == .all }; + errdefer self.deinit(alloc); + if (!self.all) { + const capacity = std.math.cast(std.StringHashMapUnmanaged(void).Size, filter.types.len) orelse + return error.GraphMetricConfigurationLimitExceeded; + try self.types.ensureTotalCapacity(alloc, capacity); + for (filter.types) |edge_type| self.types.putAssumeCapacity(edge_type, {}); + } + return self; + } + + fn deinit(self: *EdgeFilterIndex, alloc: Allocator) void { + self.types.deinit(alloc); + self.* = undefined; + } + + fn allows(self: EdgeFilterIndex, edge_type: []const u8) bool { + return self.all or self.types.contains(edge_type); + } +}; + +/// Directly consumes persisted ordinals. No edge-string allocations, node +/// hashing, or reverse-adjacency materialization occur in this path. +fn prepareTopologyFromPackedAlloc(alloc: Allocator, payload: []const u8, cancellation: CancellationToken, limits: Limits) !CompiledTopology { + if (payload.len >= limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes - payload.len); + return compilePackedTopologyAlloc(limiter.allocator(), payload, cancellation, limits) catch |err| { + if ((err == error.OutOfMemory and limiter.limit_exceeded) or err == error.DecodedArtifactTooLarge) + return error.GraphMetricBuildBudgetExceeded; + return err; + }; +} + +fn compilePackedTopologyAlloc(alloc: Allocator, payload: []const u8, cancellation: CancellationToken, limits: Limits) !CompiledTopology { + const compact = graph_segment.codec.compact; + var graph = try compact.viewAlloc(alloc, payload, .{}, cancellation); + defer graph.deinit(alloc); + if (graph.adjacencies.len > limits.max_nodes) return error.GraphMetricBuildBudgetExceeded; + const missing = std.math.maxInt(u32); + const ordinals = try alloc.alloc(u32, graph.nodes.len); + defer alloc.free(ordinals); + @memset(ordinals, missing); + var node_ids = std.ArrayListUnmanaged([]const u8).empty; + errdefer node_ids.deinit(alloc); + try node_ids.ensureTotalCapacity(alloc, graph.adjacencies.len); + for (graph.adjacencies, 0..) |adjacency, i| { + if (i % 256 == 0) try cancellation.check(); + if (ordinals[adjacency.node] != missing) return error.InvalidGraphMetricBuildOptions; + ordinals[adjacency.node] = @intCast(i); + node_ids.appendAssumeCapacity(graph.nodes[adjacency.node]); + } + var edge_types = std.ArrayListUnmanaged([]const u8).empty; + errdefer edge_types.deinit(alloc); + try edge_types.appendSlice(alloc, graph.edge_types); + const edge_type_offsets = try alloc.alloc(u32, graph.edge_types.len + 1); + errdefer alloc.free(edge_type_offsets); + @memset(edge_type_offsets, 0); + var source_edge_count: usize = 0; + var local_edge_count: usize = 0; + for (graph.adjacencies) |adjacency| { + source_edge_count = std.math.add(usize, source_edge_count, adjacency.out.len / compact.edge_len) catch return error.GraphMetricBuildBudgetExceeded; + if (source_edge_count > limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + for (0..adjacency.out.len / compact.edge_len) |i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = compact.readEdge(adjacency.out, i); + if (edge.table != null) continue; + if (ordinals[edge.node] == missing) return error.InvalidGraphMetricBuildOptions; + edge_type_offsets[edge.edge_type + 1] = std.math.add(u32, edge_type_offsets[edge.edge_type + 1], 1) catch return error.GraphMetricBuildBudgetExceeded; + local_edge_count += 1; + } + } + for (1..edge_type_offsets.len) |i| edge_type_offsets[i] = std.math.add(u32, edge_type_offsets[i - 1], edge_type_offsets[i]) catch return error.GraphMetricBuildBudgetExceeded; + const owned_edges = try alloc.alloc(CompiledEdge, local_edge_count); + errdefer alloc.free(owned_edges); + const cursors = try alloc.dupe(u32, edge_type_offsets[0..graph.edge_types.len]); + defer alloc.free(cursors); + for (graph.adjacencies, 0..) |adjacency, source| { + for (0..adjacency.out.len / compact.edge_len) |i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = compact.readEdge(adjacency.out, i); + if (edge.table != null) continue; + owned_edges[cursors[edge.edge_type]] = .{ .source = @intCast(source), .target = ordinals[edge.node] }; + cursors[edge.edge_type] += 1; + } + } + const owned_node_ids = try node_ids.toOwnedSlice(alloc); + errdefer alloc.free(owned_node_ids); + const owned_edge_types = try edge_types.toOwnedSlice(alloc); + errdefer alloc.free(owned_edge_types); + var string_bytes_len: usize = 0; + for (owned_node_ids, 0..) |node_id, i| { + if (i % 256 == 0) try cancellation.check(); + string_bytes_len = std.math.add(usize, string_bytes_len, node_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + for (owned_edge_types, 0..) |edge_type, i| { + if (i % 256 == 0) try cancellation.check(); + string_bytes_len = std.math.add(usize, string_bytes_len, edge_type.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + const string_bytes = try alloc.alloc(u8, string_bytes_len); + errdefer alloc.free(string_bytes); + var string_offset: usize = 0; + for (owned_node_ids, 0..) |*node_id, i| { + if (i % 256 == 0) try cancellation.check(); + const len = node_id.*.len; + @memcpy(string_bytes[string_offset .. string_offset + len], node_id.*); + node_id.* = string_bytes[string_offset .. string_offset + len]; + string_offset += len; + } + for (owned_edge_types, 0..) |*edge_type, i| { + if (i % 256 == 0) try cancellation.check(); + const len = edge_type.*.len; + @memcpy(string_bytes[string_offset .. string_offset + len], edge_type.*); + edge_type.* = string_bytes[string_offset .. string_offset + len]; + string_offset += len; + } + std.debug.assert(string_offset == string_bytes.len); + var retained_bytes: usize = 0; + try addPeakArrayBytes(&retained_bytes, owned_node_ids.len, []u8); + try addPeakArrayBytes(&retained_bytes, owned_edge_types.len, []u8); + try addPeakBytes(&retained_bytes, string_bytes.len); + try addPeakArrayBytes(&retained_bytes, edge_type_offsets.len, u32); + try addPeakArrayBytes(&retained_bytes, owned_edges.len, CompiledEdge); + return .{ + .node_ids = owned_node_ids, + .edge_types = owned_edge_types, + .string_bytes = string_bytes, + .edge_type_offsets = edge_type_offsets, + .edges = owned_edges, + .source_node_count = graph.adjacencies.len, + .source_edge_count = source_edge_count, + .retained_bytes = retained_bytes, + }; +} + +/// Benchmark/reference oracle only. Both paths read the same current wire; +/// the reference intentionally recreates the former unpack/hash preparation. +pub const SelectedPreparationBenchmark = struct { edges: usize, retained_nodes: usize, read_bytes: usize, digest: [32]u8 }; + +/// Source preparation plus the semantic identity step performed by publication. +/// Both paths must produce exactly the same selected topology digest. +pub fn benchmarkSelectedArtifactPreparation(alloc: Allocator, artifacts: *artifact_store.ArtifactStore, source: artifact_ref.ArtifactRef, config: graph_mod.GraphMetricConfig, reference: bool) !SelectedPreparationBenchmark { + var budget = graph_metric_policy.Budget{ .limits = .{} }; + var prepared = if (reference) blk: { + try budget.chargeGraphPayload(source.artifact_id, source.checksum, source.byte_len); + break :blk try prepareGraphArtifactAlloc(alloc, artifacts, source, .none, budget.limits); + } else (try prepareSelectedGraphArtifactAlloc(alloc, artifacts, source, &.{config}, .none, budget.limits, &budget)) orelse return error.InvalidBenchmarkResult; + defer prepared.deinit(alloc); + const checksums = if (!reference and prepared.topology.type_checksums.len == prepared.topology.edge_types.len) + try alloc.dupe([32]u8, prepared.topology.type_checksums) + else + try typeChecksumsAlloc(alloc, prepared.topology, budget.limits, .none); + defer alloc.free(checksums); + return .{ .edges = try selectedEdgeCount(prepared.topology, config.edge_filter, .none), .retained_nodes = prepared.topology.node_ids.len, .read_bytes = budget.graph_payload_bytes, .digest = selectedTopologyChecksum(prepared.topology, checksums, config.edge_filter) }; +} + +pub fn benchmarkPreparation(alloc: Allocator, payload: []const u8, reference: bool) !usize { + if (!reference) { + var topology = try prepareTopologyFromPackedAlloc(alloc, payload, .none, .{}); + defer topology.deinit(alloc); + return topology.edges.len; + } + var graph = try graph_segment.decodeAlloc(alloc, payload); + defer graph.deinit(alloc); + _ = try discardInboundEdges(alloc, &graph); + var topology = try compileTopologyWithinBudgetAlloc(alloc, graph, 0, 1024 * 1024 * 1024, .none); + defer topology.deinit(alloc); + return topology.edges.len; +} + +/// Source preparation plus sixteen exhausted projection groups. The oracle +/// models the former build-then-charge ordering; neither path runs a kernel. +pub fn benchmarkRejectedPreparation(alloc: Allocator, payload: []const u8, reference: bool) !usize { + var topology = try prepareTopologyFromPackedAlloc(alloc, payload, .none, .{}); + defer topology.deinit(alloc); + var budget = graph_metric_policy.Budget{ .limits = .{ .max_total_work_items = 0 } }; + const options = BuildOptions{ + .graph_index_name = "bench", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "fixture", .byte_len = payload.len }, + }; + for (0..16) |_| { + if (reference) { + var projection = try buildAdmittedProjectionFromTopologyAlloc(alloc, topology, 0, options); + projection.deinit(alloc); + } else { + var bounded = options; + bounded.batch_budget = &budget; + var projection = buildProjectionFromTopologyAlloc(alloc, topology, 0, bounded) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => continue, + else => return err, + }; + projection.deinit(alloc); + return error.InvalidBenchmarkResult; + } + } + return topology.edges.len; +} + +/// Includes one source/projection preparation. The reference executes and +/// encodes PageRank before discovering an exhausted output quota. +pub fn benchmarkRejectedOutput(alloc: Allocator, payload: []const u8, reference: bool) !usize { + var topology = try prepareTopologyFromPackedAlloc(alloc, payload, .none, .{}); + defer topology.deinit(alloc); + var options = BuildOptions{ + .graph_index_name = "bench", + .config = .{ .name = "rank", .kind = .pagerank, .max_iterations = 3 }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "a" ** 64, .byte_len = payload.len }, + }; + var projection = try buildProjectionFromTopologyAlloc(alloc, topology, 0, options); + defer projection.deinit(alloc); + if (reference) { + var built = try buildAdmittedProjectionAlloc(alloc, projection, options); + built.deinit(alloc); + } else { + var budget = graph_metric_policy.Budget{ .limits = .{ .max_total_metric_payload_bytes = 0 } }; + options.batch_budget = &budget; + var built = buildFromProjectionAlloc(alloc, projection, options) catch |err| switch (err) { + error.GraphMetricBuildBudgetExceeded => return topology.edges.len, + else => return err, + }; + built.deinit(alloc); + return error.InvalidBenchmarkResult; + } + return topology.edges.len; +} + +fn compileTopologyWithinBudgetAlloc( + alloc: Allocator, + graph: graph_segment.Segment, + decoded_retained_bytes: usize, + max_peak_memory_bytes: usize, + cancellation: CancellationToken, +) !CompiledTopology { + if (decoded_retained_bytes >= max_peak_memory_bytes) + return error.GraphMetricBuildBudgetExceeded; + // Enforce the configured ceiling against allocations actually made by + // compilation. This admits graphs with many edges but few distinct edge + // types without relying on std hash-map layout guesses or O(E) worst-case + // string estimates. The decoded graph remains charged as retained memory. + var limiter = bounded_decode.AllocationLimiter.init( + alloc, + max_peak_memory_bytes - decoded_retained_bytes, + ) catch return error.InvalidGraphMetricBuildOptions; + return compileTopologyAlloc(limiter.allocator(), graph, cancellation) catch |err| switch (err) { + error.OutOfMemory => if (limiter.limit_exceeded) + error.GraphMetricBuildBudgetExceeded + else + error.OutOfMemory, + else => err, + }; +} + +fn compileTopologyAlloc( + alloc: Allocator, + graph: graph_segment.Segment, + cancellation: CancellationToken, +) !CompiledTopology { + var ordinals = std.StringHashMapUnmanaged(u32).empty; + defer ordinals.deinit(alloc); + // Keys borrow the decoded graph only during compilation. The final slices + // are rebound into one packed allocation below, avoiding O(V + T) heap + // allocations in long-lived prepared artifacts. + var node_ids = std.ArrayListUnmanaged([]const u8).empty; + errdefer node_ids.deinit(alloc); + var edge_type_ordinals = std.StringHashMapUnmanaged(u32).empty; + defer edge_type_ordinals.deinit(alloc); + var edge_types = std.ArrayListUnmanaged([]const u8).empty; + errdefer edge_types.deinit(alloc); + var edge_type_counts = std.ArrayListUnmanaged(u32).empty; + defer edge_type_counts.deinit(alloc); + + for (graph.adjacencies, 0..) |adjacency, i| { + if (i % 256 == 0) try cancellation.check(); + if (i > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + if (ordinals.contains(adjacency.node_id)) return error.InvalidGraphMetricBuildOptions; + try node_ids.append(alloc, adjacency.node_id); + try ordinals.put(alloc, adjacency.node_id, @intCast(i)); + } + var source_edge_count: usize = 0; + var local_edge_count: usize = 0; + for (graph.adjacencies, 0..) |adjacency, adjacency_index| { + if (adjacency_index % 256 == 0) try cancellation.check(); + for (adjacency.out_edges) |edge| { + source_edge_count = std.math.add(usize, source_edge_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + if (source_edge_count % 4096 == 0) try cancellation.check(); + // A serverless graph artifact is table-scoped. Qualified endpoints + // belong to another table's artifact and cannot be represented by + // this metric segment's intentionally unqualified node-id key. + if (edge.neighbor_table_id != null) continue; + _ = ordinals.get(edge.neighbor_id) orelse return error.InvalidGraphMetricBuildOptions; + const edge_type_id = if (edge_type_ordinals.get(edge.edge_type)) |existing| + existing + else blk: { + if (edge_types.items.len > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + const id: u32 = @intCast(edge_types.items.len); + try edge_types.append(alloc, edge.edge_type); + try edge_type_counts.append(alloc, 0); + try edge_type_ordinals.put(alloc, edge.edge_type, id); + break :blk id; + }; + if (edge_type_counts.items[edge_type_id] == std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + edge_type_counts.items[edge_type_id] += 1; + local_edge_count = std.math.add(usize, local_edge_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + } + } + if (local_edge_count > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + + const edge_type_offsets = try alloc.alloc(u32, edge_types.items.len + 1); + errdefer alloc.free(edge_type_offsets); + edge_type_offsets[0] = 0; + for (edge_type_counts.items, 0..) |count, i| { + edge_type_offsets[i + 1] = std.math.add(u32, edge_type_offsets[i], count) catch + return error.GraphMetricBuildBudgetExceeded; + } + const owned_edges = try alloc.alloc(CompiledEdge, local_edge_count); + errdefer alloc.free(owned_edges); + const edge_type_cursors = try alloc.dupe(u32, edge_type_offsets[0..edge_types.items.len]); + defer alloc.free(edge_type_cursors); + var visited_edges: usize = 0; + for (graph.adjacencies) |adjacency| { + const source = ordinals.get(adjacency.node_id) orelse return error.InvalidGraphMetricBuildOptions; + for (adjacency.out_edges) |edge| { + visited_edges += 1; + if (visited_edges % 4096 == 0) try cancellation.check(); + if (edge.neighbor_table_id != null) continue; + const target = ordinals.get(edge.neighbor_id) orelse return error.InvalidGraphMetricBuildOptions; + const edge_type_id = edge_type_ordinals.get(edge.edge_type) orelse return error.InvalidGraphMetricBuildOptions; + const destination = edge_type_cursors[edge_type_id]; + if (@as(usize, destination) >= owned_edges.len) return error.InvalidGraphMetricBuildOptions; + owned_edges[destination] = .{ .source = source, .target = target }; + edge_type_cursors[edge_type_id] += 1; + } + } + + const owned_node_ids = try node_ids.toOwnedSlice(alloc); + errdefer alloc.free(owned_node_ids); + const owned_edge_types = try edge_types.toOwnedSlice(alloc); + errdefer alloc.free(owned_edge_types); + var string_bytes_len: usize = 0; + for (owned_node_ids, 0..) |node_id, i| { + if (i % 256 == 0) try cancellation.check(); + string_bytes_len = std.math.add(usize, string_bytes_len, node_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + for (owned_edge_types, 0..) |edge_type, i| { + if (i % 256 == 0) try cancellation.check(); + string_bytes_len = std.math.add(usize, string_bytes_len, edge_type.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + const string_bytes = try alloc.alloc(u8, string_bytes_len); + errdefer alloc.free(string_bytes); + var string_offset: usize = 0; + for (owned_node_ids, 0..) |*node_id, i| { + if (i % 256 == 0) try cancellation.check(); + const len = node_id.*.len; + @memcpy(string_bytes[string_offset .. string_offset + len], node_id.*); + node_id.* = string_bytes[string_offset .. string_offset + len]; + string_offset += len; + } + for (owned_edge_types, 0..) |*edge_type, i| { + if (i % 256 == 0) try cancellation.check(); + const len = edge_type.*.len; + @memcpy(string_bytes[string_offset .. string_offset + len], edge_type.*); + edge_type.* = string_bytes[string_offset .. string_offset + len]; + string_offset += len; + } + std.debug.assert(string_offset == string_bytes.len); + var retained_bytes: usize = 0; + try addPeakArrayBytes(&retained_bytes, owned_node_ids.len, []u8); + try addPeakArrayBytes(&retained_bytes, owned_edge_types.len, []u8); + try addPeakBytes(&retained_bytes, string_bytes.len); + try addPeakArrayBytes(&retained_bytes, edge_type_offsets.len, u32); + try addPeakArrayBytes(&retained_bytes, owned_edges.len, CompiledEdge); + return .{ + .node_ids = owned_node_ids, + .edge_types = owned_edge_types, + .string_bytes = string_bytes, + .edge_type_offsets = edge_type_offsets, + .edges = owned_edges, + .source_node_count = graph.adjacencies.len, + .source_edge_count = source_edge_count, + .retained_bytes = retained_bytes, + }; +} + +fn buildProjectionFromTopologyAlloc( + alloc: Allocator, + topology: CompiledTopology, + decoded_retained_bytes: usize, + options: BuildOptions, +) !Projection { + try options.cancellation.check(); + const retained = std.math.add(usize, decoded_retained_bytes, topology.retained_bytes) catch + return error.GraphMetricBuildBudgetExceeded; + if (retained >= options.limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + // Charge the census before touching edges, even if the later exact-sized + // projection cannot fit. Rejected work must never disappear from accounting. + const census_work = try compiledProjectionCensusWork(topology, options); + if (census_work > options.limits.max_work_items) return error.GraphMetricBuildBudgetExceeded; + if (options.batch_budget) |budget| try budget.chargeWork(census_work); + // Unlike an estimate made after the census, this also bounds scratch + // allocations on every failure path. Result buffers use the backing allocator. + var limiter = try bounded_decode.AllocationLimiter.init(alloc, options.limits.max_peak_memory_bytes - retained); + return buildAdmittedProjectionFromTopologyAlloc(limiter.allocator(), topology, decoded_retained_bytes, options) catch |err| switch (err) { + error.OutOfMemory => if (limiter.limit_exceeded) error.GraphMetricBuildBudgetExceeded else error.OutOfMemory, + else => err, + }; +} + +fn buildAdmittedProjectionFromTopologyAlloc( + alloc: Allocator, + topology: CompiledTopology, + decoded_retained_bytes: usize, + options: BuildOptions, +) !Projection { + return buildProjectionWithEdgeCopyAlloc(alloc, topology, decoded_retained_bytes, options, false); +} + +fn buildProjectionWithEdgeCopyAlloc(alloc: Allocator, topology: CompiledTopology, decoded_retained_bytes: usize, options: BuildOptions, comptime copy_edges: bool) !Projection { + try options.cancellation.check(); + if (topology.edge_type_offsets.len != topology.edge_types.len + 1 or + @as(usize, topology.edge_type_offsets[topology.edge_types.len]) != topology.edges.len) + { + return error.InvalidGraphMetricBuildOptions; + } + if (topology.source_node_count > options.limits.max_nodes or topology.source_edge_count > options.limits.max_edges) + return error.GraphMetricBuildBudgetExceeded; + const requirements = options.topology_requirements orelse topologyRequirementsForKind(options.config.kind); + if (!requirements.satisfies(topologyRequirementsForKind(options.config.kind))) + return error.InvalidGraphMetricBuildOptions; + const selected_edges = try selectedEdgeCount(topology, options.config.edge_filter, options.cancellation); + // Sorting a tiny endpoint set avoids allocating/clearing source-wide + // bitsets and ordinal/count arrays. Dense projections retain O(V + E) CSR. + if (!copy_edges and useSparseProjection(topology.node_ids.len, selected_edges)) + return buildSparseProjectionAlloc(alloc, topology, options, selected_edges, requirements); + if (requirements.incoming == .degrees and requirements.outgoing == .degrees) { + return try buildDegreeProjectionFromTopologyAlloc(alloc, topology, decoded_retained_bytes, options); + } + var projection = Projection{ + .source_node_count = topology.source_node_count, + .source_edge_count = topology.source_edge_count, + .census_work = try compiledProjectionCensusWork(topology, options), + }; + errdefer projection.deinit(alloc); + var filter = try EdgeFilterIndex.init(alloc, options.config.edge_filter); + defer filter.deinit(alloc); + + const allowed_edge_types = try alloc.alloc(bool, topology.edge_types.len); + defer alloc.free(allowed_edge_types); + for (topology.edge_types, 0..) |edge_type, edge_type_id| { + allowed_edge_types[edge_type_id] = filter.allows(edge_type); + } + + var active_nodes = try std.DynamicBitSetUnmanaged.initEmpty(alloc, topology.node_ids.len); + defer active_nodes.deinit(alloc); + var projected_edge_count: usize = 0; + var inspected_edges: usize = 0; + for (allowed_edge_types, 0..) |allowed, edge_type_id| { + if (!allowed) continue; + const range_start: usize = @intCast(topology.edge_type_offsets[edge_type_id]); + const range_end: usize = @intCast(topology.edge_type_offsets[edge_type_id + 1]); + if (range_start > range_end or range_end > topology.edges.len) return error.InvalidGraphMetricBuildOptions; + projected_edge_count = std.math.add(usize, projected_edge_count, range_end - range_start) catch + return error.GraphMetricBuildBudgetExceeded; + if (projected_edge_count > options.limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + for (topology.edges[range_start..range_end]) |edge| { + inspected_edges += 1; + if (inspected_edges % 4096 == 0) try options.cancellation.check(); + active_nodes.set(edge.source); + active_nodes.set(edge.target); + } + } + var projected_node_count: usize = 0; + for (0..topology.node_ids.len) |node_ordinal| { + if (!active_nodes.isSet(node_ordinal)) continue; + projected_node_count = std.math.add(usize, projected_node_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + } + if (projected_node_count > options.limits.max_nodes) return error.GraphMetricBuildBudgetExceeded; + + try chargeProjectionConstruction(options, topology, projected_node_count, projected_edge_count); + + var construction_peak = std.math.add(usize, decoded_retained_bytes, topology.retained_bytes) catch + return error.GraphMetricBuildBudgetExceeded; + try addPeakArrayBytes(&construction_peak, topology.edge_types.len, bool); + const active_node_word_count = std.math.divCeil(usize, topology.node_ids.len, @bitSizeOf(usize)) catch + return error.GraphMetricBuildBudgetExceeded; + try addPeakArrayBytes(&construction_peak, active_node_word_count, usize); + try addPeakArrayBytes(&construction_peak, topology.node_ids.len, u32); + try addPeakArrayBytes(&construction_peak, projected_node_count, []const u8); + if (copy_edges) try addPeakArrayBytes(&construction_peak, projected_edge_count, metrics.Edge); + const projected_offset_count = std.math.add(usize, projected_node_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + if (requirements.incoming != .none) try addPeakArrayBytes(&construction_peak, projected_offset_count, u32); + if (requirements.outgoing != .none) try addPeakArrayBytes(&construction_peak, projected_offset_count, u32); + if (requirements.incoming == .neighbors) { + try addPeakArrayBytes(&construction_peak, projected_edge_count, u32); + try addPeakArrayBytes(&construction_peak, projected_node_count, u32); + } + if (requirements.outgoing == .neighbors) { + try addPeakArrayBytes(&construction_peak, projected_edge_count, u32); + try addPeakArrayBytes(&construction_peak, projected_node_count, u32); + } + if (construction_peak > options.limits.max_peak_memory_bytes) + return error.GraphMetricBuildBudgetExceeded; + + const unassigned = std.math.maxInt(u32); + const global_to_local = try alloc.alloc(u32, topology.node_ids.len); + defer alloc.free(global_to_local); + @memset(global_to_local, unassigned); + try projection.node_ids.ensureTotalCapacityPrecise(alloc, projected_node_count); + for (topology.node_ids, 0..) |node_id, global_ordinal| { + if (!active_nodes.isSet(global_ordinal)) continue; + if (projection.node_ids.items.len > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + global_to_local[global_ordinal] = @intCast(projection.node_ids.items.len); + projection.node_ids.appendAssumeCapacity(node_id); + projection.node_id_bytes = std.math.add(usize, projection.node_id_bytes, node_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + const source = ProjectedEdges{ .topology = topology, .allowed = allowed_edge_types, .ordinals = global_to_local }; + projection.topology = if (copy_edges) blk: { + const copied = try alloc.alloc(metrics.Edge, projected_edge_count); + defer alloc.free(copied); + var iterator = source; + for (copied) |*edge| edge.* = iterator.next() orelse return error.InvalidGraphMetricBuildOptions; + break :blk try metrics.Topology.initAllocFor(alloc, projected_node_count, copied, requirements, options.cancellation); + } else try metrics.Topology.initFromSourceAlloc( + alloc, + projection.node_ids.items.len, + projected_edge_count, + source, + requirements, + options.cancellation, + ); + return projection; +} + +/// Replays selected immutable type runs in their original order, retaining +/// only the ordinal map. Both CSR passes therefore preserve summation order. +const ProjectedEdges = struct { + topology: CompiledTopology, + allowed: []const bool, + ordinals: []const u32, + sparse: bool = false, + type_index: usize = 0, + edge_index: usize = 0, + + pub fn next(self: *@This()) ?metrics.Edge { + while (self.type_index < self.allowed.len) { + const end = self.topology.edge_type_offsets[self.type_index + 1]; + if (!self.allowed[self.type_index] or self.edge_index == end) { + self.edge_index = end; + self.type_index += 1; + continue; + } + const edge = self.topology.edges[self.edge_index]; + self.edge_index += 1; + return .{ .source = self.localOrdinal(edge.source), .target = self.localOrdinal(edge.target) }; + } + return null; + } + + fn localOrdinal(self: @This(), ordinal: u32) u32 { + if (!self.sparse) return self.ordinals[ordinal]; + const index = std.sort.lowerBound(u32, self.ordinals, ordinal, struct { + fn order(a: u32, b: u32) std.math.Order { + return std.math.order(a, b); + } + }.order); + std.debug.assert(index < self.ordinals.len and self.ordinals[index] == ordinal); + return @intCast(index); + } +}; + +fn selectedEdgeCount(topology: CompiledTopology, filter: graph_mod.GraphMetricEdgeFilter, cancellation: CancellationToken) !usize { + if (topology.edge_type_offsets.len != topology.edge_types.len + 1 or + topology.edge_type_offsets[topology.edge_types.len] != topology.edges.len) return error.InvalidGraphMetricBuildOptions; + var count: usize = 0; + for (topology.edge_types, 0..) |edge_type, i| { + if (i % 4096 == 0) try cancellation.check(); + const start = topology.edge_type_offsets[i]; + const end = topology.edge_type_offsets[i + 1]; + if (start > end or end > topology.edges.len) return error.InvalidGraphMetricBuildOptions; + if (filter.mode != .all and for (filter.types) |allowed| { + if (std.mem.eql(u8, allowed, edge_type)) break false; + } else true) continue; + count += end - start; + } + return count; +} + +fn useSparseProjection(nodes: usize, edges: usize) bool { + return edges <= nodes / 64; +} + +fn compiledProjectionCensusWork(topology: CompiledTopology, options: BuildOptions) !u64 { + const edges = try selectedEdgeCount(topology, options.config.edge_filter, options.cancellation); + return projectionCensusWork(topology, options.config.edge_filter, edges); +} + +fn projectionCensusWork(topology: CompiledTopology, filter: graph_mod.GraphMetricEdgeFilter, edges: usize) !u64 { + const filter_work = try graph_metric_policy.workItems(topology.edge_types.len, filter.types.len, 1, @max(1, topology.edge_types.len)); + const census = if (useSparseProjection(topology.node_ids.len, edges)) blk: { + // Endpoint sort plus binary searches in both replay passes. This + // upper bound is charged before scratch allocation, including rejects. + const levels: u64 = if (edges == 0) 1 else std.math.log2_int_ceil(usize, edges * 2) + 1; + break :blk try graph_metric_policy.workItems(0, edges, 1, 6 * levels + 1); + } else try graph_metric_policy.workItems(topology.node_ids.len, edges, 1, 1); + return std.math.add(u64, filter_work, census) catch error.GraphMetricBuildBudgetExceeded; +} + +fn buildSparseProjectionAlloc(alloc: Allocator, topology: CompiledTopology, options: BuildOptions, edge_count: usize, requirements: metrics.TopologyRequirements) !Projection { + var projection = Projection{ + .source_node_count = topology.source_node_count, + .source_edge_count = topology.source_edge_count, + .census_work = try compiledProjectionCensusWork(topology, options), + }; + errdefer projection.deinit(alloc); + var filter = try EdgeFilterIndex.init(alloc, options.config.edge_filter); + defer filter.deinit(alloc); + const allowed = try alloc.alloc(bool, topology.edge_types.len); + defer alloc.free(allowed); + const endpoints = try alloc.alloc(u32, edge_count * 2); + defer alloc.free(endpoints); + var count: usize = 0; + for (topology.edge_types, 0..) |edge_type, i| { + allowed[i] = filter.allows(edge_type); + if (!allowed[i]) continue; + for (topology.edges[topology.edge_type_offsets[i]..topology.edge_type_offsets[i + 1]]) |edge| { + if (count % 4096 == 0) try options.cancellation.check(); + if (edge.source >= topology.node_ids.len or edge.target >= topology.node_ids.len) return error.InvalidGraphMetricBuildOptions; + endpoints[count] = edge.source; + endpoints[count + 1] = edge.target; + count += 2; + } + } + std.mem.sort(u32, endpoints, {}, std.sort.asc(u32)); + try options.cancellation.check(); + var active_count: usize = 0; + for (endpoints) |ordinal| { + if (active_count != 0 and endpoints[active_count - 1] == ordinal) continue; + endpoints[active_count] = ordinal; + active_count += 1; + } + try chargeProjectionConstruction(options, topology, active_count, edge_count); + try projection.node_ids.ensureTotalCapacityPrecise(alloc, active_count); + for (endpoints[0..active_count]) |ordinal| { + const id = topology.node_ids[ordinal]; + projection.node_ids.appendAssumeCapacity(id); + projection.node_id_bytes += id.len; + } + projection.topology = try metrics.Topology.initFromSourceAlloc(alloc, active_count, edge_count, ProjectedEdges{ + .topology = topology, + .allowed = allowed, + .ordinals = endpoints[0..active_count], + .sparse = true, + }, requirements, options.cancellation); + return projection; +} + +/// Prepared-source sparse projection versus the former source-wide scratch. +pub fn benchmarkSparseProjection(alloc: Allocator, ids: []const []const u8, kind: graph_mod.GraphMetricKind, reference: bool) !u64 { + const topology = sparseProjectionFixture(ids); + const options = BuildOptions{ + .graph_index_name = "bench", + .config = .{ .name = "metric", .kind = kind }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "fixture", .byte_len = 1 }, + }; + var projection = if (reference) + try buildProjectionWithEdgeCopyAlloc(alloc, topology, 0, options, true) + else + try buildProjectionFromTopologyAlloc(alloc, topology, 0, options); + defer projection.deinit(alloc); + var hash = std.hash.Wyhash.init(0); + hash.update(std.mem.sliceAsBytes(projection.topology.?.incoming_offsets)); + hash.update(std.mem.sliceAsBytes(projection.topology.?.incoming_sources)); + hash.update(std.mem.sliceAsBytes(projection.topology.?.outgoing_offsets)); + for (projection.node_ids.items) |id| hash.update(id); + return hash.final(); +} + +fn sparseProjectionFixture(ids: []const []const u8) CompiledTopology { + std.debug.assert(ids.len == 1_000_000); + return .{ + .node_ids = ids, + .edge_types = &.{"cites"}, + .string_bytes = &.{}, + .edge_type_offsets = &.{ 0, 2 }, + .edges = &.{ .{ .source = 0, .target = 999_999 }, .{ .source = 999_999, .target = 999_999 } }, + .source_node_count = ids.len, + .source_edge_count = 2, + .retained_bytes = ids.len * @sizeOf([]const u8), + }; +} + +test "serverless sparse projections bound scratch independently of the source dictionary" { + const alloc = std.testing.allocator; + const ids = try alloc.alloc([]const u8, 1_000_000); + defer alloc.free(ids); + @memset(ids, "unused"); + ids[0] = "a"; + ids[ids.len - 1] = "z"; + for ([_]graph_mod.GraphMetricKind{ .degree, .pagerank, .eigenvector, .hits_authority }) |kind| { + try std.testing.expectEqual(try benchmarkSparseProjection(alloc, ids, kind, true), try benchmarkSparseProjection(alloc, ids, kind, false)); + const topology = sparseProjectionFixture(ids); + var budget = graph_metric_policy.Budget{ .limits = .{} }; + var projection = try buildProjectionFromTopologyAlloc(alloc, topology, 0, .{ + .graph_index_name = "graph", + .config = .{ .name = "metric", .kind = kind }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "fixture", .byte_len = 1 }, + .limits = .{ .max_peak_memory_bytes = topology.retained_bytes + 2048, .max_work_items = 100 }, + .batch_budget = &budget, + }); + defer projection.deinit(alloc); + try std.testing.expectEqualSlices([]const u8, &.{ "a", "z" }, projection.node_ids.items); + try std.testing.expect(budget.work_items < 100); + } +} + +test "serverless sparse projection group admission uses selected endpoints and deduplicates aliases" { + const alloc = std.testing.allocator; + const ids = try alloc.alloc([]const u8, 1_000_000); + defer alloc.free(ids); + @memset(ids, "unused"); + const topology = sparseProjectionFixture(ids); + const configs = [_]graph_mod.GraphMetricConfig{ + .{ .name = "rank", .kind = .pagerank, .max_iterations = 10 }, + .{ .name = "rank_alias", .kind = .pagerank, .max_iterations = 10 }, + .{ .name = "eigen", .kind = .eigenvector, .max_iterations = 10 }, + .{ .name = "hub", .kind = .hits_hub, .max_iterations = 10 }, + }; + const limits = Limits{ .max_peak_memory_bytes = topology.retained_bytes + 2 * 1024 * 1024, .max_work_items = 2000, .max_total_work_items = 2000 }; + try std.testing.expect(projectionGroupFits(topology, &configs, &@as([4]bool, @splat(false)), .{}, limits, .{ .limits = limits })); + var budget = graph_metric_policy.Budget{ .limits = limits, .work_items = 1999 }; + try std.testing.expect(!projectionGroupFits(topology, &configs, &@as([4]bool, @splat(false)), .{}, limits, budget)); + budget.work_items = 0; + const one = configs[0..1]; + const aliases = configs[0..2]; + var cap: u64 = 1; + while (cap <= limits.max_total_work_items) : (cap += 1) { + var restricted = limits; + restricted.max_total_work_items = cap; + try std.testing.expectEqual( + projectionGroupFits(topology, one, &.{false}, .{}, restricted, budget), + projectionGroupFits(topology, aliases, &.{ false, false }, .{}, restricted, budget), + ); + } +} + +/// Exact former edge-copy oracle; source preparation is common to both paths. +pub fn benchmarkProjection(alloc: Allocator, payload: []const u8, reference: bool) !usize { + var topology = try prepareTopologyFromPackedAlloc(alloc, payload, .none, .{}); + defer topology.deinit(alloc); + const options = BuildOptions{ + .graph_index_name = "bench", + .config = .{ .name = "rank", .kind = .pagerank }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "fixture", .byte_len = payload.len }, + }; + var projection = if (reference) + try buildProjectionWithEdgeCopyAlloc(alloc, topology, 0, options, true) + else + try buildProjectionFromTopologyAlloc(alloc, topology, 0, options); + defer projection.deinit(alloc); + // Check a stable checksum over the exact adjacency, not merely row count. + var hash = std.hash.Wyhash.init(0); + hash.update(std.mem.sliceAsBytes(projection.topology.?.incoming_offsets)); + hash.update(std.mem.sliceAsBytes(projection.topology.?.incoming_sources)); + hash.update(std.mem.sliceAsBytes(projection.topology.?.outgoing_offsets)); + return @intCast(hash.final()); +} + +/// Degree-only materializations do not need edge ordinals after counting. +/// Count directly from the compiled edge-type runs in one O(E) pass instead +/// of copying projected edges and scanning them twice more to build CSR. +fn buildDegreeProjectionFromTopologyAlloc( + alloc: Allocator, + topology: CompiledTopology, + decoded_retained_bytes: usize, + options: BuildOptions, +) !Projection { + if (topology.source_node_count > options.limits.max_nodes or topology.source_edge_count > options.limits.max_edges) + return error.GraphMetricBuildBudgetExceeded; + var projection = Projection{ + .source_node_count = topology.source_node_count, + .source_edge_count = topology.source_edge_count, + .census_work = try compiledProjectionCensusWork(topology, options), + }; + errdefer projection.deinit(alloc); + var filter = try EdgeFilterIndex.init(alloc, options.config.edge_filter); + defer filter.deinit(alloc); + const allowed_edge_types = try alloc.alloc(bool, topology.edge_types.len); + defer alloc.free(allowed_edge_types); + for (topology.edge_types, 0..) |edge_type, edge_type_id| { + allowed_edge_types[edge_type_id] = filter.allows(edge_type); + } + + var active_nodes = try std.DynamicBitSetUnmanaged.initEmpty(alloc, topology.node_ids.len); + defer active_nodes.deinit(alloc); + const incoming_counts = try alloc.alloc(u32, topology.node_ids.len); + defer alloc.free(incoming_counts); + const outgoing_counts = try alloc.alloc(u32, topology.node_ids.len); + defer alloc.free(outgoing_counts); + @memset(incoming_counts, 0); + @memset(outgoing_counts, 0); + var projected_edge_count: usize = 0; + for (allowed_edge_types, 0..) |allowed, edge_type_id| { + if (!allowed) continue; + const range_start: usize = @intCast(topology.edge_type_offsets[edge_type_id]); + const range_end: usize = @intCast(topology.edge_type_offsets[edge_type_id + 1]); + if (range_start > range_end or range_end > topology.edges.len) return error.InvalidGraphMetricBuildOptions; + for (topology.edges[range_start..range_end]) |edge| { + projected_edge_count = std.math.add(usize, projected_edge_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + if (projected_edge_count > options.limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + if (projected_edge_count % 4096 == 0) try options.cancellation.check(); + incoming_counts[edge.target] = std.math.add(u32, incoming_counts[edge.target], 1) catch + return error.GraphMetricBuildBudgetExceeded; + outgoing_counts[edge.source] = std.math.add(u32, outgoing_counts[edge.source], 1) catch + return error.GraphMetricBuildBudgetExceeded; + active_nodes.set(edge.source); + active_nodes.set(edge.target); + } + } + const projected_node_count = active_nodes.count(); + if (projected_node_count > options.limits.max_nodes) return error.GraphMetricBuildBudgetExceeded; + try chargeProjectionConstruction(options, topology, projected_node_count, projected_edge_count); + var construction_peak = std.math.add(usize, decoded_retained_bytes, topology.retained_bytes) catch + return error.GraphMetricBuildBudgetExceeded; + try addPeakArrayBytes(&construction_peak, topology.edge_types.len, bool); + try addPeakArrayBytes(&construction_peak, std.math.divCeil(usize, topology.node_ids.len, @bitSizeOf(usize)) catch + return error.GraphMetricBuildBudgetExceeded, usize); + try addPeakArrayBytes(&construction_peak, topology.node_ids.len, u32); + try addPeakArrayBytes(&construction_peak, topology.node_ids.len, u32); + try addPeakArrayBytes(&construction_peak, projected_node_count, []const u8); + try addPeakArrayBytes(&construction_peak, projected_node_count + 1, u32); + try addPeakArrayBytes(&construction_peak, projected_node_count + 1, u32); + if (construction_peak > options.limits.max_peak_memory_bytes) return error.GraphMetricBuildBudgetExceeded; + + try projection.node_ids.ensureTotalCapacityPrecise(alloc, projected_node_count); + const incoming_offsets = try alloc.alloc(u32, projected_node_count + 1); + errdefer alloc.free(incoming_offsets); + const outgoing_offsets = try alloc.alloc(u32, projected_node_count + 1); + errdefer alloc.free(outgoing_offsets); + incoming_offsets[0] = 0; + outgoing_offsets[0] = 0; + var local_index: usize = 0; + for (topology.node_ids, 0..) |node_id, global_ordinal| { + if (!active_nodes.isSet(global_ordinal)) continue; + projection.node_ids.appendAssumeCapacity(node_id); + projection.node_id_bytes = std.math.add(usize, projection.node_id_bytes, node_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + incoming_offsets[local_index + 1] = std.math.add(u32, incoming_offsets[local_index], incoming_counts[global_ordinal]) catch + return error.GraphMetricBuildBudgetExceeded; + outgoing_offsets[local_index + 1] = std.math.add(u32, outgoing_offsets[local_index], outgoing_counts[global_ordinal]) catch + return error.GraphMetricBuildBudgetExceeded; + local_index += 1; + } + std.debug.assert(local_index == projected_node_count); + projection.topology = .{ + .node_count = projected_node_count, + .edge_count = projected_edge_count, + .requirements = .degree, + .incoming_offsets = incoming_offsets, + .incoming_sources = @constCast(&[_]u32{}), + .outgoing_offsets = outgoing_offsets, + .outgoing_targets = @constCast(&[_]u32{}), + }; + return projection; +} + +const PreparedProjection = struct { + projection: *Projection, + built: bool, +}; + +fn preparedProjectionAlloc( + alloc: Allocator, + prepared: *PreparedGraphArtifact, + options: BuildOptions, +) !PreparedProjection { + // Prepared artifacts may outlive a single publication request. Re-admit + // the immutable source topology against every caller's limits before a + // cache hit can bypass projection construction. + if (prepared.topology.source_node_count > options.limits.max_nodes or + prepared.topology.source_edge_count > options.limits.max_edges) + { + return error.GraphMetricBuildBudgetExceeded; + } + const requirements = options.topology_requirements orelse topologyRequirementsForKind(options.config.kind); + if (prepared.cached_projection != null and + prepared.cached_projection_filter.?.equivalent(options.config.edge_filter) and + prepared.cached_projection_requirements.satisfies(requirements)) + { + return .{ .projection = &prepared.cached_projection.?, .built = false }; + } + + // Clone the tiny filter identity before releasing the previous projection, + // then replace the cache in place. A failed rebuild leaves it empty rather + // than retaining a projection whose requirements do not satisfy the call. + var filter = try options.config.edge_filter.cloneAlloc(alloc); + errdefer filter.deinit(alloc); + if (prepared.cached_projection) |*projection| projection.deinit(alloc); + prepared.cached_projection = null; + if (prepared.cached_projection_filter) |*previous_filter| previous_filter.deinit(alloc); + prepared.cached_projection_filter = null; + + var projection = try buildProjectionFromTopologyAlloc(alloc, prepared.topology, 0, options); + projection.decoded_retained_bytes = prepared.topology.retained_bytes; + prepared.cached_projection = projection; + prepared.cached_projection_filter = filter; + prepared.cached_projection_requirements = requirements; + return .{ .projection = &prepared.cached_projection.?, .built = true }; +} + +fn buildProjectionAlloc(alloc: Allocator, graph: graph_segment.Segment, options: BuildOptions) !Projection { + try options.cancellation.check(); + const requirements = options.topology_requirements orelse topologyRequirementsForKind(options.config.kind); + if (!requirements.satisfies(topologyRequirementsForKind(options.config.kind))) + return error.InvalidGraphMetricBuildOptions; + if (graph.adjacencies.len > options.limits.max_nodes) return error.GraphMetricBuildBudgetExceeded; + var projection = Projection{}; + errdefer projection.deinit(alloc); + projection.source_node_count = graph.adjacencies.len; + var filter = try EdgeFilterIndex.init(alloc, options.config.edge_filter); + defer filter.deinit(alloc); + var projected_edges = std.ArrayListUnmanaged(metrics.Edge).empty; + defer projected_edges.deinit(alloc); + + for (graph.adjacencies, 0..) |adjacency, adjacency_index| { + if (adjacency_index % 256 == 0) try options.cancellation.check(); + for (adjacency.out_edges) |edge| { + projection.source_edge_count = std.math.add(usize, projection.source_edge_count, 1) catch + return error.GraphMetricBuildBudgetExceeded; + if (projection.source_edge_count > options.limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + if (projection.source_edge_count % 4096 == 0) try options.cancellation.check(); + if (edge.neighbor_table_id != null) continue; + if (!filter.allows(edge.edge_type)) continue; + const source = try getOrPutNode(alloc, &projection.ordinals, &projection.node_ids, adjacency.node_id, options.limits.max_nodes); + const target = try getOrPutNode(alloc, &projection.ordinals, &projection.node_ids, edge.neighbor_id, options.limits.max_nodes); + if (projected_edges.items.len >= options.limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + if (source > std.math.maxInt(u32) or target > std.math.maxInt(u32)) return error.GraphMetricBuildBudgetExceeded; + try projected_edges.append(alloc, .{ .source = @intCast(source), .target = @intCast(target) }); + } + } + for (projection.node_ids.items) |node_id| { + projection.node_id_bytes = std.math.add(usize, projection.node_id_bytes, node_id.len) catch + return error.GraphMetricBuildBudgetExceeded; + } + projection.topology = try metrics.Topology.initAllocFor( + alloc, + projection.node_ids.items.len, + projected_edges.items, + requirements, + options.cancellation, + ); + return projection; +} + +fn addPeakBytes(total: *usize, amount: usize) !void { + total.* = std.math.add(usize, total.*, amount) catch return error.GraphMetricBuildBudgetExceeded; +} + +fn addPeakArrayBytes(total: *usize, count: usize, comptime Element: type) !void { + const bytes = std.math.mul(usize, count, @sizeOf(Element)) catch + return error.GraphMetricBuildBudgetExceeded; + try addPeakBytes(total, bytes); +} + +fn projectionResidentMemoryBytes(projection: Projection) !usize { + var total = projection.decoded_retained_bytes; + const topology = projection.topology orelse return error.InvalidGraphMetricBuildOptions; + try addPeakArrayBytes(&total, topology.incoming_offsets.len, u32); + try addPeakArrayBytes(&total, topology.incoming_sources.len, u32); + try addPeakArrayBytes(&total, topology.outgoing_offsets.len, u32); + try addPeakArrayBytes(&total, topology.outgoing_targets.len, u32); + try addPeakArrayBytes(&total, projection.node_ids.capacity, []const u8); + // StringHashMap's exact control-byte layout is intentionally private. A + // conservative per-node allowance covers keys, values, control bytes, and + // normal load-factor slack without coupling admission to std internals. + if (projection.ordinals.count() != 0) { + try addPeakBytes(&total, std.math.mul(usize, projection.node_ids.items.len, 64) catch + return error.GraphMetricBuildBudgetExceeded); + } + return total; +} + +fn estimatedPeakMemoryBytes(projection: Projection, options: BuildOptions, simultaneous_outputs: usize) !usize { + if (simultaneous_outputs == 0) return error.InvalidGraphMetricBuildOptions; + var total = try projectionResidentMemoryBytes(projection); + + const kernel_vectors: usize = switch (options.config.kind) { + .degree => 1, + .pagerank => 3, + .eigenvector => 2, + .hits_authority, .hits_hub => 4, + }; + try addPeakBytes(&total, std.math.mul( + usize, + projection.node_ids.items.len, + kernel_vectors * @sizeOf(f64), + ) catch return error.GraphMetricBuildBudgetExceeded); + const seed_vectors: usize = @intFromBool(options.initial_scores != null) + + @intFromBool(options.initial_authorities != null) + + @intFromBool(options.initial_hubs != null); + try addPeakBytes(&total, std.math.mul( + usize, + projection.node_ids.items.len, + seed_vectors * @sizeOf(f64), + ) catch return error.GraphMetricBuildBudgetExceeded); + + // Encoding borrows canonical node IDs from the projection. The output + // buffer still repeats IDs in its primary stream and bounded top-score + // routing tier, so both encoded copies remain part of peak admission. + const per_output_fixed = std.math.mul( + usize, + projection.node_ids.items.len, + @sizeOf(metric_segment.Score) + 32, + ) catch return error.GraphMetricBuildBudgetExceeded; + const per_output_node_ids = std.math.mul(usize, projection.node_id_bytes, 2) catch + return error.GraphMetricBuildBudgetExceeded; + const per_output = std.math.add(usize, per_output_fixed, per_output_node_ids) catch + return error.GraphMetricBuildBudgetExceeded; + try addPeakBytes(&total, std.math.mul(usize, per_output, simultaneous_outputs) catch + return error.GraphMetricBuildBudgetExceeded); + try addPeakBytes(&total, 1024 * 1024); + return total; +} + +fn admitPeakMemory(projection: Projection, options: BuildOptions, simultaneous_outputs: usize) !void { + if ((try estimatedPeakMemoryBytes(projection, options, simultaneous_outputs)) > options.limits.max_peak_memory_bytes) + return error.GraphMetricBuildBudgetExceeded; +} + +fn kernelOptions(options: BuildOptions) metrics.Options { + return .{ + .damping = if (options.config.kind == .pagerank) options.config.damping else 0.85, + .tolerance = if (options.config.kind == .degree) 0 else options.config.tolerance, + .max_iterations = if (options.config.kind == .degree) 1 else options.config.max_iterations, + .max_nodes = options.limits.max_nodes, + .max_edges = options.limits.max_edges, + .max_work_items = options.limits.max_work_items, + .cancellation = options.cancellation, + .io = options.io, + .max_parallelism = if (options.io == null) 1 else options.max_parallelism, + .initial_scores = options.initial_scores, + .initial_authorities = options.initial_authorities, + .initial_hubs = options.initial_hubs, + }; +} + +fn projectionWorkItems(projection: Projection, options: BuildOptions) !u64 { + const requirements = options.topology_requirements orelse topologyRequirementsForKind(options.config.kind); + const projected_passes: u64 = if (requirements.incoming == .neighbors or requirements.outgoing == .neighbors) 4 else 3; + if (projection.census_work) |census| return std.math.add(u64, census, try graph_metric_policy.workItems(projection.node_ids.items.len, projection.edgeCount(), 1, projected_passes)) catch error.GraphMetricBuildBudgetExceeded; + return try graph_metric_policy.projectionWorkItems( + projection.source_node_count, + projection.source_edge_count, + projection.node_ids.items.len, + projection.edgeCount(), + projected_passes, + ); +} + +fn chargeProjectionConstruction(options: BuildOptions, topology: CompiledTopology, nodes: usize, edges: usize) !void { + const requirements = options.topology_requirements orelse topologyRequirementsForKind(options.config.kind); + const passes: u64 = if (requirements.incoming == .neighbors or requirements.outgoing == .neighbors) 4 else 3; + const total = std.math.add(u64, try compiledProjectionCensusWork(topology, options), try graph_metric_policy.workItems(nodes, edges, 1, passes)) catch return error.GraphMetricBuildBudgetExceeded; + if (total > options.limits.max_work_items) return error.GraphMetricBuildBudgetExceeded; + if (options.batch_budget) |budget| try budget.chargeWork(try graph_metric_policy.workItems(nodes, edges, 1, passes)); +} + +fn chargeKernelWork(options: BuildOptions, projection: Projection) !void { + const kernel_amount = try graph_metric_policy.metricWorkItems( + options.config.kind, + projection.node_ids.items.len, + projection.edgeCount(), + options.config.max_iterations, + ); + const materialization_amount = std.math.add( + u64, + try projectionWorkItems(projection, options), + kernel_amount, + ) catch return error.GraphMetricBuildBudgetExceeded; + // The per-materialization limit still includes projection even when a + // batch amortizes that projection across multiple compatible metrics. + if (materialization_amount > options.limits.max_work_items) + return error.GraphMetricBuildBudgetExceeded; + if (options.batch_budget) |budget| { + try budget.chargeWork(kernel_amount); + } +} + +fn admitProjectionKernel(projection: Projection, options: BuildOptions) !void { + try admitMinimumOutput(projection, options, 1); + try admitPeakMemory(projection, options, 1); + try chargeKernelWork(options, projection); +} + +fn admitMinimumOutput(projection: Projection, options: BuildOptions, outputs: usize) !void { + // Every primary row stores a value and suffix length, independent of score + // ordering/prefix compression. Reject impossible output before any kernel, + // warm-start I/O, or top-tier selection; do not charge numerical work. + const control = try metric_segment.controlProbeLen(std.math.maxInt(u64), options.source_graph.artifact_id, options.source_graph.checksum, options.config.edge_filter); + const rows = std.math.mul(usize, projection.node_ids.items.len, 10) catch return error.GraphMetricBuildBudgetExceeded; + const minimum = std.math.add(usize, control, rows) catch return error.GraphMetricBuildBudgetExceeded; + if (minimum > options.limits.max_metric_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + const total = std.math.mul(usize, minimum, outputs) catch return error.GraphMetricBuildBudgetExceeded; + if (options.batch_budget) |budget| if (total > budget.limits.max_total_metric_payload_bytes -| budget.metric_payload_bytes) + return error.GraphMetricBuildBudgetExceeded; +} + +fn buildFromProjectionAlloc(alloc: Allocator, projection: Projection, options: BuildOptions) !BuildResult { + try admitProjectionKernel(projection, options); + return buildAdmittedProjectionAlloc(alloc, projection, options); +} + +fn buildAdmittedProjectionAlloc(alloc: Allocator, projection: Projection, options: BuildOptions) !BuildResult { + const kernel_options = kernelOptions(options); + const topology = projection.topology orelse return error.InvalidGraphMetricBuildOptions; + var result = switch (options.config.kind) { + .degree => try metrics.degreeTopologyAlloc(alloc, topology, kernel_options), + .pagerank => try metrics.pageRankTopologyAlloc(alloc, topology, kernel_options), + .eigenvector => try metrics.eigenvectorTopologyAlloc(alloc, topology, kernel_options), + .hits_authority, .hits_hub => blk: { + const pair = try metrics.hitsTopologyAlloc(alloc, topology, kernel_options); + const selected = if (options.config.kind == .hits_authority) pair.authorities else pair.hubs; + const unused = if (options.config.kind == .hits_authority) pair.hubs else pair.authorities; + alloc.free(unused); + break :blk metrics.Result{ .scores = selected, .iterations_completed = pair.iterations_completed, .converged = pair.converged, .delta = pair.delta }; + }, + }; + defer result.deinit(alloc); + return try encodeMetricResultAlloc(alloc, projection.node_ids.items, options, result); +} + +fn buildFromTopologyAlloc( + alloc: Allocator, + topology: CompiledTopology, + options: BuildOptions, +) !BuildResult { + var identified = options; + if (typeChecksumsAlloc(alloc, topology, options.limits, options.cancellation)) |checksums| { + defer alloc.free(checksums); + identified.topology_checksum = selectedTopologyChecksum(topology, checksums, options.config.edge_filter); + } else |err| if (err != error.GraphMetricBuildBudgetExceeded) return err; + var projection = try buildProjectionFromTopologyAlloc(alloc, topology, 0, options); + defer projection.deinit(alloc); + projection.decoded_retained_bytes = topology.retained_bytes; + return try buildFromProjectionAlloc(alloc, projection, identified); +} + +fn publishHitsPairFromProjectionAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + projection: Projection, + first_options: BuildOptions, + second_config: graph_mod.GraphMetricConfig, + cancellation: CancellationToken, +) ![2]artifact_ref.ArtifactRef { + if (!graph_mod.graphMetricHitsPairCompatible(first_options.config, second_config)) return error.InvalidGraphMetricBuildOptions; + // Both vectors are computed once, while immutable artifacts are encoded + // and published one at a time. This makes output memory independent of the + // number of paired HITS lanes. + try admitMinimumOutput(projection, first_options, 2); + try admitPeakMemory(projection, first_options, 1); + try chargeKernelWork(first_options, projection); + + const topology = projection.topology orelse return error.InvalidGraphMetricBuildOptions; + var pair = try metrics.hitsTopologyAlloc(alloc, topology, kernelOptions(first_options)); + defer pair.deinit(alloc); + const first_scores = if (first_options.config.kind == .hits_authority) pair.authorities else pair.hubs; + const second_scores = if (second_config.kind == .hits_authority) pair.authorities else pair.hubs; + const first_result = metrics.Result{ .scores = first_scores, .iterations_completed = pair.iterations_completed, .converged = pair.converged, .delta = pair.delta }; + const second_result = metrics.Result{ .scores = second_scores, .iterations_completed = pair.iterations_completed, .converged = pair.converged, .delta = pair.delta }; + var second_options = first_options; + second_options.config = second_config; + const first_plan = try prepareMetricOutputPlan(alloc, projection.node_ids.items, first_options, first_result); + const second_plan = try prepareMetricOutputPlan(alloc, projection.node_ids.items, second_options, second_result); + // Reserve the pair atomically. A second-lane quota failure must never leave + // the first lane uploaded only to be discarded as a rejected pair. + const pair_bytes = std.math.add(usize, first_plan.size, second_plan.size) catch return error.GraphMetricBuildBudgetExceeded; + if (first_options.batch_budget) |budget| try budget.chargePayload(pair_bytes); + errdefer if (first_options.batch_budget) |budget| { + budget.metric_payload_bytes -= pair_bytes; + }; + const first_ref = blk: { + var first = try encodeMetricResultWithPlanAlloc(alloc, projection.node_ids.items, first_options, first_result, &first_plan, true); + defer first.deinit(alloc); + break :blk try putBuildResultAlloc(alloc, artifacts, &first, cancellation); + }; + errdefer freeArtifactRef(alloc, first_ref); + if (first_options.config.kind == .hits_authority) { + alloc.free(pair.authorities); + pair.authorities = @constCast(&[_]f64{}); + } else { + alloc.free(pair.hubs); + pair.hubs = @constCast(&[_]f64{}); + } + + const second_ref = blk: { + var second = try encodeMetricResultWithPlanAlloc(alloc, projection.node_ids.items, second_options, second_result, &second_plan, true); + defer second.deinit(alloc); + break :blk try putBuildResultAlloc(alloc, artifacts, &second, cancellation); + }; + return .{ first_ref, second_ref }; +} + +fn encodeMetricResultAlloc( + alloc: Allocator, + node_ids: []const []const u8, + options: BuildOptions, + result: metrics.Result, +) !BuildResult { + return encodeMetricResultWithPlanAlloc(alloc, node_ids, options, result, null, false); +} + +fn prepareMetricOutputPlan(alloc: Allocator, node_ids: []const []const u8, options: BuildOptions, result: metrics.Result) !metric_segment.codec.EncodingPlan { + const scores = try makeScoresAlloc(alloc, node_ids, result.scores, options.cancellation); + var owned = true; + errdefer if (owned) alloc.free(scores); + var segment = try makeMetricSegmentAlloc(alloc, options, result, scores); + owned = false; + defer segment.deinit(alloc); + const plan = try metric_segment.codec.prepareEncoding(segment, options.cancellation); + if (plan.size > options.limits.max_metric_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + return plan; +} + +fn encodeMetricResultWithPlanAlloc(alloc: Allocator, node_ids: []const []const u8, options: BuildOptions, result: metrics.Result, prepared: ?*const metric_segment.codec.EncodingPlan, reserved: bool) !BuildResult { + const scores = try makeScoresAlloc(alloc, node_ids, result.scores, options.cancellation); + var scores_owned = true; + defer if (scores_owned) alloc.free(scores); + + var segment = try makeMetricSegmentAlloc(alloc, options, result, scores); + scores_owned = false; + defer segment.deinit(alloc); + const local_plan = if (prepared == null) try metric_segment.codec.prepareEncoding(segment, options.cancellation) else undefined; + const plan = prepared orelse &local_plan; + if (plan.size > options.limits.max_metric_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + if (!reserved) if (options.batch_budget) |budget| try budget.chargePayload(plan.size); + errdefer if (!reserved) { + if (options.batch_budget) |budget| budget.metric_payload_bytes -= plan.size; + }; + const payload = metric_segment.codec.encodePreparedAlloc( + alloc, + segment, + options.cancellation, + plan, + options.limits.max_metric_payload_bytes, + ) catch |err| switch (err) { + error.GraphMetricSegmentTooLarge => return error.GraphMetricBuildBudgetExceeded, + else => return err, + }; + errdefer alloc.free(payload); + const name = try artifactNameAlloc(alloc, options.graph_index_name, options.config.name); + errdefer alloc.free(name); + const artifact_id = try std.fmt.allocPrint(alloc, "lake-graph-metric:{d}:{s}:{d}", .{ name.len, name, payload.len }); + errdefer alloc.free(artifact_id); + const checksum = try std.fmt.allocPrint(alloc, "len:{d}", .{payload.len}); + errdefer alloc.free(checksum); + var artifact = artifact_ref.ArtifactRef{ + .kind = .graph_metric_segment, + .name = name, + .artifact_id = artifact_id, + .byte_len = @intCast(payload.len), + .checksum = checksum, + .metadata_version = metric_segment.wire_version, + .published_generation = options.provenance.published_generation, + .edge_generation = options.provenance.edge_generation, + .computed_at_ms = options.provenance.computed_at_ms, + .materializer_fingerprint = segment.materializer_fingerprint, + }; + try populateGraphMetricIntegrity(&artifact, segment, payload); + return .{ .payload = payload, .artifact = artifact }; +} + +fn populateGraphMetricIntegrity(ref: *artifact_ref.ArtifactRef, segment: metric_segment.Segment, payload: []const u8) !void { + const integrity = try metric_segment.artifactIntegrity(segment, payload); + ref.graph_metric_control_len = integrity.control_len; + ref.graph_metric_routing_footer_len = integrity.routing_footer_len; + ref.graph_metric_control_checksum = integrity.control_checksum; + ref.graph_metric_routing_checksum = integrity.routing_checksum; + ref.graph_metric_point_index_checksum = integrity.point_index_checksum; + ref.graph_metric_config_fingerprint = segment.config_fingerprint; + ref.graph_metric_topology_checksum = segment.topology_checksum; + ref.graph_metric_source_checksum = artifact_store.sha256DigestFromChecksum(segment.source_graph_checksum) catch + return error.ArtifactIntegrityMismatch; + ref.graph_metric_materialization_state = @enumFromInt(@intFromEnum(segment.materialization_state)); + ref.graph_metric_rejection_reason = @enumFromInt(@intFromEnum(segment.rejection_reason)); +} + +/// Maps the last published node-sorted vector onto the current canonical node +/// dictionary without materializing the old segment's node IDs. The verified +/// primary score windows are merged into one dense output vector; added nodes +/// remain zero and deleted nodes are skipped. No full prior payload is retained. +fn warmStartVectorAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + prior_artifact: ?artifact_ref.ArtifactRef, + current_node_ids: []const []const u8, + config: graph_mod.GraphMetricConfig, + cancellation: CancellationToken, + limits: Limits, + existing_resident_bytes: usize, + execution_peak_bytes: usize, + batch_budget: ?*graph_metric_policy.Budget, +) !?[]f64 { + if (!metrics.warm_start.supported(config.kind) or prior_artifact == null or current_node_ids.len == 0) return null; + const prior = prior_artifact.?; + if (prior.kind != .graph_metric_segment or prior.metadata_version != metric_segment.wire_version or + prior.byte_len == 0 or prior.byte_len > limits.max_metric_payload_bytes or + prior.graph_metric_materialization_state != .ready or + prior.graph_metric_config_fingerprint != configFingerprint(config)) return null; + const seed_bytes = std.math.mul(usize, current_node_ids.len, @sizeOf(f64)) catch return error.GraphMetricBuildBudgetExceeded; + // Admit both lifetimes before doing optional I/O. A seed that fits during + // preparation may not fit alongside the kernel and encoded output. + if (execution_peak_bytes > limits.max_peak_memory_bytes or + seed_bytes > limits.max_peak_memory_bytes - execution_peak_bytes) return null; + if (existing_resident_bytes >= limits.max_peak_memory_bytes) return null; + var local_budget = graph_metric_policy.Budget{ .limits = limits }; + const budget = batch_budget orelse &local_budget; + if (!budget.admitSeed(0, current_node_ids.len)) return null; + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_peak_memory_bytes - existing_resident_bytes); + return readWarmStartVectorAlloc(&limiter, artifacts, prior, current_node_ids, config, cancellation, budget) catch |err| switch (err) { + error.Canceled => return err, + error.OutOfMemory => if (limiter.limit_exceeded) null else error.OutOfMemory, + else => { + // Optional acceleration never poisons an otherwise valid cold build. + try cancellation.check(); + if (err != error.ArtifactReadBudgetExceeded) + std.log.warn("graph metric warm start unavailable; using cold seed metric={s} artifact={s} err={s}", .{ config.name, prior.artifact_id, @errorName(err) }); + return null; + }, + }; +} + +/// A range reader charges provider verification as well as requested bytes +/// before I/O. Cold providers without pinned integrity may require a full +/// verification; that must fit the optional budget or the build uses cold rank. +const SeedReader = struct { + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + prior: artifact_ref.ArtifactRef, + cancellation: CancellationToken, + budget: *graph_metric_policy.Budget, + + fn read(self: @This(), offset: u64, len: usize, checksum: ?[32]u8) ![]u8 { + var remaining = @min( + self.budget.limits.max_total_seed_payload_bytes -| self.budget.seed_payload_bytes, + self.budget.limits.max_total_seed_work_items -| self.budget.seed_work_items, + ); + const before = remaining; + defer { + self.budget.seed_payload_bytes += before - remaining; + self.budget.seed_work_items += before - remaining; + } + const bytes = try self.artifacts.getVerifiedRangeAllocWithBudget(self.alloc, self.prior.artifact_id, self.prior.byte_len, self.prior.checksum, offset, len, self.cancellation, &remaining); + errdefer self.alloc.free(bytes); + if (checksum) |expected| try verifySeedChecksum(bytes, expected); + return bytes; + } +}; + +fn verifySeedChecksum(bytes: []const u8, expected: [32]u8) !void { + var actual: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &actual, .{}); + if (!std.mem.eql(u8, &actual, &expected)) return error.ArtifactIntegrityMismatch; +} + +fn seedNodeLowerBound(nodes: []const []const u8, node: []const u8) usize { + return std.sort.lowerBound([]const u8, nodes, node, struct { + fn order(a: []const u8, b: []const u8) std.math.Order { + return std.mem.order(u8, a, b); + } + }.order); +} + +/// Read only primary score windows intersecting the current sorted dictionary. +/// Dense selections coalesce adjacent blocks (up to 1 MiB); sparse selections +/// skip unrelated pages and blocks. Root/directory and one page/window are the +/// only retained old-artifact buffers, regardless of the old payload size. +fn readWarmStartVectorAlloc(limiter: *bounded_decode.AllocationLimiter, artifacts: *artifact_store.ArtifactStore, prior: artifact_ref.ArtifactRef, current_node_ids: []const []const u8, config: graph_mod.GraphMetricConfig, cancellation: CancellationToken, budget: *graph_metric_policy.Budget) !?[]f64 { + const alloc = limiter.allocator(); + const codec = metric_segment.codec; + const reader = SeedReader{ .alloc = alloc, .artifacts = artifacts, .prior = prior, .cancellation = cancellation, .budget = budget }; + const control_bytes = try reader.read(0, prior.graph_metric_control_len, prior.graph_metric_control_checksum); + defer alloc.free(control_bytes); + const control = try metric_segment.decodeControl(control_bytes, config.edge_filter); + if (control.header.kind != config.kind or control.header.materialization_state != .ready or + control.header.config_fingerprint != configFingerprint(config) or + control.score_data_offset != control_bytes.len or control.score_count == 0) return null; + const root_len = codec.routingRootLen(control.score_count); + if (root_len > prior.byte_len or prior.graph_metric_routing_footer_len > prior.byte_len) return null; + const root_bytes = try reader.read(prior.byte_len - root_len, root_len, prior.graph_metric_routing_checksum); + defer alloc.free(root_bytes); + var root = try codec.decodeRoutingRootAlloc(alloc, root_bytes, prior.byte_len, control.header.version, cancellation); + defer root.deinit(alloc); + if (root.primary_data_offset != control.score_data_offset or + root.footer_offset != prior.byte_len - prior.graph_metric_routing_footer_len or + !std.mem.eql(u8, &root.point_index_checksum, &prior.graph_metric_point_index_checksum)) return null; + const block_count = std.math.divCeil(usize, control.score_count, codec.score_block_entries) catch return null; + const directory_offset = prior.byte_len - root_len - root.directory_len; + const directory_bytes = try reader.read(directory_offset, root.directory_len, root.directory_checksum); + defer alloc.free(directory_bytes); + const directory = try codec.decodePointDirectoryAlloc(alloc, directory_bytes, directory_offset, root.footer_offset, block_count, cancellation); + defer alloc.free(directory); + const seed = try alloc.alloc(f64, current_node_ids.len); + var seed_owned = true; + defer if (seed_owned) alloc.free(seed); + @memset(seed, 0); + var current: usize = 0; + var positive_sum: f64 = 0; + for (directory, 0..) |page, page_index| { + try cancellation.check(); + current += seedNodeLowerBound(current_node_ids[current..], page.first_node_id); + if (current == current_node_ids.len) break; + const page_end = if (page_index + 1 < directory.len) seedNodeLowerBound(current_node_ids, directory[page_index + 1].first_node_id) else current_node_ids.len; + if (current >= page_end) continue; + const page_bytes = try reader.read(page.offset, page.len, page.checksum); + defer alloc.free(page_bytes); + const entries = try codec.decodePointPageAlloc(alloc, page_bytes, page, block_count, root.primary_data_offset, root.primary_data_end, cancellation); + defer alloc.free(entries); + if (page_index + 1 < directory.len and std.mem.order(u8, entries[entries.len - 1].first_node_id, directory[page_index + 1].first_node_id) != .lt) return error.InvalidGraphMetricSegment; + var index: usize = 0; + while (index < entries.len and current < page_end) { + const first = entries[index]; + if (index + 1 < entries.len and std.mem.order(u8, current_node_ids[current], entries[index + 1].first_node_id) != .lt) { + index += 1; + continue; + } + var end = index + 1; + var window_len: usize = first.len; + const window_limit = @min(1024 * 1024, limiter.max_live_bytes -| limiter.live_bytes); + while (end < entries.len) : (end += 1) { + const candidate = seedNodeLowerBound(current_node_ids[current..page_end], entries[end].first_node_id) + current; + if (candidate == page_end or + (end + 1 < entries.len and std.mem.order(u8, current_node_ids[candidate], entries[end + 1].first_node_id) != .lt) or + entries[end].len > window_limit -| window_len) break; + window_len += entries[end].len; + } + const window = try reader.read(first.offset, window_len, null); + defer alloc.free(window); + for (entries[index..end]) |entry| { + const bytes = window[@intCast(entry.offset - first.offset)..][0..entry.len]; + try verifySeedChecksum(bytes, entry.checksum); + const decoded = try codec.decodeScoreBlockWithCancellation(bytes, cancellation); + const expected_rows = @min(codec.score_block_entries, control.score_count - entry.block_index * codec.score_block_entries); + if (decoded.len != expected_rows or decoded.scores[0].orderNode(decoded.node_prefix, entry.first_node_id) != .eq) return error.InvalidGraphMetricSegment; + const local_index = entry.block_index - page.block_index; + const upper: ?[]const u8 = if (local_index + 1 < entries.len) entries[local_index + 1].first_node_id else if (page_index + 1 < directory.len) directory[page_index + 1].first_node_id else null; + if (upper) |bound| if (decoded.scores[decoded.len - 1].orderNode(decoded.node_prefix, bound) != .lt) return error.InvalidGraphMetricSegment; + var old: usize = 0; + var comparisons: usize = 0; + while (current < page_end and old < decoded.len) { + if (comparisons % 4096 == 0) try cancellation.check(); + comparisons += 1; + switch (decoded.scores[old].orderNode(decoded.node_prefix, current_node_ids[current])) { + .lt => old += 1, + .gt => current += 1, + .eq => { + seed[current] = decoded.scores[old].value; + positive_sum += seed[current]; + old += 1; + current += 1; + }, + } + } + } + index = end; + } + } + if (!std.math.isFinite(positive_sum) or positive_sum <= 0) return null; + seed_owned = false; + return seed; +} + +fn validateOptions(graph_payload: []const u8, options: BuildOptions) !void { + try (ComputeRuntime{ .io = options.io, .max_parallelism = options.max_parallelism }).validate(); + try graph_metric_policy.validateConfigs(&.{options.config}, options.limits); + if (graph_payload.len > options.limits.max_graph_payload_bytes) return error.GraphMetricBuildBudgetExceeded; + if (graph_payload.len == 0 or options.graph_index_name.len == 0 or options.config.name.len == 0 or options.source_graph.kind != .graph_segment or options.source_graph.artifact_id.len == 0 or options.source_graph.checksum.len == 0 or options.limits.max_metric_payload_bytes == 0) return error.InvalidGraphMetricBuildOptions; + if (options.source_graph.byte_len != graph_payload.len) return error.ArtifactIntegrityMismatch; + artifact_store.validateSha256ArtifactIdentity(options.source_graph.artifact_id, options.source_graph.checksum) catch return error.ArtifactIntegrityMismatch; + try artifact_store.validatePayloadSha256WithCancellation(graph_payload, options.source_graph.checksum, options.cancellation); + try graph_mod.validateGraphMetricEdgeFilters(&.{}, &.{options.config}); +} + +fn makeMetricSegmentAlloc(alloc: Allocator, options: BuildOptions, result: metrics.Result, scores: []metric_segment.Score) !metric_segment.Segment { + const source_artifact_id = try alloc.dupe(u8, options.source_graph.artifact_id); + errdefer alloc.free(source_artifact_id); + const source_checksum = try alloc.dupe(u8, options.source_graph.checksum); + errdefer alloc.free(source_checksum); + var edge_filter = try cloneSortedEdgeFilterAlloc(alloc, options.config.edge_filter); + errdefer edge_filter.deinit(alloc); + return .{ + .kind = options.config.kind, + .source_graph_artifact_id = source_artifact_id, + .source_graph_checksum = source_checksum, + .topology_checksum = options.topology_checksum, + .config_fingerprint = configFingerprint(options.config), + .materializer_fingerprint = graph_metric_policy.materializerFingerprint(options.limits), + .published_generation = options.provenance.published_generation, + .edge_generation = options.provenance.edge_generation, + .computed_at_ms = options.provenance.computed_at_ms, + .edge_filter = edge_filter, + .converged = result.converged, + .iterations_completed = result.iterations_completed, + .delta = result.delta, + .scores = scores, + .owns_score_node_ids = false, + }; +} + +fn makeScoresAlloc(alloc: Allocator, node_ids: []const []const u8, values: []const f64, cancellation: CancellationToken) ![]metric_segment.Score { + if (node_ids.len != values.len) return error.InvalidGraphMetricScore; + try cancellation.check(); + const scores = try alloc.alloc(metric_segment.Score, node_ids.len); + errdefer alloc.free(scores); + for (node_ids, values, 0..) |node_id, value, i| { + if (i % 4096 == 0) try cancellation.check(); + if (!std.math.isFinite(value)) return error.InvalidGraphMetricScore; + scores[i] = .{ .node_id = @constCast(node_id), .value = value }; + } + try cancellation.check(); + // Canonical graph artifacts already carry node IDs in lexical order and + // the compiled projection preserves that order. Keep the fallback for the + // legacy direct-projection test seam, but avoid an O(V log V) sort in the + // production path. + if (!std.sort.isSorted(metric_segment.Score, scores, {}, lessScore)) { + std.mem.sort(metric_segment.Score, scores, {}, lessScore); + } + try cancellation.check(); + return scores; +} + +fn getOrPutNode(alloc: Allocator, ordinals: *std.StringHashMapUnmanaged(usize), node_ids: *std.ArrayListUnmanaged([]const u8), node_id: []const u8, max_nodes: usize) !usize { + if (ordinals.get(node_id)) |ordinal| return ordinal; + if (node_ids.items.len >= max_nodes) return error.GraphMetricBuildBudgetExceeded; + const ordinal = node_ids.items.len; + try node_ids.append(alloc, node_id); + errdefer _ = node_ids.pop(); + try ordinals.put(alloc, node_id, ordinal); + return ordinal; +} + +fn cloneSortedEdgeFilterAlloc(alloc: Allocator, filter: graph_mod.GraphMetricEdgeFilter) !graph_mod.GraphMetricEdgeFilter { + if (filter.types.len == 0) return .{ .mode = filter.mode }; + const edge_types = try alloc.alloc([]const u8, filter.types.len); + errdefer alloc.free(edge_types); + var initialized: usize = 0; + errdefer for (edge_types[0..initialized]) |edge_type| alloc.free(edge_type); + for (filter.types, 0..) |edge_type, i| { + edge_types[i] = try alloc.dupe(u8, edge_type); + initialized += 1; + } + std.mem.sort([]const u8, edge_types, {}, lessString); + return .{ .mode = filter.mode, .types = edge_types }; +} +fn lessString(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); +} +fn lessScore(_: void, a: metric_segment.Score, b: metric_segment.Score) bool { + return std.mem.lessThan(u8, a.node_id, b.node_id); +} + +pub fn configFingerprint(config: graph_mod.GraphMetricConfig) u64 { + var hasher = std.hash.Wyhash.init(0); + hashU64(&hasher, @intFromEnum(config.kind)); + hashU64(&hasher, @bitCast(config.damping)); + hashU64(&hasher, @bitCast(config.tolerance)); + hashU64(&hasher, config.max_iterations); + hashU64(&hasher, @intFromEnum(config.edge_filter.mode)); + hashU64(&hasher, config.edge_filter.types.len); + const sorted = config.edge_filter.types; + // The storage fingerprint is order-independent. Avoid allocating by + // repeatedly selecting the next lexical value; config limits keep this tiny. + var last: ?[]const u8 = null; + for (0..sorted.len) |_| { + var next: ?[]const u8 = null; + for (sorted) |candidate| if ((last == null or std.mem.order(u8, candidate, last.?) == .gt) and (next == null or std.mem.lessThan(u8, candidate, next.?))) { + next = candidate; + }; + const value = next orelse break; + hashU64(&hasher, value.len); + hasher.update(value); + last = value; + } + const value = hasher.final() & std.math.maxInt(i64); + return if (value == 0) 1 else value; +} + +pub fn materializerFingerprint(limits: Limits) u64 { + return graph_metric_policy.materializerFingerprint(limits); +} +fn hashU64(hasher: *std.hash.Wyhash, value: u64) void { + var raw = value; + hasher.update(std.mem.asBytes(&raw)); +} + +pub fn freeArtifactRef(alloc: Allocator, artifact: artifact_ref.ArtifactRef) void { + if (artifact.name.len > 0) alloc.free(artifact.name); + alloc.free(artifact.artifact_id); + alloc.free(artifact.checksum); +} + +test "serverless graph metric decode admission includes the live source payload" { + try admitGraphDecodePeak(10, 20, 30); + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, admitGraphDecodePeak(10, 20, 29)); + try std.testing.expectError( + error.GraphMetricBuildBudgetExceeded, + admitGraphDecodePeak(std.math.maxInt(usize), 1, std.math.maxInt(usize)), + ); +} + +test "serverless graph metric impossible payload is rejected before artifact IO" { + const State = struct { + calls: usize = 0, + fn fail(ptr: *anyopaque) anyerror { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.calls += 1; + return error.UnexpectedArtifactIO; + } + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(ptr: *anyopaque, _: Allocator, _: []const u8) !artifact_store.ArtifactMetadata { + return fail(ptr); + } + fn get(ptr: *anyopaque, _: Allocator, _: []const u8) ![]u8 { + return fail(ptr); + } + fn range(ptr: *anyopaque, _: Allocator, _: []const u8, _: u64, _: usize) ![]u8 { + return fail(ptr); + } + fn stat(ptr: *anyopaque, _: Allocator, _: []const u8) !artifact_store.ArtifactMetadata { + return fail(ptr); + } + fn delete(ptr: *anyopaque, _: []const u8) !void { + return fail(ptr); + } + }; + var state = State{}; + var store = artifact_store.ArtifactStore{ .allocator = std.testing.allocator, .ptr = &state, .vtable = &.{ + .deinit = State.deinit, + .put = State.put, + .get_alloc = State.get, + .get_range_alloc = State.range, + .stat = State.stat, + .delete = State.delete, + } }; + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 }); + for ([_]u64{ 1024, 1025 }) |bytes| { + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, prepareGraphArtifactAlloc(failing.allocator(), &store, .{ + .kind = .graph_segment, + .artifact_id = "sha256:" ++ "0" ** 64, + .checksum = "0" ** 64, + .byte_len = bytes, + }, .none, .{ .max_graph_payload_bytes = 2048, .max_peak_memory_bytes = 1024 })); + } + try std.testing.expectEqual(@as(usize, 0), state.calls); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); +} + +test "serverless graph metric topology does not alias qualified endpoints with local nodes" { + const alloc = std.testing.allocator; + var graph = graph_segment.Segment{ + .neighbor_tables = try alloc.alloc([]u8, 1), + .adjacencies = try alloc.alloc(graph_segment.Adjacency, 2), + }; + defer graph.deinit(alloc); + graph.neighbor_tables[0] = try alloc.dupe(u8, "entities"); + graph.adjacencies[0] = .{ + .node_id = try alloc.dupe(u8, "source"), + .out_edges = try alloc.alloc(graph_segment.Edge, 2), + .in_edges = try alloc.alloc(graph_segment.Edge, 0), + }; + graph.adjacencies[0].out_edges[0] = .{ + .neighbor_id = try alloc.dupe(u8, "shared"), + .edge_type = try alloc.dupe(u8, "external"), + .weight = 1, + .neighbor_table_id = 0, + }; + graph.adjacencies[0].out_edges[1] = .{ + .neighbor_id = try alloc.dupe(u8, "shared"), + .edge_type = try alloc.dupe(u8, "local"), + .weight = 1, + }; + graph.adjacencies[1] = .{ + .node_id = try alloc.dupe(u8, "shared"), + .out_edges = try alloc.alloc(graph_segment.Edge, 0), + .in_edges = try alloc.alloc(graph_segment.Edge, 1), + }; + graph.adjacencies[1].in_edges[0] = .{ + .neighbor_id = try alloc.dupe(u8, "source"), + .edge_type = try alloc.dupe(u8, "local"), + .weight = 1, + }; + + var topology = try compileTopologyAlloc(alloc, graph, .none); + defer topology.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), topology.edges.len); + try std.testing.expectEqual(@as(usize, 1), topology.edge_types.len); + try std.testing.expectEqualStrings("local", topology.edge_types[0]); + + var projection = try buildProjectionFromTopologyAlloc(alloc, topology, 0, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = .{ .kind = .graph_segment, .name = "graph", .artifact_id = "sha256:placeholder", .byte_len = 1, .checksum = "placeholder" }, + }); + defer projection.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), projection.node_ids.items.len); + try std.testing.expectEqual(@as(usize, 1), projection.edgeCount()); + try std.testing.expectEqual(@as(usize, 0), projection.ordinals.count()); + const compact = projection.topology.?; + try std.testing.expectEqualSlices(u32, &.{ 0, 0, 1 }, compact.incoming_offsets); + try std.testing.expectEqualSlices(u32, &.{ 0, 1, 1 }, compact.outgoing_offsets); + try std.testing.expectEqual(@as(usize, 0), compact.incoming_sources.len); + try std.testing.expectEqual(@as(usize, 0), compact.outgoing_targets.len); +} + +test "serverless graph metric indexed preparation selects topology and cleans up allocation failures" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/indexed-topology", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + var builder = graph_segment.Builder{ .alloc = alloc }; + defer builder.deinit(); + try builder.addEdge("a", "b", "selected", 1, null); + try builder.addEdge("b", "a", "selected", 1, null); + try builder.addEdge("a", "foreign", "selected", 1, "elsewhere"); + for (0..4096) |i| { + const node = try std.fmt.allocPrint(alloc, "unrelated-{d:0>8}", .{i}); + defer alloc.free(node); + try builder.addEdge(node, node, "noise", 1, null); + } + const payload = try builder.encodeAlloc(16 * 1024 * 1024, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + var source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = metadata.artifact_id, .checksum = metadata.checksum, .byte_len = metadata.byte_len }; + try graph_segment.codec.compact.bindTopologyControl(&source, payload); + // Prime the reference verifier, then compare warm preparation against a + // separate cold store below. Bound block reads require neither priming. + try artifacts.verifyContentWithCancellationUsingAllocator(alloc, source.artifact_id, source.byte_len, source.checksum, .none); + const config = graph_mod.GraphMetricConfig{ .name = "degree", .kind = .degree, .edge_filter = .{ .mode = .types, .types = &.{"selected"} } }; + const reference = try benchmarkSelectedArtifactPreparation(alloc, &artifacts, source, config, true); + const indexed = try benchmarkSelectedArtifactPreparation(alloc, &artifacts, source, config, false); + try std.testing.expectEqualSlices(u8, &reference.digest, &indexed.digest); + try std.testing.expectEqual(@as(usize, 2), indexed.edges); + try std.testing.expectEqual(@as(usize, 2), indexed.retained_nodes); + // Authentication expands the two selected ranges to 64 KiB blocks. It + // still avoids the majority of this small fixture's full-source bytes. + try std.testing.expect(indexed.read_bytes < source.byte_len / 2); + var cold_fs = try fs_artifact_store.FsStore.init(alloc, root); + var cold_artifacts = cold_fs.artifactStore(); + defer cold_artifacts.deinit(); + var cold_budget = graph_metric_policy.Budget{ .limits = .{ .max_total_graph_payload_bytes = indexed.read_bytes } }; + var cold = (try prepareSelectedGraphArtifactAlloc(alloc, &cold_artifacts, source, &.{config}, .none, cold_budget.limits, &cold_budget)).?; + defer cold.deinit(alloc); + try std.testing.expectEqual(indexed.read_bytes, cold_budget.graph_payload_bytes); + var corrupted = source; + corrupted.graph_topology_control_checksum[0] ^= 1; + var bad_budget = graph_metric_policy.Budget{ .limits = .{} }; + try std.testing.expectError(error.ArtifactIntegrityMismatch, prepareSelectedGraphArtifactAlloc(alloc, &cold_artifacts, corrupted, &.{config}, .none, bad_budget.limits, &bad_budget)); + var remaining: u64 = 1024 * 1024; + var context = try indexed_topology.Context.init(alloc, &cold_artifacts, source, .none, &remaining); + defer context.deinit(); + // A response that disagrees with the authenticated block table is rejected + // even when footer/directory verification already succeeded. + @constCast(context.directory.?.block_checksums)[0] ^= 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, indexed_topology.readPreparedAlloc(alloc, &context, &[_]graph_mod.GraphMetricConfig{config}, bad_budget.limits, .none)); + const Runner = struct { + fn run(failing: Allocator, store: *artifact_store.ArtifactStore, ref: artifact_ref.ArtifactRef, cfg: graph_mod.GraphMetricConfig) !void { + var budget = graph_metric_policy.Budget{ .limits = .{} }; + var prepared = (try prepareSelectedGraphArtifactAlloc(failing, store, ref, &.{cfg}, .none, budget.limits, &budget)).?; + prepared.deinit(failing); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{ &artifacts, source, config }); + var tiny = graph_metric_policy.Budget{ .limits = .{ .max_peak_memory_bytes = 128 } }; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, prepareSelectedGraphArtifactAlloc(alloc, &artifacts, source, &.{config}, .none, tiny.limits, &tiny)); + var no_reads = graph_metric_policy.Budget{ .limits = .{ .max_total_graph_payload_bytes = 0 } }; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, prepareSelectedGraphArtifactAlloc(alloc, &artifacts, source, &.{config}, .none, no_reads.limits, &no_reads)); +} + +test "serverless graph metric semantic reuse authenticates current provenance and skips numerical work" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/semantic-reuse", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + const config = graph_mod.GraphMetricConfig{ .name = "rank", .kind = .pagerank, .max_iterations = 3, .edge_filter = .{ .mode = .types, .types = &.{"cites"} } }; + var prior: ?artifact_ref.ArtifactRef = null; + defer if (prior) |ref| freeArtifactRef(alloc, ref); + var first_id: []u8 = &.{}; + defer alloc.free(first_id); + for (0..3) |round| { + var graph = graph_segment.Builder{ .alloc = alloc }; + defer graph.deinit(); + try graph.addEdge("m", "n", "cites", if (round == 0) 1 else 9, null); + if (round > 0) { + // Also renumber all global ordinals and add qualified endpoints. + try graph.addEdge("a", "b", "other", 1, null); + try graph.addEdge("m", "external", "cites", 1, "other-table"); + try graph.addNode("isolated"); + } + if (round == 2) try graph.addEdge("n", "new", "cites", 1, null); + const payload = try graph.encodeAlloc(65536, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = metadata.artifact_id, .checksum = metadata.checksum, .byte_len = metadata.byte_len }; + const request = PublicationRequest{ .graph_index_name = "graph", .source_graph = source, .config = config, .prior_artifact = prior, .provenance = .{ .published_generation = round + 1, .edge_generation = round + 1, .computed_at_ms = (round + 1) * 10 } }; + var budget = graph_metric_policy.Budget{ .limits = .{} }; + const refs = try publishRequestsWithPriorAlloc(alloc, &artifacts, &.{request}, if (prior) |ref| &.{ref} else &.{}, .none, .{}, &budget, .{}); + defer alloc.free(refs); + if (prior) |ref| freeArtifactRef(alloc, ref); + prior = refs[0]; + const encoded = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, prior.?.artifact_id, prior.?.byte_len, prior.?.checksum, .none); + defer alloc.free(encoded); + const header = try metric_segment.decodeHeader(encoded); + try std.testing.expect(metricSourceMatches(header, source, prior.?)); + var corrupt = prior.?; + corrupt.graph_metric_topology_checksum[0] ^= 1; + try std.testing.expect(!metricSourceMatches(header, source, corrupt)); + corrupt = prior.?; + corrupt.graph_metric_source_checksum[0] ^= 1; + try std.testing.expect(!metricSourceMatches(header, source, corrupt)); + if (round == 0) first_id = try alloc.dupe(u8, prior.?.artifact_id) else if (round == 1) { + try std.testing.expectEqualStrings(first_id, prior.?.artifact_id); + try std.testing.expectEqual(@as(u64, 0), budget.work_items); + try std.testing.expectEqual(@as(u64, 0), budget.graph_payload_bytes); + try std.testing.expect(budget.identity_work_bytes > 0 and budget.identity_work_bytes < payload.len); + try std.testing.expectEqual(@as(u64, 10), prior.?.computed_at_ms); + try std.testing.expectEqual(@as(u64, 2), prior.?.edge_generation); + } else { + try std.testing.expect(!std.mem.eql(u8, first_id, prior.?.artifact_id)); + try std.testing.expect(budget.work_items > 0); + } + } +} + +test "serverless graph metric directory reuse preserves source admission limits" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/directory-admission", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + const config = graph_mod.GraphMetricConfig{ .name = "degree", .kind = .degree, .edge_filter = .{ .mode = .types, .types = &.{"cites"} } }; + const limits = Limits{ .max_nodes = 2 }; + var prior: ?artifact_ref.ArtifactRef = null; + defer if (prior) |ref| freeArtifactRef(alloc, ref); + for (0..2) |round| { + var builder = graph_segment.Builder{ .alloc = alloc }; + defer builder.deinit(); + try builder.addEdge("a", "b", "cites", 1, null); + if (round == 1) try builder.addEdge("c", "d", "unrelated", 1, null); + const payload = try builder.encodeAlloc(4096, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .artifact_id = metadata.artifact_id, .checksum = metadata.checksum, .byte_len = metadata.byte_len }; + const request = PublicationRequest{ .graph_index_name = "graph", .source_graph = source, .config = config, .prior_artifact = prior, .provenance = .{ .published_generation = round + 1, .edge_generation = round + 1, .computed_at_ms = round + 1 } }; + var budget = graph_metric_policy.Budget{ .limits = limits }; + const refs = try publishRequestsWithPriorAlloc(alloc, &artifacts, &.{request}, if (prior) |ref| &.{ref} else &.{}, .none, limits, &budget, .{}); + defer alloc.free(refs); + if (prior) |ref| freeArtifactRef(alloc, ref); + prior = refs[0]; + try std.testing.expectEqual(if (round == 0) artifact_ref.GraphMetricMaterializationState.ready else .rejected, prior.?.graph_metric_materialization_state); + } +} + +test "serverless graph metric topology admission tracks distinct edge types" { + const alloc = std.testing.allocator; + const edges = try alloc.alloc(graph_segment.Edge, 20_000); + defer alloc.free(edges); + for (edges) |*edge| edge.* = .{ + .neighbor_id = @constCast("b"), + .edge_type = @constCast("repeated"), + .weight = 1, + }; + var adjacencies = [_]graph_segment.Adjacency{ + .{ .node_id = @constCast("a"), .out_edges = edges, .in_edges = @constCast(&.{}) }, + .{ .node_id = @constCast("b"), .out_edges = @constCast(&.{}), .in_edges = @constCast(&.{}) }, + }; + const graph = graph_segment.Segment{ .adjacencies = &adjacencies }; + var topology = try compileTopologyWithinBudgetAlloc(alloc, graph, 0, 1024 * 1024, .none); + defer topology.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), topology.edge_types.len); + try std.testing.expectEqual(edges.len, topology.edges.len); + try std.testing.expectError( + error.GraphMetricBuildBudgetExceeded, + compileTopologyWithinBudgetAlloc(alloc, graph, 0, 1024, .none), + ); +} + +test "serverless packed topology matches reference kernels across filters and qualified edges" { + const alloc = std.testing.allocator; + var out_z = [_]graph_segment.Edge{ + .{ .neighbor_id = @constCast("a"), .edge_type = @constCast("alpha"), .weight = 1 }, + .{ .neighbor_id = @constCast("a"), .edge_type = @constCast("external"), .weight = 1, .neighbor_table_id = 0 }, + .{ .neighbor_id = @constCast("m"), .edge_type = @constCast("zeta"), .weight = 2 }, + }; + var out_a = [_]graph_segment.Edge{ + .{ .neighbor_id = @constCast("m"), .edge_type = @constCast("alpha"), .weight = 1 }, + }; + var out_m = [_]graph_segment.Edge{ + .{ .neighbor_id = @constCast("z"), .edge_type = @constCast("zeta"), .weight = 1 }, + }; + var adjacencies = [_]graph_segment.Adjacency{ + .{ .node_id = @constCast("z"), .out_edges = &out_z, .in_edges = @constCast(&.{}) }, + .{ .node_id = @constCast("a"), .out_edges = &out_a, .in_edges = @constCast(&.{}) }, + .{ .node_id = @constCast("m"), .out_edges = &out_m, .in_edges = @constCast(&.{}) }, + .{ .node_id = @constCast("isolated"), .out_edges = @constCast(&.{}), .in_edges = @constCast(&.{}) }, + }; + var tables = [_][]u8{@constCast("entities")}; + const graph = graph_segment.Segment{ .adjacencies = &adjacencies, .neighbor_tables = &tables }; + const payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(payload); + var packed_topology = try prepareTopologyFromPackedAlloc(alloc, payload, .none, .{}); + defer packed_topology.deinit(alloc); + var reference = try compileTopologyAlloc(alloc, graph, .none); + defer reference.deinit(alloc); + const Runner = struct { + fn prepare(failing_alloc: Allocator, input: []const u8) !void { + var topology = try prepareTopologyFromPackedAlloc(failing_alloc, input, .none, .{}); + defer topology.deinit(failing_alloc); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.prepare, .{payload}); + var filter_types = [_][]u8{@constCast("alpha")}; + for ([_]graph_mod.GraphMetricEdgeFilter{ .{}, .{ .mode = .types, .types = &filter_types } }) |filter| { + for ([_]graph_mod.GraphMetricKind{ .degree, .pagerank, .eigenvector, .hits_authority, .hits_hub }) |kind| { + const options = BuildOptions{ + .graph_index_name = "graph", + .config = .{ .name = "metric", .kind = kind, .edge_filter = filter }, + .source_graph = .{ .kind = .graph_segment, .name = "graph", .artifact_id = "sha256:placeholder", .byte_len = payload.len, .checksum = "placeholder" }, + }; + var a = try buildProjectionFromTopologyAlloc(alloc, reference, 0, options); + defer a.deinit(alloc); + var b = try buildProjectionFromTopologyAlloc(alloc, packed_topology, 0, options); + defer b.deinit(alloc); + try std.testing.expectEqual(a.edgeCount(), b.edgeCount()); + try std.testing.expectEqual(a.node_ids.items.len, b.node_ids.items.len); + for (a.node_ids.items, b.node_ids.items) |left, right| try std.testing.expectEqualStrings(left, right); + const ta = a.topology.?; + const tb = b.topology.?; + try std.testing.expectEqualSlices(u32, ta.incoming_offsets, tb.incoming_offsets); + try std.testing.expectEqualSlices(u32, ta.incoming_sources, tb.incoming_sources); + try std.testing.expectEqualSlices(u32, ta.outgoing_offsets, tb.outgoing_offsets); + try std.testing.expectEqualSlices(u32, ta.outgoing_targets, tb.outgoing_targets); + } + } +} + +test "serverless lake graph metrics build immutable pagerank and degree vectors" { + const alloc = std.testing.allocator; + var graph = graph_segment.Segment{ .adjacencies = try alloc.alloc(graph_segment.Adjacency, 3) }; + defer graph.deinit(alloc); + graph.adjacencies[0] = .{ .node_id = try alloc.dupe(u8, "a"), .out_edges = try alloc.alloc(graph_segment.Edge, 1), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + graph.adjacencies[0].out_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "b"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[1] = .{ .node_id = try alloc.dupe(u8, "b"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 1) }; + graph.adjacencies[1].in_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "a"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[2] = .{ .node_id = try alloc.dupe(u8, "isolated"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + const graph_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(graph_payload); + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(graph_payload, &digest, .{}); + const checksum = std.fmt.bytesToHex(digest, .lower); + var artifact_id_buf: [artifact_store.sha256_artifact_id_prefix.len + checksum.len]u8 = undefined; + const artifact_id = try std.fmt.bufPrint(&artifact_id_buf, "{s}{s}", .{ artifact_store.sha256_artifact_id_prefix, checksum }); + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = artifact_id, .byte_len = graph_payload.len, .checksum = &checksum }; + var built = try buildFromGraphPayloadAlloc(alloc, graph_payload, .{ .graph_index_name = "graph", .config = .{ .name = "pagerank" }, .source_graph = source }); + defer built.deinit(alloc); + var decoded = try metric_segment.decodeAlloc(alloc, built.payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), decoded.scores.len); + try std.testing.expect(decoded.score("b").? > decoded.score("a").?); + try std.testing.expect(decoded.score("isolated") == null); + try std.testing.expectEqualStrings("5:graph8:pagerank", built.artifact.name); + + var tampered = try alloc.dupe(u8, graph_payload); + defer alloc.free(tampered); + tampered[tampered.len - 1] ^= 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, buildFromGraphPayloadAlloc(alloc, tampered, .{ + .graph_index_name = "graph", + .config = .{ .name = "pagerank" }, + .source_graph = source, + })); + var wrong_length = source; + wrong_length.byte_len += 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, buildFromGraphPayloadAlloc(alloc, graph_payload, .{ + .graph_index_name = "graph", + .config = .{ .name = "pagerank" }, + .source_graph = wrong_length, + })); +} + +test "serverless graph metric filter groups isolate preparation admission and reuse authenticated digests" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/filter-groups", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + var builder = graph_segment.Builder{ .alloc = alloc }; + defer builder.deinit(); + try builder.addEdge("a", "b", "selected", 1, null); + for (0..65536) |i| { + const node = try std.fmt.allocPrint(alloc, "noise-{d:0>8}", .{i}); + defer alloc.free(node); + try builder.addEdge(node, node, "noise", 1, null); + } + const payload = try builder.encodeAlloc(16 * 1024 * 1024, .none); + defer alloc.free(payload); + var metadata = try artifacts.put(payload); + defer metadata.deinit(alloc); + var source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = metadata.artifact_id, .checksum = metadata.checksum, .byte_len = metadata.byte_len }; + try graph_segment.codec.compact.bindTopologyControl(&source, payload); + const small = PublicationRequest{ .graph_index_name = "graph", .source_graph = source, .config = .{ .name = "small", .kind = .degree, .edge_filter = .{ .mode = .types, .types = &.{"selected"} } }, .provenance = .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 } }; + var broad = small; + broad.config = .{ .name = "broad", .kind = .degree }; + for ([_][2]PublicationRequest{ .{ broad, small }, .{ small, broad } }) |requests| { + var budget = graph_metric_policy.Budget{ .limits = .{ .max_peak_memory_bytes = 2 * 1024 * 1024, .max_total_identity_work_bytes = 1 } }; + const results = try publishRequestsAlloc(alloc, &artifacts, &requests, .none, budget.limits, &budget, .{}); + defer { + for (results) |ref| freeArtifactRef(alloc, ref); + alloc.free(results); + } + for (requests, results) |request, result| { + try std.testing.expectEqual(if (request.config.edge_filter.mode == .all) artifact_ref.GraphMetricMaterializationState.rejected else .ready, result.graph_metric_materialization_state); + if (request.config.edge_filter.mode == .types) try std.testing.expect(!std.mem.eql(u8, &result.graph_metric_topology_checksum, &@as([32]u8, @splat(0)))); + } + try std.testing.expectEqual(@as(u64, 0), budget.identity_work_bytes); + } +} + +test "serverless lake graph metrics persist a budget rejection with exact provenance" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/graph-metric-artifacts", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + + var graph = graph_segment.Segment{ .adjacencies = try alloc.alloc(graph_segment.Adjacency, 2) }; + defer graph.deinit(alloc); + graph.adjacencies[0] = .{ .node_id = try alloc.dupe(u8, "a"), .out_edges = try alloc.alloc(graph_segment.Edge, 1), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + graph.adjacencies[0].out_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "b"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[1] = .{ .node_id = try alloc.dupe(u8, "b"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 1) }; + graph.adjacencies[1].in_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "a"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + const graph_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(graph_payload); + var source_metadata = try artifacts.put(graph_payload); + defer source_metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ + .kind = .graph_segment, + .name = "graph", + .artifact_id = source_metadata.artifact_id, + .byte_len = source_metadata.byte_len, + .checksum = source_metadata.checksum, + }; + const configs = [_]graph_mod.GraphMetricConfig{.{ .name = "degree", .kind = .degree }}; + const published = try publishManyFromGraphArtifactAlloc(alloc, &artifacts, "graph", source, &configs, .none, .{ .max_nodes = 1 }, .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }); + defer { + for (published) |ref| freeArtifactRef(alloc, ref); + alloc.free(published); + } + try std.testing.expectEqual(@as(usize, 1), published.len); + const payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator( + alloc, + published[0].artifact_id, + published[0].byte_len, + published[0].checksum, + .none, + ); + defer alloc.free(payload); + var decoded = try metric_segment.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(metric_segment.MaterializationState.rejected, decoded.materialization_state); + try std.testing.expectEqual(metric_segment.RejectionReason.build_budget_exceeded, decoded.rejection_reason); + try std.testing.expectEqualStrings(source.artifact_id, decoded.source_graph_artifact_id); + try std.testing.expectEqualStrings(source.checksum, decoded.source_graph_checksum); + try std.testing.expectEqual(configFingerprint(configs[0]), decoded.config_fingerprint); + try std.testing.expectEqual(materializerFingerprint(.{ .max_nodes = 1 }), decoded.materializer_fingerprint); +} + +test "serverless graph metric warm starts read bounded sparse and dense primary windows" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root_path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/seed-windows", .{tmp.sub_path}); + defer alloc.free(root_path); + var fs = try fs_artifact_store.FsStore.init(alloc, root_path); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + const checksum = "a" ** 64; + const config = graph_mod.GraphMetricConfig{ .name = "rank" }; + var segment = metric_segment.Segment{ + .kind = .pagerank, + .source_graph_artifact_id = try alloc.dupe(u8, "sha256:" ++ checksum), + .source_graph_checksum = try alloc.dupe(u8, checksum), + .config_fingerprint = configFingerprint(config), + .materializer_fingerprint = materializerFingerprint(.{}), + .edge_filter = .{}, + .converged = true, + .iterations_completed = 2, + .delta = 0, + .scores = try alloc.alloc(metric_segment.Score, 8192), + }; + for (segment.scores, 0..) |*score, i| score.* = .{ + .node_id = try std.fmt.allocPrint(alloc, "node:{d:0>8}", .{i}), + .value = @floatFromInt(i + 1), + }; + defer segment.deinit(alloc); + const payload = try metric_segment.encodeAlloc(alloc, segment); + defer alloc.free(payload); + var meta = try artifacts.put(payload); + defer meta.deinit(alloc); + var ref = artifact_ref.ArtifactRef{ .kind = .graph_metric_segment, .artifact_id = meta.artifact_id, .checksum = meta.checksum, .byte_len = meta.byte_len, .metadata_version = metric_segment.wire_version }; + try populateGraphMetricIntegrity(&ref, segment, payload); + // Establish the provider's immutable verification identity before measuring + // warm reads; unverified full-object costs must still be paid separately. + var verification_budget: u64 = ref.byte_len + 1; + const verified = try artifacts.getVerifiedRangeAllocWithBudget(alloc, ref.artifact_id, ref.byte_len, ref.checksum, 0, 1, .none, &verification_budget); + alloc.free(verified); + var sparse_budget = graph_metric_policy.Budget{ .limits = .{ .max_peak_memory_bytes = 64 * 1024, .max_total_seed_payload_bytes = 64 * 1024 } }; + try std.testing.expect(ref.byte_len > sparse_budget.limits.max_total_seed_payload_bytes); + const ids = [_][]const u8{ "absent-before", segment.scores[4096].node_id, "zzzz" }; + const sparse = (try warmStartVectorAlloc(alloc, &artifacts, ref, &ids, config, .none, sparse_budget.limits, 0, 0, &sparse_budget)) orelse return error.TestExpectedWarmSeed; + defer alloc.free(sparse); + try std.testing.expectEqualSlices(f64, &.{ 0, 4097, 0 }, sparse); + try std.testing.expect(sparse_budget.seed_payload_bytes < ref.byte_len / 2); + const all_ids = try alloc.alloc([]const u8, segment.scores.len); + defer alloc.free(all_ids); + for (segment.scores, all_ids) |score, *id| id.* = score.node_id; + // The entire primary lane plus the seed cannot coexist in this allowance; + // the coalescer must shrink windows to remaining live-allocation headroom. + var dense_budget = graph_metric_policy.Budget{ .limits = .{ .max_peak_memory_bytes = 128 * 1024 } }; + const dense = (try warmStartVectorAlloc(alloc, &artifacts, ref, all_ids, config, .none, dense_budget.limits, 0, 0, &dense_budget)).?; + defer alloc.free(dense); + for (dense, segment.scores) |value, score| try std.testing.expectEqual(score.value, value); + try std.testing.expect(dense_budget.seed_payload_bytes < ref.byte_len); + // Rejection after a partial read accounts the bytes and releases the seed. + var exhausted = graph_metric_policy.Budget{ .limits = .{ .max_total_seed_payload_bytes = ref.graph_metric_control_len } }; + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, ref, &ids, config, .none, exhausted.limits, 0, 0, &exhausted)) == null); + try std.testing.expectEqual(ref.graph_metric_control_len, exhausted.seed_payload_bytes); +} + +test "serverless graph metric warm start maps an authenticated prior vector onto new ordinals" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/graph-metric-warm-start", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + + var graph = graph_segment.Segment{ .adjacencies = try alloc.alloc(graph_segment.Adjacency, 2) }; + defer graph.deinit(alloc); + graph.adjacencies[0] = .{ .node_id = try alloc.dupe(u8, "a"), .out_edges = try alloc.alloc(graph_segment.Edge, 1), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + graph.adjacencies[0].out_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "b"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[1] = .{ .node_id = try alloc.dupe(u8, "b"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 1) }; + graph.adjacencies[1].in_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "a"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + const graph_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(graph_payload); + var source_metadata = try artifacts.put(graph_payload); + defer source_metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ + .kind = .graph_segment, + .name = "graph", + .artifact_id = source_metadata.artifact_id, + .byte_len = source_metadata.byte_len, + .checksum = source_metadata.checksum, + }; + const config = graph_mod.GraphMetricConfig{ .name = "rank", .kind = .pagerank }; + const prior = try publishFromGraphArtifactAlloc(alloc, &artifacts, .{ + .graph_index_name = "graph", + .config = config, + .source_graph = source, + .provenance = .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, + }); + defer freeArtifactRef(alloc, prior); + + const current_nodes = [_][]const u8{ "a", "b", "c" }; + const seed = (try warmStartVectorAlloc(alloc, &artifacts, prior, ¤t_nodes, config, .none, .{}, 0, 0, null)).?; + defer alloc.free(seed); + try std.testing.expect(seed[0] > 0); + try std.testing.expect(seed[1] > 0); + try std.testing.expectEqual(@as(f64, 0), seed[2]); + + // Optional preparation must fall back before fetching an artifact if its + // retained vector would crowd out otherwise admissible cold execution. + var unavailable = prior; + unavailable.artifact_id = "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + unavailable.checksum = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, unavailable, ¤t_nodes, config, .none, .{}, 0, 0, null)) == null); + var mismatched = prior; + mismatched.byte_len += 1; + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, mismatched, ¤t_nodes, config, .none, .{}, 0, 0, null)) == null); + var canceled = std.atomic.Value(bool).init(true); + try std.testing.expectError(error.Canceled, warmStartVectorAlloc(alloc, &artifacts, prior, ¤t_nodes, config, CancellationToken.fromAtomic(&canceled), .{}, 0, 0, null)); + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.OutOfMemory, warmStartVectorAlloc(failing.allocator(), &artifacts, prior, ¤t_nodes, config, .none, .{}, 0, 0, null)); + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, unavailable, ¤t_nodes, config, .none, .{}, 0, (Limits{}).max_peak_memory_bytes, null)) == null); + var exhausted = graph_metric_policy.Budget{ .limits = .{ .max_total_seed_payload_bytes = 0 } }; + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, unavailable, ¤t_nodes, config, .none, exhausted.limits, 0, 0, &exhausted)) == null); + var spectral = config; + spectral.kind = .eigenvector; + try std.testing.expect((try warmStartVectorAlloc(alloc, &artifacts, unavailable, ¤t_nodes, spectral, .none, .{}, 0, 0, null)) == null); + + // A content-addressed but semantically malformed score block is not a + // usable seed. The nullable fallback must also release the dense vector it + // allocated before block decoding failed. + const prior_payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator( + alloc, + prior.artifact_id, + prior.byte_len, + prior.checksum, + .none, + ); + defer alloc.free(prior_payload); + const prior_control = try metric_segment.decodeControl(prior_payload, config.edge_filter); + const score_offset = std.math.cast(usize, prior_control.score_data_offset) orelse return error.TestUnexpectedResult; + if (prior_payload.len - score_offset < 2) return error.TestUnexpectedResult; + const malformed_payload = try alloc.dupe(u8, prior_payload); + defer alloc.free(malformed_payload); + @memset(malformed_payload[score_offset..][0..2], 0xff); + var malformed_metadata = try artifacts.put(malformed_payload); + defer malformed_metadata.deinit(alloc); + var malformed_prior = prior; + malformed_prior.artifact_id = malformed_metadata.artifact_id; + malformed_prior.byte_len = malformed_metadata.byte_len; + malformed_prior.checksum = malformed_metadata.checksum; + try std.testing.expect((try warmStartVectorAlloc( + alloc, + &artifacts, + malformed_prior, + ¤t_nodes, + config, + .none, + .{}, + 0, + 0, + null, + )) == null); +} + +test "serverless graph metric projection bounds filtered scans and inner-loop cancellation" { + const alloc = std.testing.allocator; + const edges = try alloc.alloc(graph_segment.Edge, 4096); + defer alloc.free(edges); + for (edges) |*edge| edge.* = .{ + .neighbor_id = @constCast("b"), + .edge_type = @constCast("ignored"), + .weight = 1, + }; + var adjacencies = [_]graph_segment.Adjacency{.{ + .node_id = @constCast("a"), + .out_edges = edges, + .in_edges = @constCast(&.{}), + }}; + const graph = graph_segment.Segment{ .adjacencies = &adjacencies }; + const source = artifact_ref.ArtifactRef{ + .kind = .graph_segment, + .artifact_id = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + .byte_len = 1, + .checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }; + const filtered = graph_mod.GraphMetricConfig{ + .name = "degree", + .kind = .degree, + .edge_filter = .{ .mode = .types, .types = &.{"wanted"} }, + }; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, buildProjectionAlloc(alloc, graph, .{ + .graph_index_name = "graph", + .config = filtered, + .source_graph = source, + .limits = .{ .max_edges = 4095 }, + })); + + const State = struct { + calls: usize = 0, + + fn cancelled(ptr: *const anyopaque) bool { + const self: *@This() = @ptrCast(@alignCast(@constCast(ptr))); + self.calls += 1; + return self.calls >= 3; + } + }; + var state = State{}; + const cancellation = CancellationToken{ .ptr = &state, .is_cancelled_fn = State.cancelled }; + try std.testing.expectError(error.Canceled, buildProjectionAlloc(alloc, graph, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = source, + .cancellation = cancellation, + .limits = .{ .max_edges = edges.len }, + })); + try std.testing.expectEqual(@as(usize, 3), state.calls); + + var admitted_projection = try buildProjectionAlloc(alloc, graph, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = source, + .limits = .{ .max_edges = edges.len }, + }); + defer admitted_projection.deinit(alloc); + admitted_projection.decoded_retained_bytes = 1024; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, admitPeakMemory(admitted_projection, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = source, + .limits = .{ .max_edges = edges.len, .max_peak_memory_bytes = 1024 }, + }, 1)); + + const one_output_peak = try estimatedPeakMemoryBytes(admitted_projection, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = source, + .limits = .{ .max_edges = edges.len }, + }, 1); + const two_output_peak = try estimatedPeakMemoryBytes(admitted_projection, .{ + .graph_index_name = "graph", + .config = .{ .name = "degree", .kind = .degree }, + .source_graph = source, + .limits = .{ .max_edges = edges.len }, + }, 2); + try std.testing.expect(two_output_peak > one_output_peak); +} + +test "serverless graph metric projection admits census before allocations and bounds rejected scratch" { + const alloc = std.testing.allocator; + const topology = CompiledTopology{ + .node_ids = &.{ "a", "b" }, + .edge_types = &.{"cites"}, + .string_bytes = &.{}, + .edge_type_offsets = &.{ 0, 1 }, + .edges = &.{.{ .source = 0, .target = 1 }}, + .source_node_count = 2, + .source_edge_count = 1, + .retained_bytes = 128, + }; + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "fixture", .byte_len = 1 }; + var exhausted = graph_metric_policy.Budget{ .limits = .{ .max_total_work_items = 0 } }; + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, buildProjectionFromTopologyAlloc(failing.allocator(), topology, 0, .{ + .graph_index_name = "graph", + .config = .{ .name = "rank" }, + .source_graph = source, + .batch_budget = &exhausted, + })); + try std.testing.expect(!failing.has_induced_failure); + for ([_]graph_mod.GraphMetricKind{ .degree, .pagerank }) |kind| { + var budget = graph_metric_policy.Budget{ .limits = .{} }; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, buildProjectionFromTopologyAlloc(alloc, topology, 0, .{ + .graph_index_name = "graph", + .config = .{ .name = "metric", .kind = kind }, + .source_graph = source, + .batch_budget = &budget, + .limits = .{ .max_peak_memory_bytes = topology.retained_bytes + 1 }, + })); + // A failed preparation still consumes its reserved census allowance. + try std.testing.expectEqual(@as(u64, 4), budget.work_items); + } +} + +test "serverless graph metric output admission rejects before kernels and reserves pairs atomically" { + const alloc = std.testing.allocator; + const topology = CompiledTopology{ + .node_ids = &.{ "a", "b" }, + .edge_types = &.{"cites"}, + .string_bytes = &.{}, + .edge_type_offsets = &.{ 0, 1 }, + .edges = &.{.{ .source = 0, .target = 1 }}, + .source_node_count = 2, + .source_edge_count = 1, + .retained_bytes = 128, + }; + var budget = graph_metric_policy.Budget{ .limits = .{ .max_total_metric_payload_bytes = 0 } }; + var options = BuildOptions{ + .graph_index_name = "graph", + .config = .{ .name = "authority", .kind = .hits_authority }, + .source_graph = .{ .kind = .graph_segment, .artifact_id = "fixture", .checksum = "a" ** 64, .byte_len = 1 }, + .batch_budget = &budget, + }; + var projection = try buildProjectionFromTopologyAlloc(alloc, topology, 0, options); + defer projection.deinit(alloc); + const before = budget.work_items; + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, buildFromProjectionAlloc(failing.allocator(), projection, options)); + try std.testing.expect(!failing.has_induced_failure); + try std.testing.expectEqual(before, budget.work_items); + // Undefined store is intentional: admission must fail before any upload. + var artifacts: artifact_store.ArtifactStore = undefined; + const hub = graph_mod.GraphMetricConfig{ .name = "hub", .kind = .hits_hub }; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, publishHitsPairFromProjectionAlloc(failing.allocator(), &artifacts, projection, options, hub, .none)); + try std.testing.expect(!failing.has_induced_failure); + try std.testing.expectEqual(before, budget.work_items); + + var pair = try metrics.hitsTopologyAlloc(alloc, projection.topology.?, kernelOptions(options)); + defer pair.deinit(alloc); + const result = metrics.Result{ .scores = pair.authorities, .iterations_completed = pair.iterations_completed, .converged = pair.converged, .delta = pair.delta }; + const first_plan = try prepareMetricOutputPlan(alloc, projection.node_ids.items, options, result); + var second = options; + second.config = hub; + const second_plan = try prepareMetricOutputPlan(alloc, projection.node_ids.items, second, .{ .scores = pair.hubs, .iterations_completed = pair.iterations_completed, .converged = pair.converged, .delta = pair.delta }); + budget.limits.max_total_metric_payload_bytes = first_plan.size + second_plan.size - 1; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, publishHitsPairFromProjectionAlloc(alloc, &artifacts, projection, options, hub, .none)); + try std.testing.expectEqual(@as(usize, 0), budget.metric_payload_bytes); + // Exact single-output admission precedes payload allocation, and every + // allocation failure after reservation refunds the entire reservation. + budget.limits.max_total_metric_payload_bytes = first_plan.size; + var succeeded = false; + for (0..32) |fail_index| { + var injected = std.testing.FailingAllocator.init(alloc, .{ .fail_index = fail_index }); + var built = encodeMetricResultAlloc(injected.allocator(), projection.node_ids.items, options, result) catch |err| { + try std.testing.expectEqual(error.OutOfMemory, err); + try std.testing.expectEqual(@as(usize, 0), budget.metric_payload_bytes); + continue; + }; + defer built.deinit(injected.allocator()); + try std.testing.expectEqual(first_plan.size, built.payload.len); + try std.testing.expectEqual(first_plan.size, budget.metric_payload_bytes); + succeeded = true; + break; + } + try std.testing.expect(succeeded); + budget.metric_payload_bytes = 0; + options.source_graph.checksum = "invalid"; + try std.testing.expectError(error.ArtifactIntegrityMismatch, encodeMetricResultAlloc(alloc, projection.node_ids.items, options, result)); + try std.testing.expectEqual(@as(usize, 0), budget.metric_payload_bytes); + options.limits.max_metric_payload_bytes = 0; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, buildFromProjectionAlloc(failing.allocator(), projection, options)); +} + +test "serverless lake graph metrics share one bounded HITS execution for a compatible pair" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/graph-metric-hits", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + + var graph = graph_segment.Segment{ .adjacencies = try alloc.alloc(graph_segment.Adjacency, 2) }; + defer graph.deinit(alloc); + graph.adjacencies[0] = .{ .node_id = try alloc.dupe(u8, "a"), .out_edges = try alloc.alloc(graph_segment.Edge, 1), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + graph.adjacencies[0].out_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "b"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[1] = .{ .node_id = try alloc.dupe(u8, "b"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 1) }; + graph.adjacencies[1].in_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "a"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + const graph_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(graph_payload); + var source_metadata = try artifacts.put(graph_payload); + defer source_metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = source_metadata.artifact_id, .byte_len = source_metadata.byte_len, .checksum = source_metadata.checksum }; + const configs = [_]graph_mod.GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority }, + .{ .name = "hub", .kind = .hits_hub }, + }; + // 604 HITS kernel work items plus sixteen projection/filter work items. + const limits = Limits{ .max_work_items = 620, .max_total_work_items = 620 }; + const published = try publishManyFromGraphArtifactAlloc(alloc, &artifacts, "graph", source, &configs, .none, limits, .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }); + defer { + for (published) |ref| freeArtifactRef(alloc, ref); + alloc.free(published); + } + try std.testing.expectEqual(@as(usize, 2), published.len); + for (published) |ref| { + const payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, ref.artifact_id, ref.byte_len, ref.checksum, .none); + defer alloc.free(payload); + var decoded = try metric_segment.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(metric_segment.MaterializationState.ready, decoded.materialization_state); + try std.testing.expectEqual(materializerFingerprint(limits), decoded.materializer_fingerprint); + } + + const degree_configs = [_]graph_mod.GraphMetricConfig{ + .{ .name = "degree_a", .kind = .degree }, + .{ .name = "degree_b", .kind = .degree }, + }; + // Fifteen projection/index work items are charged once for the shared edge + // scope; each degree kernel then consumes two more. Rebuilding the + // projection per metric would exceed this exact 19-item budget. + const shared_projection_limits = Limits{ .max_work_items = 19, .max_total_work_items = 19 }; + const degree_published = try publishManyFromGraphArtifactAlloc( + alloc, + &artifacts, + "graph", + source, + °ree_configs, + .none, + shared_projection_limits, + .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, + ); + defer { + for (degree_published) |ref| freeArtifactRef(alloc, ref); + alloc.free(degree_published); + } + for (degree_published) |ref| { + const payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, ref.artifact_id, ref.byte_len, ref.checksum, .none); + defer alloc.free(payload); + var decoded = try metric_segment.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(metric_segment.MaterializationState.ready, decoded.materialization_state); + } +} + +test "serverless lake graph metrics reject work beyond the aggregate publication budget" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/graph-metric-aggregate", .{tmp.sub_path}); + defer alloc.free(root); + var fs = try fs_artifact_store.FsStore.init(alloc, root); + var artifacts = fs.artifactStore(); + defer artifacts.deinit(); + + var graph = graph_segment.Segment{ .adjacencies = try alloc.alloc(graph_segment.Adjacency, 2) }; + defer graph.deinit(alloc); + graph.adjacencies[0] = .{ .node_id = try alloc.dupe(u8, "a"), .out_edges = try alloc.alloc(graph_segment.Edge, 1), .in_edges = try alloc.alloc(graph_segment.Edge, 0) }; + graph.adjacencies[0].out_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "b"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + graph.adjacencies[1] = .{ .node_id = try alloc.dupe(u8, "b"), .out_edges = try alloc.alloc(graph_segment.Edge, 0), .in_edges = try alloc.alloc(graph_segment.Edge, 1) }; + graph.adjacencies[1].in_edges[0] = .{ .neighbor_id = try alloc.dupe(u8, "a"), .edge_type = try alloc.dupe(u8, "cites"), .weight = 1 }; + const graph_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(graph_payload); + var source_metadata = try artifacts.put(graph_payload); + defer source_metadata.deinit(alloc); + const source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "graph", .artifact_id = source_metadata.artifact_id, .byte_len = source_metadata.byte_len, .checksum = source_metadata.checksum }; + const configs = [_]graph_mod.GraphMetricConfig{ + .{ .name = "rank_a", .kind = .pagerank }, + .{ .name = "rank_b", .kind = .pagerank, .damping = 0.75 }, + }; + // One PageRank consumes 254 kernel work items plus sixteen projection + // items; the table-wide budget admits the first and rejects the second. + const published = try publishManyFromGraphArtifactAlloc(alloc, &artifacts, "graph", source, &configs, .none, .{ .max_work_items = 270, .max_total_work_items = 270 }, .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }); + defer { + for (published) |ref| freeArtifactRef(alloc, ref); + alloc.free(published); + } + for (published, 0..) |ref, i| { + const payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, ref.artifact_id, ref.byte_len, ref.checksum, .none); + defer alloc.free(payload); + var decoded = try metric_segment.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(if (i == 0) metric_segment.MaterializationState.ready else .rejected, decoded.materialization_state); + } + + // An unaffordable PageRank must not force its larger CSR projection on a + // degree metric. Degree fits exactly: source 3 + filter 1 + projection 9 + kernel 2. + const pressure = try publishManyFromGraphArtifactAlloc(alloc, &artifacts, "pressure", source, &.{ + .{ .name = "rank", .kind = .pagerank }, + .{ .name = "degree", .kind = .degree }, + }, .none, .{ .max_work_items = 15, .max_total_work_items = 15 }, .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }); + defer { + for (pressure) |ref| freeArtifactRef(alloc, ref); + alloc.free(pressure); + } + try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.rejected, pressure[0].graph_metric_materialization_state); + try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.ready, pressure[1].graph_metric_materialization_state); + + var shared_budget = graph_metric_policy.Budget{ .limits = .{ .max_work_items = 270, .max_total_work_items = 270 } }; + const one_config = [_]graph_mod.GraphMetricConfig{.{ .name = "shared_rank", .kind = .pagerank }}; + var prepared = try prepareGraphArtifactAlloc(alloc, &artifacts, source, .none, shared_budget.limits); + defer prepared.deinit(alloc); + try std.testing.expect(prepared.identifies(source)); + // Exercise the real publication entry point: failures in either scratch + // allocation must release the already allocated result array. + for ([_]usize{ 1, 2 }) |fail_index| { + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = fail_index }); + var failure_budget = graph_metric_policy.Budget{ .limits = .{} }; + try std.testing.expectError(error.OutOfMemory, publishManyFromPreparedGraphWithBudgetAlloc( + failing.allocator(), + &artifacts, + "graph", + source, + &one_config, + .none, + failure_budget.limits, + &failure_budget, + &prepared, + .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, + .{}, + )); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); + } + var wrong_source = source; + wrong_source.byte_len += 1; + try std.testing.expect(!prepared.identifies(wrong_source)); + const first_index = try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + &artifacts, + "graph_a", + source, + &one_config, + .none, + shared_budget.limits, + &shared_budget, + &prepared, + .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, + .{}, + ); + defer { + for (first_index) |ref| freeArtifactRef(alloc, ref); + alloc.free(first_index); + } + const second_index = try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + &artifacts, + "graph_b", + source, + &one_config, + .none, + shared_budget.limits, + &shared_budget, + &prepared, + .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, + .{}, + ); + defer { + for (second_index) |ref| freeArtifactRef(alloc, ref); + alloc.free(second_index); + } + const expected_states = [_]metric_segment.MaterializationState{ .ready, .rejected }; + for ([_][]const artifact_ref.ArtifactRef{ first_index, second_index }, expected_states) |refs, expected_state| { + const payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, refs[0].artifact_id, refs[0].byte_len, refs[0].checksum, .none); + defer alloc.free(payload); + var decoded = try metric_segment.decodeAlloc(alloc, payload); + defer decoded.deinit(alloc); + try std.testing.expectEqual(expected_state, decoded.materialization_state); + } + + // Logical aliases must not perturb immutable metric content. With enough + // aggregate work budget both publications reuse the cached projection and + // converge on the same object-store identity; only their manifest names + // differ. + // Two 254-item kernels plus one 16-item projection fit exactly. A second + // projection would force the alias publication into terminal rejection. + const alias_limits = Limits{ .max_work_items = 524, .max_total_work_items = 524 }; + var alias_budget = graph_metric_policy.Budget{ .limits = alias_limits }; + var alias_prepared = try prepareGraphArtifactAlloc(alloc, &artifacts, source, .none, alias_limits); + defer alias_prepared.deinit(alloc); + const alias_a = try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + &artifacts, + "alias_a", + source, + &one_config, + .none, + alias_limits, + &alias_budget, + &alias_prepared, + .{ .published_generation = 2, .edge_generation = 1, .computed_at_ms = 2 }, + .{}, + ); + defer { + for (alias_a) |ref| freeArtifactRef(alloc, ref); + alloc.free(alias_a); + } + const alias_b = try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + &artifacts, + "alias_b", + source, + &one_config, + .none, + alias_limits, + &alias_budget, + &alias_prepared, + .{ .published_generation = 2, .edge_generation = 1, .computed_at_ms = 2 }, + .{}, + ); + defer { + for (alias_b) |ref| freeArtifactRef(alloc, ref); + alloc.free(alias_b); + } + try std.testing.expectEqual(@as(usize, 1), alias_a.len); + try std.testing.expectEqual(@as(usize, 1), alias_b.len); + try std.testing.expect(!std.mem.eql(u8, alias_a[0].name, alias_b[0].name)); + try std.testing.expectEqualStrings(alias_a[0].artifact_id, alias_b[0].artifact_id); + try std.testing.expectEqualStrings(alias_a[0].checksum, alias_b[0].checksum); + + // Whole-publication planning must survive A -> B -> A and all -> typed + // -> all ordering without another source read, projection, kernel or PUT. + graph.adjacencies[0].out_edges[0].weight = 2; + const other_payload = try graph_segment.encodeAlloc(alloc, graph); + defer alloc.free(other_payload); + var other_metadata = try artifacts.put(other_payload); + defer other_metadata.deinit(alloc); + const other_source = artifact_ref.ArtifactRef{ .kind = .graph_segment, .name = "other", .artifact_id = other_metadata.artifact_id, .byte_len = other_metadata.byte_len, .checksum = other_metadata.checksum }; + const CountingStore = struct { + inner: *artifact_store.ArtifactStore, + reads: usize = 0, + range_reads: usize = 0, + writes: usize = 0, + verifications: usize = 0, + header_reads: usize = 0, + fn get(ptr: *anyopaque, allocator: Allocator, id: []const u8) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.reads += 1; + return self.inner.getAllocWithCancellationUsingAllocator(allocator, id, .none); + } + fn put(ptr: *anyopaque, allocator: Allocator, bytes: []const u8) !artifact_store.ArtifactMetadata { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.writes += 1; + return self.inner.vtable.put(self.inner.ptr, allocator, bytes); + } + fn deinit(_: Allocator, _: *anyopaque) void {} + fn range(ptr: *anyopaque, allocator: Allocator, id: []const u8, offset: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.range_reads += 1; + return self.inner.getRangeAllocWithCancellationUsingAllocator(allocator, id, offset, len, .none); + } + fn stat(ptr: *anyopaque, allocator: Allocator, id: []const u8) !artifact_store.ArtifactMetadata { + const self: *@This() = @ptrCast(@alignCast(ptr)); + return self.inner.statWithCancellationUsingAllocator(allocator, id, .none); + } + fn verify(ptr: *anyopaque, allocator: Allocator, id: []const u8, len: u64, checksum: []const u8, cancellation: CancellationToken) !void { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.verifications += 1; + return self.inner.verifyContentWithCancellationUsingAllocator(allocator, id, len, checksum, cancellation); + } + fn verifiedRange(ptr: *anyopaque, allocator: Allocator, id: []const u8, byte_len: u64, checksum: []const u8, offset: u64, len: usize, cancellation: CancellationToken) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.header_reads += 1; + return self.inner.getVerifiedRangeAllocWithCancellationUsingAllocator(allocator, id, byte_len, checksum, offset, len, cancellation); + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.UnexpectedDelete; + } + }; + var counting = CountingStore{ .inner = &artifacts }; + var counted = artifact_store.ArtifactStore{ .allocator = alloc, .ptr = &counting, .vtable = &.{ .deinit = CountingStore.deinit, .put = CountingStore.put, .get_alloc = CountingStore.get, .get_range_alloc = CountingStore.range, .stat = CountingStore.stat, .delete = CountingStore.delete, .verify_content = CountingStore.verify, .get_verified_range_alloc_with_cancellation = CountingStore.verifiedRange } }; + const request_a = PublicationRequest{ .graph_index_name = "a", .source_graph = source, .config = one_config[0], .provenance = .{ .published_generation = 2, .edge_generation = 1, .computed_at_ms = 2 } }; + // Even numerically equal signed zeroes have distinct persisted config + // fingerprints. Deduplication must preserve the exact storage identity. + try std.testing.expect(!sameComputation(.{ .name = "zero", .tolerance = 0.0 }, .{ .name = "negative_zero", .tolerance = -0.0 })); + var request_b = request_a; + request_b.graph_index_name = "b"; + request_b.source_graph = other_source; + var request_typed = request_a; + request_typed.config.name = "typed_rank"; + request_typed.config.edge_filter = .{ .mode = .types, .types = &.{"cites"} }; + var request_alias = request_a; + request_alias.graph_index_name = "alias"; + request_alias.provenance = .{ .published_generation = 4, .edge_generation = 3, .computed_at_ms = 4 }; + var request_authority = request_a; + request_authority.config = .{ .name = "authority", .kind = .hits_authority }; + var request_hub = request_alias; + request_hub.config = .{ .name = "hub", .kind = .hits_hub }; + var baseline_budget = graph_metric_policy.Budget{ .limits = .{} }; + const baseline = try publishRequestsAlloc(alloc, &counted, &.{ request_a, request_typed, request_b, request_authority, request_hub }, .none, baseline_budget.limits, &baseline_budget, .{}); + defer { + for (baseline) |ref| freeArtifactRef(alloc, ref); + alloc.free(baseline); + } + try std.testing.expectEqual(@as(usize, 0), counting.reads); + const preparation_ranges = counting.header_reads + counting.range_reads; + // Two sources, each needing its footer, directory, and one authenticated + // data block. All filters and dictionary reads reuse that block. + try std.testing.expectEqual(@as(usize, 6), preparation_ranges); + try std.testing.expectEqual(@as(usize, 5), counting.writes); + counting.reads = 0; + counting.writes = 0; + counting.header_reads = 0; + counting.range_reads = 0; + var planned_budget = graph_metric_policy.Budget{ .limits = .{ + .max_total_work_items = baseline_budget.work_items, + .max_total_graph_payload_bytes = baseline_budget.graph_payload_bytes, + .max_total_metric_payload_bytes = baseline_budget.metric_payload_bytes, + } }; + const planned = try publishRequestsAlloc(alloc, &counted, &.{ request_a, request_b, request_typed, request_alias, request_hub, request_typed, request_authority, request_authority }, .none, planned_budget.limits, &planned_budget, .{}); + defer { + for (planned) |ref| freeArtifactRef(alloc, ref); + alloc.free(planned); + } + try std.testing.expectEqual(@as(usize, 0), counting.reads); + try std.testing.expectEqual(preparation_ranges, counting.header_reads + counting.range_reads); + try std.testing.expectEqual(@as(usize, 5), counting.writes); + try std.testing.expectEqual(baseline_budget.work_items, planned_budget.work_items); + try std.testing.expectEqualStrings(planned[0].artifact_id, planned[3].artifact_id); + try std.testing.expectEqualStrings(planned[2].artifact_id, planned[5].artifact_id); + try std.testing.expectEqualStrings(planned[6].artifact_id, planned[7].artifact_id); + try std.testing.expectEqualStrings("1:a11:shared_rank", planned[0].name); + try std.testing.expectEqualStrings("5:alias11:shared_rank", planned[3].name); + try std.testing.expectEqual(@as(u64, 3), planned[3].edge_generation); + for (planned) |ref| try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.ready, ref.graph_metric_materialization_state); + + // Adding/renaming aliases uses one authenticated prior computation without + // reading the graph, running a kernel, encoding, uploading, or using a seed. + counting = .{ .inner = &artifacts }; + var reuse_budget = graph_metric_policy.Budget{ .limits = baseline_budget.limits }; + var renamed = request_alias; + renamed.config.name = "renamed_rank"; + const reused = try publishRequestsWithPriorAlloc(alloc, &counted, &.{ request_alias, request_a, renamed }, baseline, .none, reuse_budget.limits, &reuse_budget, .{}); + defer { + for (reused) |ref| freeArtifactRef(alloc, ref); + alloc.free(reused); + } + try std.testing.expectEqual(@as(usize, 0), counting.reads); + try std.testing.expectEqual(@as(usize, 0), counting.writes); + try std.testing.expectEqual(@as(usize, 0), counting.verifications); + try std.testing.expectEqual(@as(usize, 1), counting.header_reads); + try std.testing.expect(reuse_budget.reuse_read_bytes > 0); + try std.testing.expectEqual(@as(u64, 0), reuse_budget.work_items); + try std.testing.expectEqual(@as(usize, 0), reuse_budget.graph_payload_bytes); + for (reused) |ref| try std.testing.expectEqualStrings(baseline[0].artifact_id, ref.artifact_id); + try std.testing.expectEqual(@as(u64, 4), reused[0].published_generation); + try std.testing.expectEqual(@as(u64, 3), reused[0].edge_generation); + try std.testing.expectEqual(baseline[0].computed_at_ms, reused[0].computed_at_ms); + + // A publication exhausted by the first computation rejects the second. + // Its unchanged plan stays terminal, but removing the expensive sibling + // must make the remaining metric eligible again, with fresh provenance. + var rejected_budget = graph_metric_policy.Budget{ .limits = .{ .max_work_items = 270, .max_total_work_items = 270 } }; + const rejected = try publishRequestsAlloc(alloc, &artifacts, &.{ request_a, request_b }, .none, rejected_budget.limits, &rejected_budget, .{}); + defer { + for (rejected) |ref| freeArtifactRef(alloc, ref); + alloc.free(rejected); + } + try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.rejected, rejected[1].graph_metric_materialization_state); + counting = .{ .inner = &artifacts }; + rejected_budget = .{ .limits = rejected_budget.limits }; + const stable = try publishRequestsWithPriorAlloc(alloc, &counted, &.{ request_a, request_b }, rejected, .none, rejected_budget.limits, &rejected_budget, .{}); + defer { + for (stable) |ref| freeArtifactRef(alloc, ref); + alloc.free(stable); + } + try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.rejected, stable[1].graph_metric_materialization_state); + try std.testing.expectEqual(@as(usize, 0), counting.reads); + try std.testing.expectEqual(@as(usize, 0), counting.writes); + var retried_request = request_b; + retried_request.provenance = .{ .published_generation = 5, .edge_generation = 1, .computed_at_ms = 5 }; + rejected_budget = .{ .limits = rejected_budget.limits }; + const retried = try publishRequestsWithPriorAlloc(alloc, &counted, &.{retried_request}, stable, .none, rejected_budget.limits, &rejected_budget, .{}); + defer { + for (retried) |ref| freeArtifactRef(alloc, ref); + alloc.free(retried); + } + try std.testing.expectEqual(artifact_ref.GraphMetricMaterializationState.ready, retried[0].graph_metric_materialization_state); + // The other source changed only weights: the directory now lets the + // previously ready sibling satisfy this retry without loading the graph. + try std.testing.expectEqual(@as(usize, 0), counting.reads); + try std.testing.expectEqual(@as(usize, 0), counting.writes); + try std.testing.expectEqual(@as(u64, 5), retried[0].published_generation); + try std.testing.expectEqual(stable[0].computed_at_ms, retried[0].computed_at_ms); + + const AllocationRunner = struct { + fn run(failing: Allocator, store: *artifact_store.ArtifactStore, prior: []const artifact_ref.ArtifactRef, requests: []const PublicationRequest) !void { + var budget = graph_metric_policy.Budget{ .limits = .{} }; + const result = try publishRequestsWithPriorAlloc(failing, store, requests, prior, .none, budget.limits, &budget, .{}); + defer { + for (result) |ref| freeArtifactRef(failing, ref); + failing.free(result); + } + } + }; + try std.testing.checkAllAllocationFailures(alloc, AllocationRunner.run, .{ &artifacts, @as([]const artifact_ref.ArtifactRef, baseline), @as([]const PublicationRequest, &.{ request_alias, request_a, renamed }) }); + + // Cache reuse must not inherit admission from the request that populated + // the cache. A later caller's stricter source-topology limit still wins. + const strict_limits = Limits{ .max_nodes = 1 }; + var strict_budget = graph_metric_policy.Budget{ .limits = strict_limits }; + const strict = try publishManyFromPreparedGraphWithBudgetAlloc( + alloc, + &artifacts, + "strict_alias", + source, + &one_config, + .none, + strict_limits, + &strict_budget, + &alias_prepared, + .{ .published_generation = 2, .edge_generation = 1, .computed_at_ms = 2 }, + .{}, + ); + defer { + for (strict) |ref| freeArtifactRef(alloc, ref); + alloc.free(strict); + } + const strict_payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator(alloc, strict[0].artifact_id, strict[0].byte_len, strict[0].checksum, .none); + defer alloc.free(strict_payload); + var strict_decoded = try metric_segment.decodeAlloc(alloc, strict_payload); + defer strict_decoded.deinit(alloc); + try std.testing.expectEqual(metric_segment.MaterializationState.rejected, strict_decoded.materialization_state); +} diff --git a/zig/pkg/antfly/src/serverless/build/lake_rebuild.zig b/zig/pkg/antfly/src/serverless/build/lake_rebuild.zig index 6f3906df66..c5fdf85986 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_rebuild.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_rebuild.zig @@ -19,6 +19,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const algebraic_segment = @import("../algebraic_segment/mod.zig"); const artifact_store = @import("../artifacts/store.zig"); const external_source = @import("../external_source/types.zig"); @@ -29,6 +30,10 @@ const source_binding = @import("../segment/source_binding.zig"); const rowsource = @import("../../storage/rowsource/types.zig"); const lake_sidecar_algebraic = @import("lake_sidecar_algebraic.zig"); const lake_sidecar_graph = @import("lake_sidecar_graph.zig"); +const graph_metric_config = @import("graph_metric_config.zig"); +const graph_metric_policy = @import("graph_metric_policy.zig"); +const graph_metric_segment = @import("../graph_metric_segment/mod.zig"); +const lake_graph_metric = @import("lake_graph_metric.zig"); const lake_sidecar_sparse = @import("lake_sidecar_sparse.zig"); const lake_sidecar_text = @import("lake_sidecar_text.zig"); const lake_sidecar_vector = @import("lake_sidecar_vector.zig"); @@ -227,14 +232,27 @@ pub const OperationPlan = struct { pub const RowSourceProvider = struct { ptr: *anyopaque, open_fn: *const fn (*anyopaque, Allocator, source_binding.Binding) anyerror!rowsource.Source, + open_with_cancellation_fn: ?*const fn (*anyopaque, Allocator, source_binding.Binding, CancellationToken) anyerror!rowsource.Source = null, pub fn open(self: RowSourceProvider, alloc: Allocator, binding: source_binding.Binding) !rowsource.Source { return try self.open_fn(self.ptr, alloc, binding); } + + pub fn openWithCancellation(self: RowSourceProvider, alloc: Allocator, binding: source_binding.Binding, cancellation: CancellationToken) !rowsource.Source { + try cancellation.check(); + var source = if (self.open_with_cancellation_fn) |open_with_cancellation| + try open_with_cancellation(self.ptr, alloc, binding, cancellation) + else + try self.open_fn(self.ptr, alloc, binding); + errdefer source.deinit(alloc); + try cancellation.check(); + return source; + } }; pub const ExecutionOptions = struct { limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const ExecutedOperation = struct { @@ -448,7 +466,7 @@ pub fn desiredArtifactsFromTableDefinitionAlloc( const graph_names = try listGraphIndexNamesAlloc(alloc, index_root); defer freeOwnedStrings(alloc, graph_names); for (graph_names) |graph_name| { - const config_json = try indexConfigJsonAlloc(alloc, index_root, graph_name); + const config_json = try graphIndexConfigJsonAlloc(alloc, index_root, graph_name); defer alloc.free(config_json); const graph_column = try configuredColumnOrDefaultAlloc(alloc, index_root, graph_name, "graph_edges"); defer alloc.free(graph_column); @@ -505,20 +523,241 @@ pub fn reconcileResolvedExternalSourceSidecarsAlloc( inventory: external_source.Inventory, table: TableIndexDefinition, published_declarations: []const sidecar_manifest.DeclaredArtifact, + provenance: lake_graph_metric.Provenance, +) !ReconciledManifest { + return try reconcileResolvedExternalSourceSidecarsWithCancellationAlloc( + alloc, + artifacts, + source_provider, + base_source, + inventory, + table, + published_declarations, + .none, + provenance, + ); +} + +pub fn reconcileResolvedExternalSourceSidecarsWithCancellationAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + source_provider: RowSourceProvider, + base_source: manifest_base_source.BaseSourceDescriptor, + inventory: external_source.Inventory, + table: TableIndexDefinition, + published_declarations: []const sidecar_manifest.DeclaredArtifact, + cancellation: CancellationToken, + provenance: lake_graph_metric.Provenance, +) !ReconciledManifest { + return try reconcileResolvedExternalSourceSidecarsWithRuntimeAlloc( + alloc, + artifacts, + source_provider, + base_source, + inventory, + table, + published_declarations, + cancellation, + provenance, + .{}, + ); +} + +/// Reconciles external sidecars while routing graph-metric kernels through the +/// caller-owned executor. The compatibility wrappers above remain serial, but +/// production builders can share their bounded std.Io runtime instead of +/// creating an unaccounted execution domain. +pub fn reconcileResolvedExternalSourceSidecarsWithRuntimeAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + source_provider: RowSourceProvider, + base_source: manifest_base_source.BaseSourceDescriptor, + inventory: external_source.Inventory, + table: TableIndexDefinition, + published_declarations: []const sidecar_manifest.DeclaredArtifact, + cancellation: CancellationToken, + provenance: lake_graph_metric.Provenance, + runtime: lake_graph_metric.ComputeRuntime, ) !ReconciledManifest { + try provenance.validate(); + try cancellation.check(); var desired = try desiredArtifactsFromResolvedExternalSourceAlloc(alloc, base_source, inventory, table); defer desired.deinit(alloc); - const published = try publishedArtifactsFromDeclarationsAlloc(alloc, published_declarations); + var base_declarations = std.ArrayListUnmanaged(sidecar_manifest.DeclaredArtifact).empty; + defer base_declarations.deinit(alloc); + for (published_declarations) |declaration| { + if (declaration.binding.sidecar_kind != .graph_metric) try base_declarations.append(alloc, declaration); + } + const published = try publishedArtifactsFromDeclarationsAlloc(alloc, base_declarations.items); defer alloc.free(published); var operation_plan = try planOperationsAlloc(alloc, desired.artifacts, published); defer operation_plan.deinit(alloc); - var executed = try executeOperationsAlloc(alloc, artifacts, source_provider, operation_plan); + var executed = try executeOperationsWithOptionsAlloc(alloc, artifacts, source_provider, operation_plan, .{ + .cancellation = cancellation, + }); defer executed.deinit(alloc); - return try reconcileExecutedOperationsAlloc(alloc, published, operation_plan, executed); + var reconciled = try reconcileExecutedOperationsAlloc(alloc, published, operation_plan, executed); + defer reconciled.deinit(alloc); + return try appendExternalGraphMetricDeclarationsAlloc(alloc, artifacts, table.indexes_json, reconciled.artifacts, published_declarations, cancellation, provenance, runtime); +} + +fn appendExternalGraphMetricDeclarationsAlloc( + alloc: Allocator, + artifacts: *artifact_store.ArtifactStore, + indexes_json: []const u8, + base_declarations: []const sidecar_manifest.DeclaredArtifact, + published_declarations: []const sidecar_manifest.DeclaredArtifact, + cancellation: CancellationToken, + provenance: lake_graph_metric.Provenance, + runtime: lake_graph_metric.ComputeRuntime, +) !ReconciledManifest { + try cancellation.check(); + const specs = try graph_metric_config.parseIndexSpecsAlloc(alloc, indexes_json); + defer graph_metric_config.freeIndexSpecs(alloc, specs); + + var declarations = std.ArrayListUnmanaged(sidecar_manifest.DeclaredArtifact).empty; + errdefer { + for (declarations.items) |declaration| freeOwnedDeclaration(alloc, declaration); + declarations.deinit(alloc); + } + try declarations.ensureUnusedCapacity(alloc, base_declarations.len); + for (base_declarations) |declaration| declarations.appendAssumeCapacity(try cloneDeclarationAlloc(alloc, declaration)); + if (specs.len > 0) { + try stampExternalGraphTopologyGenerations( + declarations.items, + published_declarations, + specs, + provenance.edge_generation, + cancellation, + ); + } + + const graph_metric_limits = lake_graph_metric.Limits{}; + var graph_metric_budget = graph_metric_policy.Budget{ .limits = graph_metric_limits }; + var requests = std.ArrayListUnmanaged(lake_graph_metric.PublicationRequest).empty; + defer requests.deinit(alloc); + var previous_metrics = std.ArrayListUnmanaged(manifest_artifact.ArtifactRef).empty; + defer previous_metrics.deinit(alloc); + for (published_declarations) |declaration| { + if (declaration.binding.sidecar_kind == .graph_metric and declaration.artifact.kind == .graph_metric_segment) + try previous_metrics.append(alloc, declaration.artifact); + } + var source_declarations = std.ArrayListUnmanaged(sidecar_manifest.DeclaredArtifact).empty; + defer source_declarations.deinit(alloc); + for (specs) |spec| { + try cancellation.check(); + const graph_declaration = findDeclaration(declarations.items, spec.index_name) orelse continue; + if (graph_declaration.binding.sidecar_kind != .graph or graph_declaration.artifact.kind != .graph_segment) return error.SidecarArtifactKindMismatch; + var effective_provenance = provenance; + effective_provenance.edge_generation = graph_declaration.artifact.edge_generation; + + for (spec.configs) |config| { + const metric_name = try graph_metric_segment.artifactNameAlloc(alloc, spec.index_name, config.name); + defer alloc.free(metric_name); + var request = lake_graph_metric.PublicationRequest{ + .graph_index_name = spec.index_name, + .source_graph = graph_declaration.artifact, + .config = config, + .provenance = effective_provenance, + }; + if (findDeclaration(published_declarations, metric_name)) |existing| { + if (existing.binding.sidecar_kind == .graph_metric and existing.artifact.kind == .graph_metric_segment) { + request.prior_artifact = existing.artifact; + } + } + try requests.append(alloc, request); + try source_declarations.append(alloc, graph_declaration); + } + } + const built = try lake_graph_metric.publishRequestsWithPriorAlloc(alloc, artifacts, requests.items, previous_metrics.items, cancellation, graph_metric_limits, &graph_metric_budget, runtime); + defer { + for (built) |ref| lake_graph_metric.freeArtifactRef(alloc, ref); + alloc.free(built); + } + try declarations.ensureUnusedCapacity(alloc, built.len); + for (requests.items, built, source_declarations.items) |request, artifact, graph_declaration| { + declarations.appendAssumeCapacity(try graphMetricDeclarationAlloc(alloc, graph_declaration, request.config, artifact)); + } + + const owned = try declarations.toOwnedSlice(alloc); + errdefer { + for (owned) |declaration| freeOwnedDeclaration(alloc, declaration); + alloc.free(owned); + } + const result = ReconciledManifest{ .artifacts = owned }; + try result.manifest().validate(); + return result; +} + +fn findDeclaration( + declarations: []const sidecar_manifest.DeclaredArtifact, + name: []const u8, +) ?sidecar_manifest.DeclaredArtifact { + for (declarations) |declaration| if (std.mem.eql(u8, declaration.name, name)) return declaration; + return null; +} + +fn stampExternalGraphTopologyGenerations( + declarations: []sidecar_manifest.DeclaredArtifact, + published_declarations: []const sidecar_manifest.DeclaredArtifact, + specs: []const graph_metric_config.IndexSpec, + next_generation: u64, + cancellation: CancellationToken, +) !void { + for (declarations) |*declaration| { + try cancellation.check(); + if (declaration.binding.sidecar_kind != .graph or declaration.artifact.kind != .graph_segment) continue; + var configured = false; + for (specs) |spec| { + if (spec.configs.len > 0 and std.mem.eql(u8, spec.index_name, declaration.name)) { + configured = true; + break; + } + } + if (!configured) continue; + declaration.artifact.edge_generation = next_generation; + const previous_graph = findDeclaration(published_declarations, declaration.name) orelse continue; + if (previous_graph.binding.sidecar_kind != .graph or + previous_graph.artifact.kind != .graph_segment or + !artifactRefsIdentifySamePayload(previous_graph.artifact, declaration.artifact)) continue; + + if (previous_graph.artifact.edge_generation != 0) { + declaration.artifact.edge_generation = previous_graph.artifact.edge_generation; + continue; + } + + // No pre-release provenance recovery: a graph first acquiring metrics + // starts at this publication, while stamped current graphs retain identity. + } +} + +fn graphMetricDeclarationAlloc( + alloc: Allocator, + graph_declaration: sidecar_manifest.DeclaredArtifact, + config: @import("../../graph/graph.zig").GraphMetricConfig, + artifact: manifest_artifact.ArtifactRef, +) !sidecar_manifest.DeclaredArtifact { + const name = try alloc.dupe(u8, artifact.name); + errdefer alloc.free(name); + var binding = try cloneBindingAlloc(alloc, graph_declaration.binding); + errdefer freeOwnedBinding(alloc, binding); + binding.sidecar_kind = .graph_metric; + const metric_config_hash = try graphMetricBindingHashAlloc(alloc, config, graph_declaration.artifact.artifact_id); + alloc.free(binding.index_config_hash); + binding.index_config_hash = metric_config_hash; + const owned_artifact = try cloneArtifactRefAlloc(alloc, artifact); + errdefer { + if (owned_artifact.name.len != 0) alloc.free(owned_artifact.name); + alloc.free(owned_artifact.artifact_id); + alloc.free(owned_artifact.checksum); + } + const declaration = sidecar_manifest.DeclaredArtifact{ .name = name, .binding = binding, .artifact = owned_artifact }; + try declaration.validate(); + return declaration; } pub fn planOperationsAlloc( @@ -587,6 +826,7 @@ pub fn executeOperationsWithOptionsAlloc( options: ExecutionOptions, ) !ExecutionResult { try options.limits.validate(); + try options.cancellation.check(); const executed = try alloc.alloc(ExecutedOperation, plan.operations.len); errdefer alloc.free(executed); const completed = try alloc.alloc(bool, plan.operations.len); @@ -597,6 +837,7 @@ pub fn executeOperationsWithOptionsAlloc( } for (plan.operations, 0..) |operation, operation_idx| { + try options.cancellation.check(); if (completed[operation_idx]) continue; switch (operation.action) { .reuse => { @@ -611,9 +852,9 @@ pub fn executeOperationsWithOptionsAlloc( .rebuild => { const group_count = countPendingRebuildsForSnapshot(plan.operations, completed, operation.binding); if (group_count == 1) { - var source = try source_provider.open(alloc, operation.binding); + var source = try source_provider.openWithCancellation(alloc, operation.binding, options.cancellation); defer source.deinit(alloc); - const declaration = try executeRebuildOperationAlloc(alloc, artifacts, source, operation, options.limits); + const declaration = try executeRebuildOperationAlloc(alloc, artifacts, source, operation, options.limits, options.cancellation); errdefer freeOwnedDeclaration(alloc, declaration); executed[operation_idx] = try makeExecutedOperation(alloc, operation, declaration, declaration.artifact.artifact_id); completed[operation_idx] = true; @@ -622,12 +863,13 @@ pub fn executeOperationsWithOptionsAlloc( const merged_binding = try mergedRebuildBindingAlloc(alloc, plan.operations, completed, operation.binding); defer source_binding.freeOwned(alloc, merged_binding); - var source = try source_provider.open(alloc, merged_binding); + var source = try source_provider.openWithCancellation(alloc, merged_binding, options.cancellation); defer source.deinit(alloc); - var replay = try lake_replay.Buffer.captureAlloc(alloc, source, options.limits); + var replay = try lake_replay.Buffer.captureWithCancellationAlloc(alloc, source, options.limits, options.cancellation); defer replay.deinit(alloc); for (plan.operations, 0..) |group_operation, group_idx| { + try options.cancellation.check(); if (completed[group_idx] or group_operation.action != .rebuild or !source_binding.sameSourceSnapshot(group_operation.binding, operation.binding)) continue; var cursor = replay.cursor(); @@ -637,6 +879,7 @@ pub fn executeOperationsWithOptionsAlloc( cursor.rowSource(), group_operation, options.limits, + options.cancellation, ); errdefer freeOwnedDeclaration(alloc, declaration); executed[group_idx] = try makeExecutedOperation( @@ -1110,6 +1353,45 @@ fn indexConfigJsonAlloc( return try std.fmt.allocPrint(alloc, "{f}", .{std.json.fmt(value, .{})}); } +fn graphIndexConfigJsonAlloc( + alloc: Allocator, + index_root: std.json.ObjectMap, + index_name: []const u8, +) ![]u8 { + const value = index_root.get(index_name) orelse return try alloc.dupe(u8, "{}"); + if (value != .object) return error.InvalidTableIndexMetadata; + var topology = std.json.ObjectMap.empty; + defer topology.deinit(alloc); + var it = value.object.iterator(); + while (it.next()) |entry| { + if (std.mem.eql(u8, entry.key_ptr.*, "metrics")) continue; + try topology.put(alloc, entry.key_ptr.*, entry.value_ptr.*); + } + return try std.fmt.allocPrint(alloc, "{f}", .{std.json.fmt(std.json.Value{ .object = topology }, .{})}); +} + +fn graphMetricBindingHashAlloc( + alloc: Allocator, + config: @import("../../graph/graph.zig").GraphMetricConfig, + graph_artifact_id: []const u8, +) ![]u8 { + return try std.fmt.allocPrint( + alloc, + "graph-metric-v2:{x}:{x}:{s}", + .{ + lake_graph_metric.configFingerprint(config), + lake_graph_metric.materializerFingerprint(.{}), + graph_artifact_id, + }, + ); +} + +fn artifactRefsIdentifySamePayload(lhs: manifest_artifact.ArtifactRef, rhs: manifest_artifact.ArtifactRef) bool { + return lhs.byte_len == rhs.byte_len and + std.mem.eql(u8, lhs.artifact_id, rhs.artifact_id) and + std.mem.eql(u8, lhs.checksum, rhs.checksum); +} + fn indexConfigHashAlloc( alloc: Allocator, kind: []const u8, @@ -1373,6 +1655,7 @@ fn builderKindForDesired(want: DesiredArtifact) !BuilderKind { .sparse => .sparse, .graph => .graph, .algebraic => error.AmbiguousLakeRebuildBuilder, + .graph_metric => error.MissingLakeRebuildBuildSpec, }; } @@ -1384,6 +1667,7 @@ fn buildSpecForDesiredAlloc(alloc: Allocator, want: DesiredArtifact) !BuildSpec .sparse => .{ .sparse = .{ .sparse_column = try alloc.dupe(u8, try defaultBoundColumn(want.binding, 0)) } }, .graph => .{ .graph = .{ .graph_column = try alloc.dupe(u8, try defaultBoundColumn(want.binding, 0)) } }, .algebraic => error.MissingLakeRebuildBuildSpec, + .graph_metric => error.MissingLakeRebuildBuildSpec, }; } @@ -1475,7 +1759,9 @@ fn executeRebuildOperationAlloc( source: rowsource.Source, operation: Operation, limits: lake_build_limits.Limits, + cancellation: CancellationToken, ) !sidecar_manifest.DeclaredArtifact { + try cancellation.check(); const build_spec = operation.build_spec orelse return error.MissingLakeRebuildBuildSpec; return switch (build_spec) { .text => |spec| blk: { @@ -1484,6 +1770,7 @@ fn executeRebuildOperationAlloc( .text_column = spec.text_column, .config_json = spec.config_json, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1495,6 +1782,7 @@ fn executeRebuildOperationAlloc( .vector_column = spec.vector_column, .embedding_name = spec.embedding_name, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1505,6 +1793,7 @@ fn executeRebuildOperationAlloc( .name = operation.name, .sparse_column = spec.sparse_column, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1515,6 +1804,7 @@ fn executeRebuildOperationAlloc( .name = operation.name, .graph_column = spec.graph_column, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1527,6 +1817,7 @@ fn executeRebuildOperationAlloc( .value_column = spec.value_column, .op = spec.op, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1537,6 +1828,7 @@ fn executeRebuildOperationAlloc( .name = operation.name, .expressions = spec.expressions, .limits = limits, + .cancellation = cancellation, }); const declaration = result.declaration; result = undefined; @@ -1685,6 +1977,22 @@ fn cloneArtifactRefAlloc(alloc: Allocator, artifact: manifest_artifact.ArtifactR .artifact_id = artifact_id, .byte_len = artifact.byte_len, .checksum = checksum, + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }; } @@ -1783,7 +2091,6 @@ const TestRowSourceState = struct { const MemoryArtifactStore = struct { alloc: Allocator, entries: std.StringArrayHashMapUnmanaged([]u8) = .empty, - next_id: usize = 0, fn init(alloc: Allocator) MemoryArtifactStore { return .{ .alloc = alloc }; @@ -1805,18 +2112,22 @@ const MemoryArtifactStore = struct { } fn put(self: *MemoryArtifactStore, alloc: Allocator, contents: []const u8) !artifact_store.ArtifactMetadata { - const artifact_id = try std.fmt.allocPrint(alloc, "mem:{d}", .{self.next_id}); + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(contents, &digest, .{}); + const hex = std.fmt.bytesToHex(digest, .lower); + const artifact_id = try std.fmt.allocPrint(alloc, "{s}{s}", .{ artifact_store.sha256_artifact_id_prefix, hex }); errdefer alloc.free(artifact_id); - self.next_id += 1; - const key = try self.alloc.dupe(u8, artifact_id); - errdefer self.alloc.free(key); - const bytes = try self.alloc.dupe(u8, contents); - errdefer self.alloc.free(bytes); - try self.entries.put(self.alloc, key, bytes); + if (!self.entries.contains(artifact_id)) { + const key = try self.alloc.dupe(u8, artifact_id); + errdefer self.alloc.free(key); + const bytes = try self.alloc.dupe(u8, contents); + errdefer self.alloc.free(bytes); + try self.entries.put(self.alloc, key, bytes); + } return .{ .artifact_id = artifact_id, .byte_len = @intCast(contents.len), - .checksum = try std.fmt.allocPrint(alloc, "len:{d}", .{contents.len}), + .checksum = try alloc.dupe(u8, &hex), }; } @@ -1835,10 +2146,13 @@ const MemoryArtifactStore = struct { fn stat(self: *MemoryArtifactStore, alloc: Allocator, artifact_id: []const u8) !artifact_store.ArtifactMetadata { const bytes = self.entries.get(artifact_id) orelse return error.ArtifactNotFound; + const owned_id = try alloc.dupe(u8, artifact_id); + errdefer alloc.free(owned_id); + const checksum = try alloc.dupe(u8, try artifact_store.sha256ChecksumFromArtifactId(artifact_id)); return .{ - .artifact_id = try alloc.dupe(u8, artifact_id), + .artifact_id = owned_id, .byte_len = @intCast(bytes.len), - .checksum = try std.fmt.allocPrint(alloc, "len:{d}", .{bytes.len}), + .checksum = checksum, }; } @@ -2010,6 +2324,30 @@ test "lake rebuild desired artifacts derive from table index metadata" { try std.testing.expectEqual(BuilderKind.graph, operations.find("graph_idx").?.builder_kind.?); } +test "serverless lake graph topology binding ignores metric-only config changes" { + const alloc = std.testing.allocator; + const source: LakeSourceSnapshot = .{ + .source_kind = .external_parquet, + .source_id = "events", + .snapshot_id = "parquet-21", + .schema_fingerprint = "schema-v4", + }; + var before = try desiredArtifactsFromTableDefinitionAlloc(alloc, source, .{ + .table_name = "events", + .indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"field\":\"edges\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":20}}}}", + }); + defer before.deinit(alloc); + var after = try desiredArtifactsFromTableDefinitionAlloc(alloc, source, .{ + .table_name = "events", + .indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"field\":\"edges\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":40}}}}", + }); + defer after.deinit(alloc); + try std.testing.expectEqualStrings( + before.find("graph_idx").?.binding.index_config_hash, + after.find("graph_idx").?.binding.index_config_hash, + ); +} + test "lake rebuild desired artifacts bind resolved external inventory identity" { const alloc = std.testing.allocator; var inventory = external_source.Inventory{ @@ -2388,7 +2726,7 @@ test "lake rebuild operation executor publishes row-source sidecars" { const executed = result.find("docs.body_text").?; try std.testing.expectEqual(Action.rebuild, executed.action); try std.testing.expect(executed.declaration != null); - try std.testing.expectEqualStrings("mem:0", executed.artifact_id); + try std.testing.expect(std.mem.startsWith(u8, executed.artifact_id, artifact_store.sha256_artifact_id_prefix)); const stored = try artifacts.getAlloc(executed.artifact_id); defer alloc.free(stored); try std.testing.expect(stored.len > 0); @@ -2465,7 +2803,7 @@ test "lake rebuild operation executor opens each source snapshot once" { try std.testing.expect(result.find("docs.title_text").?.declaration != null); } -test "lake rebuild reconciles resolved external sidecars end to end" { +test "serverless lake rebuild reconciles resolved external sidecars end to end" { const alloc = std.testing.allocator; var memory = MemoryArtifactStore.init(alloc); var artifacts = memory.artifactStore(); @@ -2493,8 +2831,14 @@ test "lake rebuild reconciles resolved external sidecars end to end" { .{ .external = .{ .source_id = "docs", .snapshot_id = "parquet-31", .file_id = "file-a.parquet", .row_group_ordinal = 0, .row_ordinal = 1 } }, }; const bodies = [_][]const u8{ "lake rebuild workflow", "sidecar reconcile" }; + const target_key = try source_binding.rowRefKeyAlloc(alloc, row_refs[1]); + defer alloc.free(target_key); + const first_graph = try std.fmt.allocPrint(alloc, "[{{\"target\":{f},\"edge_type\":\"cites\"}}]", .{std.json.fmt(target_key, .{})}); + defer alloc.free(first_graph); + const graph_values = [_][]const u8{ first_graph, "[]" }; const columns = [_]rowsource.ColumnVector{ .{ .name = "body", .values = .{ .bytes = &bodies } }, + .{ .name = "graph_edges", .values = .{ .json = &graph_values } }, }; const batches = [_]rowsource.ColumnBatch{.{ .snapshot = .{ .table_id = "docs", .snapshot_id = "parquet-31" }, @@ -2511,23 +2855,134 @@ test "lake rebuild reconciles resolved external sidecars end to end" { inventory, .{ .table_name = "docs", - .indexes_json = "{\"body_text\":{\"type\":\"full_text\",\"field\":\"body\"}}", + .indexes_json = "{\"body_text\":{\"type\":\"full_text\",\"field\":\"body\"},\"graph_idx\":{\"type\":\"graph\",\"field\":\"graph_edges\",\"metrics\":{\"degree\":{\"kind\":\"degree\"},\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":20}}}}", }, &.{}, + .{ .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }, ); defer reconciled.deinit(alloc); - try std.testing.expectEqual(@as(usize, 1), reconciled.artifacts.len); + try std.testing.expectEqual(@as(usize, 4), reconciled.artifacts.len); const declaration = reconciled.find("body_text").?; try std.testing.expectEqual(source_binding.SidecarKind.text, declaration.binding.sidecar_kind); try std.testing.expectEqual(rowsource.SourceKind.external_parquet, declaration.binding.source_kind); try std.testing.expectEqualStrings("docs", declaration.binding.source_id); try std.testing.expectEqualStrings("parquet-31", declaration.binding.snapshot_id); try std.testing.expectEqualStrings("schema-v3", declaration.binding.schema_fingerprint); - try std.testing.expectEqualStrings("mem:0", declaration.artifact.artifact_id); + try std.testing.expect(std.mem.startsWith(u8, declaration.artifact.artifact_id, artifact_store.sha256_artifact_id_prefix)); const stored = try artifacts.getAlloc(declaration.artifact.artifact_id); defer alloc.free(stored); try std.testing.expect(stored.len > 0); + + const graph_declaration = reconciled.find("graph_idx").?; + try std.testing.expectEqual(@as(u64, 1), graph_declaration.artifact.edge_generation); + const metric_artifact_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_idx", "rank"); + defer alloc.free(metric_artifact_name); + const metric_declaration = reconciled.find(metric_artifact_name).?; + try std.testing.expectEqual(source_binding.SidecarKind.graph_metric, metric_declaration.binding.sidecar_kind); + const metric_payload = try artifacts.getVerifiedAllocWithCancellationUsingAllocator( + alloc, + metric_declaration.artifact.artifact_id, + metric_declaration.artifact.byte_len, + metric_declaration.artifact.checksum, + .none, + ); + defer alloc.free(metric_payload); + var metric = try graph_metric_segment.decodeAlloc(alloc, metric_payload); + defer metric.deinit(alloc); + try std.testing.expectEqualStrings(graph_declaration.artifact.artifact_id, metric.source_graph_artifact_id); + try std.testing.expect(metric.score(target_key) != null); + + const degree_artifact_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_idx", "degree"); + defer alloc.free(degree_artifact_name); + const degree_declaration = reconciled.find(degree_artifact_name).?; + var updated = try reconcileResolvedExternalSourceSidecarsAlloc( + alloc, + &artifacts, + source_provider.provider(), + base_source, + inventory, + .{ + .table_name = "docs", + .indexes_json = "{\"body_text\":{\"type\":\"full_text\",\"field\":\"body\"},\"graph_idx\":{\"type\":\"graph\",\"field\":\"graph_edges\",\"metrics\":{\"degree\":{\"kind\":\"degree\"},\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":40},\"centrality\":{\"kind\":\"eigenvector\"},\"degree_alias\":{\"kind\":\"degree\"}}},\"graph_alias\":{\"type\":\"graph\",\"field\":\"graph_edges\",\"metrics\":{\"degree\":{\"kind\":\"degree\"}}}}", + }, + reconciled.artifacts, + .{ .published_generation = 2, .edge_generation = 2, .computed_at_ms = 2 }, + ); + defer updated.deinit(alloc); + + try std.testing.expectEqual(@as(usize, 8), updated.artifacts.len); + const updated_graph = updated.find("graph_idx").?; + const updated_metric = updated.find(metric_artifact_name).?; + const updated_degree = updated.find(degree_artifact_name).?; + const centrality_artifact_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_idx", "centrality"); + defer alloc.free(centrality_artifact_name); + const updated_centrality = updated.find(centrality_artifact_name).?; + try std.testing.expectEqualStrings(graph_declaration.artifact.artifact_id, updated_graph.artifact.artifact_id); + try std.testing.expectEqual(graph_declaration.artifact.edge_generation, updated_graph.artifact.edge_generation); + try std.testing.expect(!std.mem.eql(u8, metric_declaration.artifact.artifact_id, updated_metric.artifact.artifact_id)); + try std.testing.expectEqual(@as(u64, 2), updated_metric.artifact.published_generation); + try std.testing.expectEqual(metric_declaration.artifact.edge_generation, updated_metric.artifact.edge_generation); + try std.testing.expectEqualStrings(degree_declaration.artifact.artifact_id, updated_degree.artifact.artifact_id); + try std.testing.expectEqual(degree_declaration.artifact.published_generation, updated_degree.artifact.published_generation); + try std.testing.expectEqual(degree_declaration.artifact.edge_generation, updated_degree.artifact.edge_generation); + try std.testing.expectEqual(degree_declaration.artifact.computed_at_ms, updated_degree.artifact.computed_at_ms); + try std.testing.expectEqual(@as(u64, 2), updated_centrality.artifact.published_generation); + try std.testing.expectEqual(graph_declaration.artifact.edge_generation, updated_centrality.artifact.edge_generation); + + const alias_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_idx", "degree_alias"); + defer alloc.free(alias_name); + const alias = updated.find(alias_name).?; + try std.testing.expectEqualStrings(degree_declaration.artifact.artifact_id, alias.artifact.artifact_id); + try std.testing.expectEqual(degree_declaration.artifact.computed_at_ms, alias.artifact.computed_at_ms); + try std.testing.expectEqual(@as(u64, 2), alias.artifact.published_generation); + try std.testing.expectEqual(updated_graph.artifact.edge_generation, alias.artifact.edge_generation); + try std.testing.expect(source_binding.sameSourceSnapshot(updated_graph.binding, alias.binding)); + try std.testing.expectEqualStrings(degree_declaration.binding.index_config_hash, alias.binding.index_config_hash); + const index_alias_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_alias", "degree"); + defer alloc.free(index_alias_name); + const index_alias = updated.find(index_alias_name).?; + try std.testing.expectEqualStrings(updated_graph.artifact.artifact_id, updated.find("graph_alias").?.artifact.artifact_id); + try std.testing.expectEqualStrings(degree_declaration.artifact.artifact_id, index_alias.artifact.artifact_id); + try std.testing.expectEqual(degree_declaration.artifact.computed_at_ms, index_alias.artifact.computed_at_ms); + try std.testing.expectEqual(@as(u64, 2), index_alias.artifact.published_generation); + try std.testing.expectEqual(updated.find("graph_alias").?.artifact.edge_generation, index_alias.artifact.edge_generation); + + // Missing pre-release provenance is not recovered from sibling metrics. + // Current publication stamps a new source generation instead. + for (updated.artifacts) |*published| { + if (published.binding.sidecar_kind == .graph and std.mem.eql(u8, published.name, "graph_idx")) { + published.artifact.edge_generation = 0; + } else if (published.binding.sidecar_kind == .graph_metric) { + published.artifact.edge_generation = 2; + published.artifact.graph_metric_source_checksum = @splat(0); + if (std.mem.eql(u8, published.name, centrality_artifact_name)) { + published.artifact.edge_generation = 1; + published.artifact.graph_metric_source_checksum = @splat(0xbb); + } + } + } + var replaced = try reconcileResolvedExternalSourceSidecarsAlloc( + alloc, + &artifacts, + source_provider.provider(), + base_source, + inventory, + .{ + .table_name = "docs", + .indexes_json = "{\"body_text\":{\"type\":\"full_text\",\"field\":\"body\"},\"graph_idx\":{\"type\":\"graph\",\"field\":\"graph_edges\",\"metrics\":{\"replacement\":{\"kind\":\"degree\"}}}}", + }, + updated.artifacts, + .{ .published_generation = 3, .edge_generation = 3, .computed_at_ms = 3 }, + ); + defer replaced.deinit(alloc); + + const replacement_name = try graph_metric_segment.artifactNameAlloc(alloc, "graph_idx", "replacement"); + defer alloc.free(replacement_name); + const replaced_graph = replaced.find("graph_idx").?; + const replacement_metric = replaced.find(replacement_name).?; + try std.testing.expectEqual(@as(u64, 3), replaced_graph.artifact.edge_generation); + try std.testing.expectEqual(replaced_graph.artifact.edge_generation, replacement_metric.artifact.edge_generation); } test "lake rebuild operation planner preserves reuse and drop artifacts" { diff --git a/zig/pkg/antfly/src/serverless/build/lake_replay.zig b/zig/pkg/antfly/src/serverless/build/lake_replay.zig index 2a1ff3e098..3ebcdbace9 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_replay.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_replay.zig @@ -18,6 +18,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const rowsource = @import("../../storage/rowsource/types.zig"); const source_binding = @import("../segment/source_binding.zig"); const lake_build_limits = @import("lake_build_limits.zig"); @@ -27,14 +28,24 @@ pub const Buffer = struct { batches: []OwnedBatch, pub fn captureAlloc(alloc: Allocator, source: rowsource.Source, limits: lake_build_limits.Limits) !Buffer { + return captureWithCancellationAlloc(alloc, source, limits, .none); + } + + pub fn captureWithCancellationAlloc( + alloc: Allocator, + source: rowsource.Source, + limits: lake_build_limits.Limits, + cancellation: CancellationToken, + ) !Buffer { + try cancellation.check(); var working_set = try lake_build_limits.WorkingSetAllocator.init(alloc, limits); - return captureBoundedAlloc(working_set.allocator(), source, limits) catch |err| { + return captureBoundedAlloc(working_set.allocator(), source, limits, cancellation) catch |err| { if (err == error.OutOfMemory and working_set.limit_exceeded) return error.LakeSidecarReplayBudgetExceeded; return err; }; } - fn captureBoundedAlloc(alloc: Allocator, source: rowsource.Source, limits: lake_build_limits.Limits) !Buffer { + fn captureBoundedAlloc(alloc: Allocator, source: rowsource.Source, limits: lake_build_limits.Limits, cancellation: CancellationToken) !Buffer { var budget = try lake_build_limits.Budget.init(limits); var batches = std.ArrayListUnmanaged(OwnedBatch).empty; errdefer { @@ -42,7 +53,9 @@ pub const Buffer = struct { batches.deinit(alloc); } var replay_bytes: usize = 0; - while (try source.next(alloc)) |batch| { + while (true) { + try cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); replay_bytes = std.math.add(usize, replay_bytes, lake_build_limits.estimateBatchBytes(batch)) catch return error.LakeSidecarReplayBudgetExceeded; diff --git a/zig/pkg/antfly/src/serverless/build/lake_sidecar_algebraic.zig b/zig/pkg/antfly/src/serverless/build/lake_sidecar_algebraic.zig index 2469afa5c8..754de88a3a 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_sidecar_algebraic.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_sidecar_algebraic.zig @@ -16,6 +16,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const algebraic_segment = @import("../algebraic_segment/mod.zig"); const aggregate_math = algebraic_segment.aggregate_math; const artifact_ref = @import("../manifest/artifact_ref.zig"); @@ -33,6 +34,7 @@ pub const AlgebraicGroupBySidecarBuildOptions = struct { op: algebraic_segment.AggregateOp, artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const AlgebraicGroupBySidecarBuildResult = struct { @@ -60,6 +62,7 @@ pub const AlgebraicExpressionSidecarBuildOptions = struct { expressions: []const algebraic_segment.ExpressionSpec, artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const AlgebraicExpressionSidecarBuildResult = struct { @@ -111,7 +114,9 @@ fn buildAlgebraicGroupBySidecarBoundedAlloc( folds.deinit(alloc); } - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try sidecar_manifest.validateBatchAgainstDeclaredArtifact(.{ .name = options.name, @@ -160,7 +165,7 @@ pub fn publishAlgebraicGroupBySidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); @@ -204,7 +209,9 @@ fn buildAlgebraicExpressionSidecarBoundedAlloc( accumulator.* = initExpressionAccumulator(spec.op); } - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try source_binding.validateBatchAgainstBinding(binding, batch); try appendBatchExpressions(accumulators, batch, options); @@ -247,7 +254,7 @@ pub fn publishAlgebraicExpressionSidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); diff --git a/zig/pkg/antfly/src/serverless/build/lake_sidecar_graph.zig b/zig/pkg/antfly/src/serverless/build/lake_sidecar_graph.zig index 10fa8bebe3..222582b5bb 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_sidecar_graph.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_sidecar_graph.zig @@ -16,6 +16,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const artifact_ref = @import("../manifest/artifact_ref.zig"); const artifact_store = @import("../artifacts/store.zig"); const graph_segment = @import("../graph_segment/mod.zig"); @@ -30,6 +31,7 @@ pub const GraphSidecarBuildOptions = struct { graph_column: []const u8, artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const GraphSidecarBuildResult = struct { @@ -74,13 +76,13 @@ fn buildGraphSidecarBoundedAlloc( try validateOptions(binding, source.kind, options); var budget = try lake_build_limits.Budget.init(options.limits); - var node_map = std.StringArrayHashMapUnmanaged(NodeEdges).empty; - defer deinitNodeMap(alloc, &node_map); - var neighbor_tables = std.StringArrayHashMapUnmanaged(void).empty; - defer deinitNeighborTableMap(alloc, &neighbor_tables); + var builder = graph_segment.Builder{ .alloc = alloc }; + defer builder.deinit(); var total_edges: usize = 0; - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try sidecar_manifest.validateBatchAgainstDeclaredArtifact(.{ .name = options.name, @@ -95,26 +97,27 @@ fn buildGraphSidecarBoundedAlloc( }, batch); const column = batch.findColumn(options.graph_column).?; - total_edges = std.math.add(usize, total_edges, try appendBatchGraph(alloc, &node_map, &neighbor_tables, batch, column)) catch + total_edges = std.math.add(usize, total_edges, try appendBatchGraph(alloc, &builder, batch, column)) catch return error.LakeSidecarBuildBudgetExceeded; - const retained_items = std.math.add(usize, node_map.count(), total_edges) catch + const retained_items = std.math.add(usize, builder.nodeCount(), total_edges) catch return error.LakeSidecarBuildBudgetExceeded; try budget.checkRetainedItems(retained_items); } if (total_edges == 0) return error.EmptyLakeSidecarGraphSegment; - var segment = try nodeMapToSegmentAlloc(alloc, &node_map, &neighbor_tables); - defer graph_segment.freeSegment(alloc, &segment); - - const encoded_size = try graph_segment.encodedSize(segment); - try budget.checkOutputBytes(encoded_size); - const payload = try graph_segment.encodeAlloc(alloc, segment); + const payload = builder.encodeAlloc( + options.limits.max_output_bytes, + options.cancellation, + ) catch |err| switch (err) { + error.GraphSegmentTooLarge => return error.LakeSidecarBuildBudgetExceeded, + else => return err, + }; errdefer alloc.free(payload); - std.debug.assert(payload.len == encoded_size); var declaration = try declaredArtifactAlloc(alloc, binding, options, payload.len); errdefer freeOwnedDeclaration(alloc, declaration); + try graph_segment.codec.compact.bindTopologyControl(&declaration.artifact, payload); try declaration.validate(); return .{ @@ -134,7 +137,7 @@ pub fn publishGraphSidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); @@ -168,8 +171,7 @@ fn validateOptions( fn appendBatchGraph( alloc: Allocator, - node_map: *std.StringArrayHashMapUnmanaged(NodeEdges), - neighbor_tables: *std.StringArrayHashMapUnmanaged(void), + builder: *graph_segment.Builder, batch: rowsource.ColumnBatch, column: rowsource.ColumnVector, ) !usize { @@ -178,14 +180,14 @@ fn appendBatchGraph( .bytes => |values| { for (values, 0..) |value, row| { if (column.nulls.isNull(row)) continue; - total_edges = std.math.add(usize, total_edges, try appendGraphDocument(alloc, node_map, neighbor_tables, batch.row_refs[row], value)) catch + total_edges = std.math.add(usize, total_edges, try appendGraphDocument(alloc, builder, batch.row_refs[row], value)) catch return error.LakeSidecarBuildBudgetExceeded; } }, .json => |values| { for (values, 0..) |value, row| { if (column.nulls.isNull(row)) continue; - total_edges = std.math.add(usize, total_edges, try appendGraphDocument(alloc, node_map, neighbor_tables, batch.row_refs[row], value)) catch + total_edges = std.math.add(usize, total_edges, try appendGraphDocument(alloc, builder, batch.row_refs[row], value)) catch return error.LakeSidecarBuildBudgetExceeded; } }, @@ -196,37 +198,18 @@ fn appendBatchGraph( fn appendGraphDocument( alloc: Allocator, - node_map: *std.StringArrayHashMapUnmanaged(NodeEdges), - neighbor_tables: *std.StringArrayHashMapUnmanaged(void), + builder: *graph_segment.Builder, row_ref: rowsource.RowRef, source_value: []const u8, ) !usize { const node_id = try source_binding.rowRefKeyAlloc(alloc, row_ref); defer alloc.free(node_id); - _ = try ensureNode(alloc, node_map, node_id); + try builder.addNode(node_id); const edges = try parseGraphEdgesAlloc(alloc, source_value); defer freeParsedGraphEdges(alloc, edges); for (edges) |edge| { - const src = try ensureNode(alloc, node_map, node_id); - const neighbor_table_id = if (edge.target_table) |table| - try internNeighborTable(alloc, neighbor_tables, table) - else - null; - try src.out_edges.append(alloc, .{ - .neighbor_id = try alloc.dupe(u8, edge.target), - .edge_type = try alloc.dupe(u8, edge.edge_type), - .weight = edge.weight, - .neighbor_table_id = neighbor_table_id, - }); - if (edge.target_table == null) { - const dst = try ensureNode(alloc, node_map, edge.target); - try dst.in_edges.append(alloc, .{ - .neighbor_id = try alloc.dupe(u8, node_id), - .edge_type = try alloc.dupe(u8, edge.edge_type), - .weight = edge.weight, - }); - } + try builder.addEdge(node_id, edge.target, edge.edge_type, edge.weight, edge.target_table); } return edges.len; } @@ -238,26 +221,11 @@ const ParsedGraphEdge = struct { target_table: ?[]u8, }; -const NodeEdges = struct { - out_edges: std.ArrayListUnmanaged(graph_segment.Edge) = .empty, - in_edges: std.ArrayListUnmanaged(graph_segment.Edge) = .empty, -}; - -fn ensureNode( - alloc: Allocator, - node_map: *std.StringArrayHashMapUnmanaged(NodeEdges), - node_id: []const u8, -) !*NodeEdges { - const gop = try node_map.getOrPut(alloc, node_id); - if (!gop.found_existing) { - gop.key_ptr.* = try alloc.dupe(u8, node_id); - gop.value_ptr.* = .{}; - } - return gop.value_ptr; -} - fn parseGraphEdgesAlloc(alloc: Allocator, value: []const u8) ![]ParsedGraphEdge { - var parsed = std.json.parseFromSlice(std.json.Value, alloc, value, .{}) catch return try alloc.alloc(ParsedGraphEdge, 0); + var parsed = std.json.parseFromSlice(std.json.Value, alloc, value, .{}) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return try alloc.alloc(ParsedGraphEdge, 0), + }; defer parsed.deinit(); const raw_edges: std.json.Array = switch (parsed.value) { .array => |items| items, @@ -285,14 +253,20 @@ fn parseGraphEdgesAlloc(alloc: Allocator, value: []const u8) ![]ParsedGraphEdge if (target.len == 0) continue; const edge_type = jsonObjectStringAny(item.object, &[_][]const u8{ "edge_type", "type" }) orelse ""; const target_table = jsonObjectStringAny(item.object, &.{"target_table"}); + const owned_target = try alloc.dupe(u8, target); + errdefer alloc.free(owned_target); + const owned_type = try alloc.dupe(u8, edge_type); + errdefer alloc.free(owned_type); + const owned_table = if (target_table) |table| + if (table.len > 0) try alloc.dupe(u8, table) else null + else + null; + errdefer if (owned_table) |table| alloc.free(table); try out.append(alloc, .{ - .target = try alloc.dupe(u8, target), - .edge_type = try alloc.dupe(u8, edge_type), + .target = owned_target, + .edge_type = owned_type, .weight = if (item.object.get("weight")) |weight| jsonValueAsF32(weight) catch 1.0 else 1.0, - .target_table = if (target_table) |table| - if (table.len > 0) try alloc.dupe(u8, table) else null - else - null, + .target_table = owned_table, }); } const edges = try out.toOwnedSlice(alloc); @@ -309,6 +283,16 @@ fn freeParsedGraphEdges(alloc: Allocator, edges: []ParsedGraphEdge) void { alloc.free(edges); } +test "serverless lake graph parser propagates allocation failure without losing edges" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, struct { + fn run(alloc: Allocator) !void { + const edges = try parseGraphEdgesAlloc(alloc, "[{\"target\":\"b\",\"edge_type\":\"link\",\"target_table\":\"other\"}]"); + defer freeParsedGraphEdges(alloc, edges); + try std.testing.expectEqual(@as(usize, 1), edges.len); + } + }.run, .{}); +} + fn sortParsedGraphEdges(edges: []ParsedGraphEdge) void { std.mem.sort(ParsedGraphEdge, edges, {}, lessParsedGraphEdge); } @@ -323,60 +307,6 @@ fn lessParsedGraphEdge(_: void, lhs: ParsedGraphEdge, rhs: ParsedGraphEdge) bool return lhs.weight < rhs.weight; } -fn nodeMapToSegmentAlloc( - alloc: Allocator, - node_map: *std.StringArrayHashMapUnmanaged(NodeEdges), - neighbor_table_map: *const std.StringArrayHashMapUnmanaged(void), -) !graph_segment.Segment { - const neighbor_tables = try alloc.alloc([]u8, neighbor_table_map.count()); - errdefer if (neighbor_tables.len > 0) alloc.free(neighbor_tables); - var initialized_tables: usize = 0; - errdefer for (neighbor_tables[0..initialized_tables]) |table| alloc.free(table); - for (neighbor_table_map.keys(), 0..) |table, idx| { - neighbor_tables[idx] = try alloc.dupe(u8, table); - initialized_tables += 1; - } - const adjacencies = try alloc.alloc(graph_segment.Adjacency, node_map.count()); - errdefer alloc.free(adjacencies); - var initialized: usize = 0; - errdefer { - for (adjacencies[0..initialized]) |*adjacency| adjacency.deinit(alloc); - } - - for (node_map.keys(), node_map.values(), 0..) |node_id, *node_edges, idx| { - sortGraphEdges(node_edges.out_edges.items); - sortGraphEdges(node_edges.in_edges.items); - adjacencies[idx] = .{ - .node_id = try alloc.dupe(u8, node_id), - .out_edges = try node_edges.out_edges.toOwnedSlice(alloc), - .in_edges = try node_edges.in_edges.toOwnedSlice(alloc), - }; - initialized += 1; - } - std.mem.sort(graph_segment.Adjacency, adjacencies, {}, lessGraphAdjacency); - return .{ .neighbor_tables = neighbor_tables, .adjacencies = adjacencies }; -} - -fn internNeighborTable( - alloc: Allocator, - tables: *std.StringArrayHashMapUnmanaged(void), - table: []const u8, -) !u32 { - if (tables.getIndex(table)) |index| return std.math.cast(u32, index) orelse error.GraphSegmentTooLarge; - const next_id = std.math.cast(u32, tables.count()) orelse return error.GraphSegmentTooLarge; - const owned = try alloc.dupe(u8, table); - errdefer alloc.free(owned); - const gop = try tables.getOrPut(alloc, owned); - std.debug.assert(!gop.found_existing); - std.debug.assert(gop.index == @as(usize, next_id)); - return next_id; -} - -fn deinitNeighborTableMap(alloc: Allocator, tables: *std.StringArrayHashMapUnmanaged(void)) void { - for (tables.keys()) |table| alloc.free(table); - tables.deinit(alloc); -} - fn optionalStringOrder(lhs: ?[]const u8, rhs: ?[]const u8) std.math.Order { if (lhs == null and rhs == null) return .eq; if (lhs == null) return .lt; @@ -384,31 +314,6 @@ fn optionalStringOrder(lhs: ?[]const u8, rhs: ?[]const u8) std.math.Order { return std.mem.order(u8, lhs.?, rhs.?); } -fn deinitNodeMap(alloc: Allocator, node_map: *std.StringArrayHashMapUnmanaged(NodeEdges)) void { - for (node_map.keys(), node_map.values()) |key, *value| { - alloc.free(key); - for (value.out_edges.items) |*edge| edge.deinit(alloc); - value.out_edges.deinit(alloc); - for (value.in_edges.items) |*edge| edge.deinit(alloc); - value.in_edges.deinit(alloc); - } - node_map.deinit(alloc); -} - -fn sortGraphEdges(edges: []graph_segment.Edge) void { - std.mem.sort(graph_segment.Edge, edges, {}, lessGraphEdge); -} - -fn lessGraphEdge(_: void, lhs: graph_segment.Edge, rhs: graph_segment.Edge) bool { - const lookup_order = graph_segment.edgeLookupOrder(lhs.edge_type, lhs.neighbor_id, rhs.edge_type, rhs.neighbor_id); - if (lookup_order != .eq) return lookup_order == .lt; - return lhs.weight < rhs.weight; -} - -fn lessGraphAdjacency(_: void, lhs: graph_segment.Adjacency, rhs: graph_segment.Adjacency) bool { - return std.mem.order(u8, lhs.node_id, rhs.node_id) == .lt; -} - fn jsonObjectStringAny(obj: std.json.ObjectMap, keys: []const []const u8) ?[]const u8 { for (keys) |key| { const value = obj.get(key) orelse continue; diff --git a/zig/pkg/antfly/src/serverless/build/lake_sidecar_sparse.zig b/zig/pkg/antfly/src/serverless/build/lake_sidecar_sparse.zig index b22a025043..d2179992e2 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_sidecar_sparse.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_sidecar_sparse.zig @@ -16,6 +16,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const artifact_ref = @import("../manifest/artifact_ref.zig"); const artifact_store = @import("../artifacts/store.zig"); const document_projection = @import("../document_projection.zig"); @@ -32,6 +33,7 @@ pub const SparseSidecarBuildOptions = struct { sparse_column: []const u8, artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const SparseSidecarBuildResult = struct { @@ -88,7 +90,9 @@ fn buildSparseSidecarBoundedAlloc( } var posting_count: usize = 0; - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try sidecar_manifest.validateBatchAgainstDeclaredArtifact(.{ .name = options.name, @@ -173,7 +177,7 @@ pub fn publishSparseSidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); diff --git a/zig/pkg/antfly/src/serverless/build/lake_sidecar_text.zig b/zig/pkg/antfly/src/serverless/build/lake_sidecar_text.zig index 31d8950b7a..e5d0cef202 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_sidecar_text.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_sidecar_text.zig @@ -20,6 +20,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const artifact_ref = @import("../manifest/artifact_ref.zig"); const artifact_store = @import("../artifacts/store.zig"); const indexed_reader = @import("../query/indexed_reader.zig"); @@ -36,6 +37,7 @@ pub const TextSidecarBuildOptions = struct { config_json: []const u8 = "{}", artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const TextSidecarBuildResult = struct { @@ -91,7 +93,9 @@ fn buildTextSidecarBoundedAlloc( } var posting_count: usize = 0; - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try sidecar_manifest.validateBatchAgainstDeclaredArtifact(.{ .name = options.name, @@ -185,7 +189,7 @@ pub fn publishTextSidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); diff --git a/zig/pkg/antfly/src/serverless/build/lake_sidecar_vector.zig b/zig/pkg/antfly/src/serverless/build/lake_sidecar_vector.zig index 31d3d78834..db8a8569e1 100644 --- a/zig/pkg/antfly/src/serverless/build/lake_sidecar_vector.zig +++ b/zig/pkg/antfly/src/serverless/build/lake_sidecar_vector.zig @@ -16,6 +16,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const shared_vector = @import("antfly_vector").vector; const artifact_ref = @import("../manifest/artifact_ref.zig"); const artifact_store = @import("../artifacts/store.zig"); @@ -36,6 +37,7 @@ pub const VectorSidecarBuildOptions = struct { embedding_name: ?[]const u8 = null, artifact_id: []const u8 = &.{}, limits: lake_build_limits.Limits = .{}, + cancellation: CancellationToken = .none, }; pub const VectorSidecarBuildResult = struct { @@ -88,7 +90,9 @@ fn buildVectorSidecarBoundedAlloc( }; var dims: ?u32 = null; - while (try source.next(alloc)) |batch| { + while (true) { + try options.cancellation.check(); + const batch = try source.next(alloc) orelse break; try budget.admitBatch(batch); try sidecar_manifest.validateBatchAgainstDeclaredArtifact(.{ .name = options.name, @@ -148,7 +152,7 @@ pub fn publishVectorSidecarFromRowSourceAlloc( defer alloc.free(built.payload); errdefer freeOwnedDeclaration(alloc, built.declaration); - var metadata = try artifacts.put(built.payload); + var metadata = try artifacts.putWithCancellation(built.payload, options.cancellation); var metadata_owned = true; errdefer if (metadata_owned) metadata.deinit(alloc); diff --git a/zig/pkg/antfly/src/serverless/build/mod.zig b/zig/pkg/antfly/src/serverless/build/mod.zig index 1424285bd1..46161a4a09 100644 --- a/zig/pkg/antfly/src/serverless/build/mod.zig +++ b/zig/pkg/antfly/src/serverless/build/mod.zig @@ -22,6 +22,8 @@ pub const external_source_plan_resolver = @import("external_source_plan_resolver pub const external_source_plan_resolver_api = @import("external_source_plan_resolver_api.zig"); pub const external_source_publish = @import("external_source_publish.zig"); pub const external_source_publication = @import("external_source_publication.zig"); +pub const graph_metric_config = @import("graph_metric_config.zig"); +pub const graph_metric_policy = @import("graph_metric_policy.zig"); pub const impact_planner = @import("impact_planner.zig"); pub const lake_gc = @import("lake_gc.zig"); pub const lake_promotion = @import("lake_promotion.zig"); @@ -30,6 +32,7 @@ pub const lake_build_limits = @import("lake_build_limits.zig"); pub const lake_replay = @import("lake_replay.zig"); pub const lake_sidecar_algebraic = @import("lake_sidecar_algebraic.zig"); pub const lake_sidecar_graph = @import("lake_sidecar_graph.zig"); +pub const lake_graph_metric = @import("lake_graph_metric.zig"); pub const lake_sidecar_sparse = @import("lake_sidecar_sparse.zig"); pub const lake_sidecar_text = @import("lake_sidecar_text.zig"); pub const lake_sidecar_vector = @import("lake_sidecar_vector.zig"); @@ -112,6 +115,8 @@ pub const executeLakeRebuildOperationsWithOptionsAlloc = lake_rebuild.executeOpe pub const deleteDroppedLakeRebuildArtifactsAfterPublishAlloc = lake_rebuild.deleteDroppedArtifactsAfterPublishAlloc; pub const reconcileLakeRebuildExecutedOperationsAlloc = lake_rebuild.reconcileExecutedOperationsAlloc; pub const reconcileResolvedExternalLakeSidecarsAlloc = lake_rebuild.reconcileResolvedExternalSourceSidecarsAlloc; +pub const reconcileResolvedExternalLakeSidecarsWithCancellationAlloc = lake_rebuild.reconcileResolvedExternalSourceSidecarsWithCancellationAlloc; +pub const reconcileResolvedExternalLakeSidecarsWithRuntimeAlloc = lake_rebuild.reconcileResolvedExternalSourceSidecarsWithRuntimeAlloc; pub const LakeAlgebraicGroupBySidecarBuildOptions = lake_sidecar_algebraic.AlgebraicGroupBySidecarBuildOptions; pub const LakeAlgebraicGroupBySidecarBuildResult = lake_sidecar_algebraic.AlgebraicGroupBySidecarBuildResult; pub const LakeAlgebraicGroupBySidecarPublishResult = lake_sidecar_algebraic.AlgebraicGroupBySidecarPublishResult; @@ -127,6 +132,14 @@ pub const LakeGraphSidecarBuildResult = lake_sidecar_graph.GraphSidecarBuildResu pub const LakeGraphSidecarPublishResult = lake_sidecar_graph.GraphSidecarPublishResult; pub const buildLakeGraphSidecarFromRowSourceAlloc = lake_sidecar_graph.buildGraphSidecarFromRowSourceAlloc; pub const publishLakeGraphSidecarFromRowSourceAlloc = lake_sidecar_graph.publishGraphSidecarFromRowSourceAlloc; +pub const LakeGraphMetricBuildLimits = lake_graph_metric.Limits; +pub const LakeGraphMetricBuildOptions = lake_graph_metric.BuildOptions; +pub const LakeGraphMetricBuildResult = lake_graph_metric.BuildResult; +pub const default_graph_metric_compute_parallelism = lake_graph_metric.default_compute_parallelism; +pub const max_graph_metric_compute_parallelism = lake_graph_metric.max_compute_parallelism; +pub const buildLakeGraphMetricFromGraphPayloadAlloc = lake_graph_metric.buildFromGraphPayloadAlloc; +pub const publishLakeGraphMetricFromGraphPayloadAlloc = lake_graph_metric.publishFromGraphPayloadAlloc; +pub const publishLakeGraphMetricFromGraphArtifactAlloc = lake_graph_metric.publishFromGraphArtifactAlloc; pub const LakeSparseSidecarBuildOptions = lake_sidecar_sparse.SparseSidecarBuildOptions; pub const LakeSparseSidecarBuildResult = lake_sidecar_sparse.SparseSidecarBuildResult; pub const LakeSparseSidecarPublishResult = lake_sidecar_sparse.SparseSidecarPublishResult; @@ -258,6 +271,8 @@ test "serverless build module compiles" { _ = deleteDroppedLakeRebuildArtifactsAfterPublishAlloc; _ = reconcileLakeRebuildExecutedOperationsAlloc; _ = reconcileResolvedExternalLakeSidecarsAlloc; + _ = reconcileResolvedExternalLakeSidecarsWithCancellationAlloc; + _ = reconcileResolvedExternalLakeSidecarsWithRuntimeAlloc; _ = LakeAlgebraicGroupBySidecarBuildOptions; _ = LakeAlgebraicGroupBySidecarBuildResult; _ = LakeAlgebraicGroupBySidecarPublishResult; @@ -268,6 +283,11 @@ test "serverless build module compiles" { _ = LakeGraphSidecarPublishResult; _ = buildLakeGraphSidecarFromRowSourceAlloc; _ = publishLakeGraphSidecarFromRowSourceAlloc; + _ = LakeGraphMetricBuildLimits; + _ = LakeGraphMetricBuildOptions; + _ = LakeGraphMetricBuildResult; + _ = buildLakeGraphMetricFromGraphPayloadAlloc; + _ = publishLakeGraphMetricFromGraphPayloadAlloc; _ = LakeSparseSidecarBuildOptions; _ = LakeSparseSidecarBuildResult; _ = LakeSparseSidecarPublishResult; diff --git a/zig/pkg/antfly/src/serverless/build/publication_plan.zig b/zig/pkg/antfly/src/serverless/build/publication_plan.zig index 3de0dcfc7c..fc04ce3487 100644 --- a/zig/pkg/antfly/src/serverless/build/publication_plan.zig +++ b/zig/pkg/antfly/src/serverless/build/publication_plan.zig @@ -88,6 +88,7 @@ pub const MetadataRepublishReasons = struct { chunk_preview_policy_changed: bool = false, chunk_embeddings_policy_changed: bool = false, rerank_terms_policy_changed: bool = false, + graph_metric_policy_changed: bool = false, pub fn any(self: MetadataRepublishReasons) bool { return self.read_schema_migration or @@ -96,7 +97,8 @@ pub const MetadataRepublishReasons = struct { self.artifact_families_changed or self.chunk_preview_policy_changed or self.chunk_embeddings_policy_changed or - self.rerank_terms_policy_changed; + self.rerank_terms_policy_changed or + self.graph_metric_policy_changed; } }; diff --git a/zig/pkg/antfly/src/serverless/catalog/service.zig b/zig/pkg/antfly/src/serverless/catalog/service.zig index dffb64f756..3f931913b3 100644 --- a/zig/pkg/antfly/src/serverless/catalog/service.zig +++ b/zig/pkg/antfly/src/serverless/catalog/service.zig @@ -14,6 +14,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const artifacts_mod = @import("../artifacts/mod.zig"); const catalog_types = @import("types.zig"); const catalog_store = @import("store.zig"); @@ -25,6 +26,10 @@ const query_mod = @import("../query/mod.zig"); const segment_mod = @import("../segment/mod.zig"); const wal_mod = @import("../wal/mod.zig"); const builder_mod = @import("../build/builder.zig"); +const graph_metric_policy = @import("../build/graph_metric_policy.zig"); +const graph_metric_config = @import("../build/graph_metric_config.zig"); +const graph_metric_segment = @import("../graph_metric_segment/mod.zig"); +const lake_graph_metric = @import("../build/lake_graph_metric.zig"); const impact_planner = @import("../build/impact_planner.zig"); const external_source_manifest = @import("../build/external_source_manifest.zig"); const publication_plan = @import("../build/publication_plan.zig"); @@ -45,6 +50,101 @@ const PublicationPlanPurpose = enum { publication, }; +const GraphMetricReadiness = struct { + configured: usize = 0, + pending: usize = 0, + rejected: usize = 0, +}; + +fn graphMetricReadinessAlloc(alloc: Allocator, manifest: ?manifest_mod.Manifest, indexes_json: []const u8) !GraphMetricReadiness { + const current = graph_metric_policy.materializerFingerprint(.{}); + const specs = try graph_metric_config.parseIndexSpecsAlloc(alloc, indexes_json); + defer graph_metric_config.freeIndexSpecs(alloc, specs); + var readiness = GraphMetricReadiness{}; + for (specs) |spec| { + const graph_artifact = if (manifest) |value| + findManifestNamedArtifact(value, .graph_segment, spec.index_name) + else + null; + for (spec.configs) |config| { + readiness.configured += 1; + const value = manifest orelse { + readiness.pending += 1; + continue; + }; + const name = try graph_metric_segment.artifactNameAlloc(alloc, spec.index_name, config.name); + defer alloc.free(name); + const artifact = findManifestNamedArtifact(value, .graph_metric_segment, name) orelse { + readiness.pending += 1; + continue; + }; + const source_digest = if (graph_artifact) |graph_ref| blk: { + artifacts_mod.validateSha256ArtifactIdentity(graph_ref.artifact_id, graph_ref.checksum) catch break :blk null; + break :blk artifacts_mod.sha256DigestFromChecksum(graph_ref.checksum) catch null; + } else null; + const current_artifact = artifact.metadata_version == graph_metric_segment.wire_version and + artifact.graph_metric_control_len != 0 and + artifact.graph_metric_routing_footer_len != 0 and + artifact.materializer_fingerprint == current and + artifact.graph_metric_config_fingerprint == lake_graph_metric.configFingerprint(config) and + source_digest != null and + std.mem.eql(u8, &source_digest.?, &artifact.graph_metric_source_checksum); + if (!current_artifact) { + readiness.pending += 1; + continue; + } + if (artifact.graph_metric_materialization_state == .rejected) readiness.rejected += 1; + } + } + return readiness; +} + +fn graphMetricMaterializationStaleAlloc(alloc: Allocator, manifest: manifest_mod.Manifest, indexes_json: []const u8) !bool { + return (try graphMetricReadinessAlloc(alloc, manifest, indexes_json)).pending != 0; +} + +test "serverless catalog schedules idle graph metric policy upgrades from manifest metadata" { + var artifacts = [_]manifest_mod.ArtifactRef{ + .{ + .kind = .graph_segment, + .name = "graph_idx", + .artifact_id = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + .byte_len = 1, + .checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + .{ + .kind = .graph_metric_segment, + .name = "9:graph_idx4:rank", + .artifact_id = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + .byte_len = 1, + .checksum = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + .materializer_fingerprint = 0, + }, + }; + var manifest = manifest_mod.Manifest{ + .namespace = "docs", + .version = 1, + .built_at_ns = 1, + .wal_start_lsn = 0, + .wal_end_lsn = 0, + .stats = .{}, + .artifacts = &artifacts, + }; + const indexes_json = "{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\"}}}}"; + try std.testing.expect(try graphMetricMaterializationStaleAlloc(std.testing.allocator, manifest, indexes_json)); + const specs = try graph_metric_config.parseIndexSpecsAlloc(std.testing.allocator, indexes_json); + defer graph_metric_config.freeIndexSpecs(std.testing.allocator, specs); + artifacts[1].metadata_version = graph_metric_segment.wire_version; + artifacts[1].materializer_fingerprint = graph_metric_policy.materializerFingerprint(.{}); + artifacts[1].graph_metric_control_len = 1; + artifacts[1].graph_metric_routing_footer_len = 1; + artifacts[1].graph_metric_config_fingerprint = lake_graph_metric.configFingerprint(specs[0].configs[0]); + artifacts[1].graph_metric_source_checksum = @splat(0xaa); + try std.testing.expect(!try graphMetricMaterializationStaleAlloc(std.testing.allocator, manifest, indexes_json)); + manifest.artifacts = manifest.artifacts[0..0]; + try std.testing.expect(try graphMetricMaterializationStaleAlloc(std.testing.allocator, manifest, indexes_json)); +} + fn ensureSchemaWritesAllowedAlloc(alloc: Allocator, schema_json: []const u8) !void { var binding = (try publication_plan.externalBindingFromSchemaJsonAlloc(alloc, schema_json)) orelse return; defer binding.deinit(alloc); @@ -482,6 +582,11 @@ pub const CatalogService = struct { const pending_materialization_rebuild = !head_republish_recommended and (plan.artifact_actions.any() or plan.derived_output_actions.any()); + const graph_metric_readiness = try graphMetricReadinessAlloc( + self.alloc, + published_head.manifest, + plan.table_definition.indexes_json, + ); const owned_namespace = try self.alloc.dupe(u8, namespace); errdefer self.alloc.free(owned_namespace); @@ -533,6 +638,9 @@ pub const CatalogService = struct { .document_lineage_versions = head_document_lineage_versions, .head_republish_recommended = head_republish_recommended, .pending_materialization_rebuild = pending_materialization_rebuild, + .graph_metrics_configured = graph_metric_readiness.configured, + .graph_metrics_pending = graph_metric_readiness.pending, + .graph_metrics_rejected = graph_metric_readiness.rejected, .pending_materialization_families = pending_materialization_families, .head_artifact_actions = head_actions.artifact_actions, .head_full_text_index_actions = head_full_text_index_actions, @@ -715,6 +823,14 @@ pub const CatalogService = struct { return try self.buildNamespaceGuarded(namespace, null); } + pub fn buildNamespaceWithCancellation(self: *CatalogService, namespace: []const u8, cancellation: CancellationToken) !builder_mod.BuildResult { + try cancellation.check(); + const policy = self.getPolicy(namespace) catch catalog_types.NamespacePolicy{}; + var plan = try self.publicationPlanForNamespaceAlloc(namespace, policy, .publication); + defer plan.deinit(self.alloc); + return self.builder.publishNamespaceWithMetricAndPlanWithCancellation(namespace, policy.vector_distance_metric, plan, cancellation); + } + pub fn buildNamespaceGuarded( self: *CatalogService, namespace: []const u8, @@ -742,9 +858,18 @@ pub const CatalogService = struct { } pub fn buildTable(self: *CatalogService, table_name: []const u8) !builder_mod.BuildResult { + return try self.buildTableWithCancellation(table_name, .none); + } + + pub fn buildTableWithCancellation( + self: *CatalogService, + table_name: []const u8, + cancellation: CancellationToken, + ) !builder_mod.BuildResult { + try cancellation.check(); const namespace = try self.resolveTableNamespaceAlloc(table_name); defer self.alloc.free(namespace); - return try self.buildNamespace(namespace); + return try self.buildNamespaceWithCancellation(namespace, cancellation); } pub fn tableBuildStatus(self: *CatalogService, table_name: []const u8) !catalog_types.BuildStatus { @@ -865,6 +990,7 @@ pub const CatalogService = struct { metadata_republish.chunk_preview_policy_changed = can_republish_chunk_preview; metadata_republish.chunk_embeddings_policy_changed = can_republish_chunk_embeddings; metadata_republish.rerank_terms_policy_changed = can_republish_rerank_terms; + metadata_republish.graph_metric_policy_changed = try graphMetricMaterializationStaleAlloc(self.alloc, manifest, table.indexes_json); const full_text_index_actions = try planFullTextIndexActionsAlloc( self.alloc, @@ -1609,7 +1735,7 @@ fn planNamedIndexActionsAlloc( if (!isNamedIndexKindValue(entry.value_ptr.*, kind)) continue; const action: publication_plan.ArtifactAction = blk: { if (before_object.get(entry.key_ptr.*)) |before_value| { - if (isNamedIndexKindValue(before_value, kind) and jsonValueEql(entry.value_ptr.*, before_value)) { + if (isNamedIndexKindValue(before_value, kind) and namedIndexConfigEql(entry.value_ptr.*, before_value, kind)) { break :blk .reuse; } } @@ -1804,7 +1930,7 @@ fn findEquivalentRenamedIndexName( while (before_it.next()) |entry| { if (!isNamedIndexKindValue(entry.value_ptr.*, kind)) continue; if (after_object.get(entry.key_ptr.*) != null) continue; - if (!jsonValueEql(entry.value_ptr.*, target_value)) continue; + if (!namedIndexConfigEql(entry.value_ptr.*, target_value, kind)) continue; if (match != null) return null; match = entry.key_ptr.*; } @@ -1977,6 +2103,38 @@ fn jsonValueEql(lhs: std.json.Value, rhs: std.json.Value) bool { }; } +fn namedIndexConfigEql(lhs: std.json.Value, rhs: std.json.Value, kind: NamedSearchSourceKind) bool { + if (kind != .graph) return jsonValueEql(lhs, rhs); + if (lhs != .object or rhs != .object) return false; + var lhs_count: usize = 0; + var lhs_it = lhs.object.iterator(); + while (lhs_it.next()) |entry| { + if (std.mem.eql(u8, entry.key_ptr.*, "metrics")) continue; + lhs_count += 1; + const other = rhs.object.get(entry.key_ptr.*) orelse return false; + if (!jsonValueEql(entry.value_ptr.*, other)) return false; + } + var rhs_count: usize = 0; + var rhs_it = rhs.object.iterator(); + while (rhs_it.next()) |entry| { + if (!std.mem.eql(u8, entry.key_ptr.*, "metrics")) rhs_count += 1; + } + return lhs_count == rhs_count; +} + +test "serverless named graph planning reuses topology for metric-only changes" { + const actions = try planNamedIndexActionsAlloc( + std.testing.allocator, + "{\"graph_idx\":{\"type\":\"graph\",\"field\":\"edges\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":20}}}}", + "{\"graph_idx\":{\"type\":\"graph\",\"field\":\"edges\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\",\"max_iterations\":40}}}}", + .graph, + 1, + ); + defer freeNamedArtifactActions(std.testing.allocator, actions); + try std.testing.expectEqual(@as(usize, 1), actions.len); + try std.testing.expectEqual(publication_plan.ArtifactAction.reuse, actions[0].action); +} + fn publishedSearchSourcesMatch( lhs: search_sources.PublishedSearchSources, rhs: search_sources.PublishedSearchSources, diff --git a/zig/pkg/antfly/src/serverless/catalog/types.zig b/zig/pkg/antfly/src/serverless/catalog/types.zig index 0a076633e1..1d98ea5d16 100644 --- a/zig/pkg/antfly/src/serverless/catalog/types.zig +++ b/zig/pkg/antfly/src/serverless/catalog/types.zig @@ -261,6 +261,9 @@ pub const BuildStatus = struct { document_lineage_versions: u64 = 0, head_republish_recommended: bool, pending_materialization_rebuild: bool, + graph_metrics_configured: usize = 0, + graph_metrics_pending: usize = 0, + graph_metrics_rejected: usize = 0, pending_materialization_families: PendingMaterializationFamilies = .{}, head_artifact_actions: ArtifactPublicationActions = .{}, head_full_text_index_actions: []FullTextIndexPublicationAction = &.{}, diff --git a/zig/pkg/antfly/src/serverless/enrichment/worker.zig b/zig/pkg/antfly/src/serverless/enrichment/worker.zig index eb7c59acf2..cfcbc22da0 100644 --- a/zig/pkg/antfly/src/serverless/enrichment/worker.zig +++ b/zig/pkg/antfly/src/serverless/enrichment/worker.zig @@ -14,6 +14,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const api_codec = @import("../api/codec.zig"); const api_types = @import("../api/types.zig"); const artifacts_mod = @import("../artifacts/mod.zig"); @@ -56,6 +57,7 @@ pub const SparseEnricherConfig = struct { stage: catalog_mod.EnrichmentStage = .lexical_sparse, model_preference: catalog_mod.EnrichmentModelPreference = .prefer_model, failure_policy: catalog_mod.EnrichmentFailurePolicy = .skip_document, + cancellation: CancellationToken = .none, }; const DerivedBodyResult = struct { @@ -110,10 +112,14 @@ pub const SparseEnricher = struct { } pub fn setSparseEmbedder(self: *SparseEnricher, embedder: embedder_mod.SparseEmbedder, embedding_name: []const u8) !void { + // Ownership transfers only after all fallible preparation succeeds. + // This keeps the current configuration usable on allocation failure + // and leaves the caller responsible for the replacement on error. + const owned_name = try self.alloc.dupe(u8, embedding_name); if (self.sparse_embedder) |current| current.deinit(self.alloc); if (self.sparse_embedding_name) |name| self.alloc.free(name); self.sparse_embedder = embedder; - self.sparse_embedding_name = try self.alloc.dupe(u8, embedding_name); + self.sparse_embedding_name = owned_name; } pub fn clearSparseEmbedder(self: *SparseEnricher) void { @@ -124,10 +130,11 @@ pub const SparseEnricher = struct { } pub fn setChunkEmbedder(self: *SparseEnricher, embedder: embedder_mod.DenseEmbedder, embedding_name: []const u8, dims: u32) !void { + const owned_name = try self.alloc.dupe(u8, embedding_name); if (self.chunk_embedder) |current| current.deinit(self.alloc); if (self.chunk_embedding_name) |name| self.alloc.free(name); self.chunk_embedder = embedder; - self.chunk_embedding_name = try self.alloc.dupe(u8, embedding_name); + self.chunk_embedding_name = owned_name; self.chunk_embedding_dims = dims; } @@ -153,6 +160,7 @@ pub const SparseEnricher = struct { cancellation: ?maintenance_cancellation.Token, ) !EnrichmentRunStats { try maintenance_cancellation.check(cancellation); + try cfg.cancellation.check(); const head = self.progress.getHead(namespace) catch |err| switch (err) { error.FileNotFound => return .{ .idle_namespaces = 1 }, else => return err, @@ -173,6 +181,7 @@ pub const SparseEnricher = struct { const docs = try self.loadPublishedDocsAlloc(manifest); defer query_mod.freeMaterializedDocuments(self.alloc, docs); try maintenance_cancellation.check(cancellation); + try cfg.cancellation.check(); // Loading and materializing the immutable manifest can be expensive. // Do not initialize progress or emit work if publication moved while @@ -223,6 +232,7 @@ pub const SparseEnricher = struct { var expected_latest_lsn = latest_lsn; for (docs[start_offset..], start_offset..) |doc, doc_index| { try maintenance_cancellation.check(cancellation); + try cfg.cancellation.check(); next_offset = doc_index + 1; const derived = buildDerivedBodyAlloc(self, cfg.stage, doc.body, cfg.pipeline_version, cfg.model_preference) catch |err| { if (isRecoverableEnrichmentError(err)) { @@ -245,6 +255,7 @@ pub const SparseEnricher = struct { }; const encoded = try api_codec.encodeMutationAlloc(self.alloc, mutation); defer self.alloc.free(encoded); + try cfg.cancellation.check(); const timestamp_ns = std.math.add(u64, doc.last_timestamp_ns, 1) catch return error.EnrichmentTimestampOverflow; var operation_id_buf: [128]u8 = undefined; @@ -909,6 +920,69 @@ const FailingSparseEmbedder = struct { } }; +const TrackingEmbedder = struct { + deinit_count: *usize, + + fn embedDense(_: *anyopaque, _: Allocator, _: []const u8, _: []const u8, _: u32) ![]f32 { + return error.UnexpectedEmbeddingCall; + } + + fn embedSparse(_: *anyopaque, _: Allocator, _: []const u8, _: []const u8) !embedder_mod.SparseEmbedding { + return error.UnexpectedEmbeddingCall; + } + + fn deinit(ptr: *anyopaque, _: Allocator) void { + const self: *TrackingEmbedder = @ptrCast(@alignCast(ptr)); + self.deinit_count.* += 1; + } + + fn denseInterface(self: *TrackingEmbedder) embedder_mod.DenseEmbedder { + return .{ .ptr = self, .dense_embed_fn = embedDense, .deinit_fn = deinit }; + } + + fn sparseInterface(self: *TrackingEmbedder) embedder_mod.SparseEmbedder { + return .{ .ptr = self, .sparse_embed_fn = embedSparse, .deinit_fn = deinit }; + } +}; + +test "serverless sparse enricher embedder replacement is transactional on allocation failure" { + var old_sparse_deinits: usize = 0; + var replacement_sparse_deinits: usize = 0; + var old_dense_deinits: usize = 0; + var replacement_dense_deinits: usize = 0; + var old_sparse = TrackingEmbedder{ .deinit_count = &old_sparse_deinits }; + var replacement_sparse = TrackingEmbedder{ .deinit_count = &replacement_sparse_deinits }; + var old_dense = TrackingEmbedder{ .deinit_count = &old_dense_deinits }; + var replacement_dense = TrackingEmbedder{ .deinit_count = &replacement_dense_deinits }; + + var enricher = SparseEnricher.init(std.testing.allocator, undefined, undefined, undefined, undefined); + defer enricher.deinit(); + try enricher.setSparseEmbedder(old_sparse.sparseInterface(), "old_sparse"); + try enricher.setChunkEmbedder(old_dense.denseInterface(), "old_dense", 32); + + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 }); + enricher.alloc = failing.allocator(); + try std.testing.expectError(error.OutOfMemory, enricher.setSparseEmbedder(replacement_sparse.sparseInterface(), "new_sparse")); + replacement_sparse.sparseInterface().deinit(std.testing.allocator); + try std.testing.expectEqualStrings("old_sparse", enricher.sparse_embedding_name.?); + try std.testing.expectEqual(@as(usize, 0), old_sparse_deinits); + + failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 }); + enricher.alloc = failing.allocator(); + try std.testing.expectError(error.OutOfMemory, enricher.setChunkEmbedder(replacement_dense.denseInterface(), "new_dense", 64)); + replacement_dense.denseInterface().deinit(std.testing.allocator); + try std.testing.expectEqualStrings("old_dense", enricher.chunk_embedding_name.?); + try std.testing.expectEqual(@as(u32, 32), enricher.chunk_embedding_dims); + try std.testing.expectEqual(@as(usize, 0), old_dense_deinits); + + enricher.alloc = std.testing.allocator; + enricher.clearSparseEmbedder(); + enricher.clearChunkEmbedder(); + try std.testing.expectEqual(@as(usize, 1), old_sparse_deinits); + try std.testing.expectEqual(@as(usize, 1), replacement_sparse_deinits); + try std.testing.expectEqual(@as(usize, 1), old_dense_deinits); + try std.testing.expectEqual(@as(usize, 1), replacement_dense_deinits); +} const CancelingSparseEmbedder = struct { requested: *std.atomic.Value(bool), diff --git a/zig/pkg/antfly/src/serverless/graph_metric_segment/codec.zig b/zig/pkg/antfly/src/serverless/graph_metric_segment/codec.zig new file mode 100644 index 0000000000..e45e0c42cc --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_metric_segment/codec.zig @@ -0,0 +1,1606 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +const bounded_decode = @import("../bounded_decode.zig"); +const graph_mod = @import("../../graph/graph.zig"); +const artifact_ref = @import("../manifest/artifact_ref.zig"); +const types = @import("types.zig"); + +pub const wire_magic = "AFGM"; +pub const wire_version: u16 = artifact_ref.graph_metric_segment_wire_version; +const fixed_header_len = wire_magic.len + @sizeOf(u16) + 4 * @sizeOf(u8) + @sizeOf(u32) + + 3 * @sizeOf(u64) + 3 * @sizeOf(u32) + 32; +const routing_magic = "AFGR"; +// Footer: AFGR/count, flat entries grouped into authenticated 64-entry pages, +// AFGD directory, AFTK root, root-length/footer-length trailer. The root binds +// both the directory digest (sparse reads) and full point digest (verification). +const directory_magic = "AFGD"; +pub const routing_page_entries: usize = 64; +const top_tier_magic = "AFTK"; +pub const routing_trailer_len: usize = 8; +pub const score_block_entries: usize = 1024; +pub const ranked_score_block_entries: usize = 256; +pub const max_persisted_top_entries: usize = 10_000; +pub const max_routing_bytes: usize = 16 * 1024 * 1024; +pub const max_score_node_id_bytes: usize = 4096; +pub const max_ranked_score_block_bytes: usize = @sizeOf(u16) + max_score_node_id_bytes + ranked_score_block_entries * (ranked_score_fixed_len + max_score_node_id_bytes); +const routing_header_len = routing_magic.len + @sizeOf(u32); +const routing_entry_fixed_len = @sizeOf(u32) + @sizeOf(u64) + @sizeOf(u32) + std.crypto.hash.sha2.Sha256.digest_length; +const top_tier_header_len = top_tier_magic.len + @sizeOf(u32) + @sizeOf(u32); +const routing_root_metadata_len = 3 * @sizeOf(u64) + 64; +const routing_root_trailer_len = @sizeOf(u32) + routing_trailer_len; +const ranked_score_fixed_len = @sizeOf(u16) + @sizeOf(u64); +const ranked_routing_entry_len = @sizeOf(u64) + @sizeOf(u32) + std.crypto.hash.sha2.Sha256.digest_length; +const max_ranked_score_blocks = (max_persisted_top_entries + ranked_score_block_entries - 1) / ranked_score_block_entries; + +pub const Header = struct { + version: u16, + kind: @import("../../graph/graph.zig").GraphMetricKind, + materialization_state: types.MaterializationState, + rejection_reason: types.RejectionReason, + config_fingerprint: u64, + materializer_fingerprint: u64, + source_graph_artifact_id: []const u8, + source_graph_checksum: []const u8, + topology_checksum: [32]u8 = @splat(0), + converged: bool, + iterations_completed: u32, + delta: f64, + edge_type_count: u32, +}; + +pub const Control = struct { + header: Header, + score_count: u32, + score_data_offset: u64, +}; + +pub const RoutingEntry = struct { + /// Global score-block ordinal, including when only selected pages are loaded. + block_index: usize = 0, + first_node_id: []const u8, + offset: u64, + len: usize, + checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = @splat(0), +}; + +pub const RankedRoutingEntry = struct { + offset: u64, + len: usize, + checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, +}; + +const PersistedTopScore = struct { + node_id: []const u8, + value: f64, +}; + +pub const ArtifactIntegrity = struct { + control_len: u32, + routing_footer_len: u32, + control_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, + routing_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, + point_index_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, +}; + +pub const RoutingIndex = struct { + entries: []RoutingEntry, + top_score_count: usize, + ranked_entries: []RankedRoutingEntry, + footer_offset: u64, + primary_data_offset: u64 = 0, + primary_data_end: u64 = 0, + point_index_checksum: [32]u8 = @splat(0), + directory_len: usize = 0, + directory_checksum: [32]u8 = @splat(0), + + pub fn deinit(self: *RoutingIndex, alloc: Allocator) void { + alloc.free(self.entries); + alloc.free(self.ranked_entries); + self.* = undefined; + } + + pub fn find(self: RoutingIndex, node_id: []const u8) ?RoutingEntry { + const index = self.findIndex(node_id) orelse return null; + return self.entries[index]; + } + + pub fn findIndex(self: RoutingIndex, node_id: []const u8) ?usize { + var low: usize = 0; + var high = self.entries.len; + while (low < high) { + const mid = low + (high - low) / 2; + if (std.mem.order(u8, self.entries[mid].first_node_id, node_id) != .gt) + low = mid + 1 + else + high = mid; + } + return if (low == 0) null else low - 1; + } +}; + +pub const BorrowedBlockScore = struct { + node_suffix: []const u8, + value: f64, + + pub fn nodeIdLen(self: @This(), node_prefix: []const u8) usize { + return node_prefix.len + self.node_suffix.len; + } + + pub fn orderNode(self: @This(), node_prefix: []const u8, node_id: []const u8) std.math.Order { + const shared_prefix = @min(node_prefix.len, node_id.len); + const prefix_order = std.mem.order(u8, node_prefix[0..shared_prefix], node_id[0..shared_prefix]); + if (prefix_order != .eq) return prefix_order; + if (node_id.len < node_prefix.len) return .gt; + return std.mem.order(u8, self.node_suffix, node_id[node_prefix.len..]); + } + + pub fn eqlNode(self: @This(), node_prefix: []const u8, node_id: []const u8) bool { + return self.nodeIdLen(node_prefix) == node_id.len and self.orderNode(node_prefix, node_id) == .eq; + } + + pub fn copyNode(self: @This(), node_prefix: []const u8, destination: []u8) ![]u8 { + if (destination.len < self.nodeIdLen(node_prefix)) return error.NoSpaceLeft; + @memcpy(destination[0..node_prefix.len], node_prefix); + @memcpy(destination[node_prefix.len..][0..self.node_suffix.len], self.node_suffix); + return destination[0..self.nodeIdLen(node_prefix)]; + } + + pub fn dupeNodeAlloc(self: @This(), alloc: Allocator, node_prefix: []const u8) ![]u8 { + const result = try alloc.alloc(u8, self.nodeIdLen(node_prefix)); + errdefer alloc.free(result); + _ = try self.copyNode(node_prefix, result); + return result; + } +}; + +pub const DecodedScoreBlock = struct { + /// Stored once per block rather than once per score, keeping the bounded + /// decode frame compact even when node IDs have long shared prefixes. + node_prefix: []const u8 = &.{}, + scores: [score_block_entries]BorrowedBlockScore = undefined, + len: usize = 0, + + pub fn score(self: *const DecodedScoreBlock, node_id: []const u8) ?f64 { + var low: usize = 0; + var high = self.len; + while (low < high) { + const mid = low + (high - low) / 2; + switch (self.scores[mid].orderNode(self.node_prefix, node_id)) { + .lt => low = mid + 1, + .gt => high = mid, + .eq => return self.scores[mid].value, + } + } + return null; + } + + /// Candidates are in canonical ID order, with original row ordinals for + /// scattering. Sparse probes retain binary search; dense spans merge once + /// through the borrowed block. Duplicate candidates do not advance the + /// block cursor and therefore preserve every caller-visible row. + pub fn populateSorted(self: *const DecodedScoreBlock, node_ids: []const []const u8, rows: []const u32, values: []?f64, cancellation: CancellationToken) !void { + if (node_ids.len != values.len) return error.InvalidGraphMetricSegment; + const search_cost = 1 + std.math.log2_int(usize, @max(self.len, 1)); + const merge = rows.len > (self.len + rows.len) / search_cost; + if (!merge) { + for (rows, 0..) |row, i| { + if (i % 256 == 0) try cancellation.check(); + if (row >= node_ids.len) return error.InvalidGraphMetricSegment; + values[row] = self.score(node_ids[row]); + } + return; + } + var position: usize = 0; + for (rows, 0..) |row, i| { + if (i % 256 == 0) try cancellation.check(); + if (row >= node_ids.len) return error.InvalidGraphMetricSegment; + while (position < self.len and self.scores[position].orderNode(self.node_prefix, node_ids[row]) == .lt) position += 1; + values[row] = if (position < self.len and self.scores[position].eqlNode(self.node_prefix, node_ids[row])) self.scores[position].value else null; + } + } +}; + +test "serverless graph metric adaptive score join preserves sparse dense duplicate and missing rows" { + var names: [1024][8]u8 = undefined; + var block = DecodedScoreBlock{ .node_prefix = "collection/", .len = names.len }; + var ids: [1026][]const u8 = undefined; + var full: [1024][19]u8 = undefined; + for (&names, &full, 0..) |*name, *id, i| { + const suffix = try std.fmt.bufPrint(name, "{d:0>8}", .{i * 2}); + ids[i] = try std.fmt.bufPrint(id, "collection/{s}", .{suffix}); + block.scores[i] = .{ .node_suffix = suffix, .value = @floatFromInt(i) }; + } + ids[1024] = ids[512]; + ids[1025] = "collection/00001025"; + var rows: [1026]u32 = undefined; + for (&rows, 0..) |*row, i| row.* = @intCast(i); + std.mem.sort(u32, &rows, &ids, struct { + fn less(input: *[1026][]const u8, a: u32, b: u32) bool { + return std.mem.lessThan(u8, input[a], input[b]); + } + }.less); + var values: [1026]?f64 = @splat(null); + for ([_]usize{ 0, 1, 7, 128, 1026 }) |count| { + @memset(&values, null); + try block.populateSorted(&ids, rows[0..count], &values, .none); + for (rows[0..count]) |row| try std.testing.expectEqual(block.score(ids[row]), values[row]); + } + try std.testing.expectEqual(@as(?f64, 512), values[1024]); + try std.testing.expectEqual(@as(?f64, null), values[1025]); + try std.testing.expectError(error.InvalidGraphMetricSegment, block.populateSorted(&ids, &.{1026}, &values, .none)); +} + +/// Exact bounded prefix needed to decode provenance from a current artifact. +/// Logical names are manifest metadata and deliberately do not affect the +/// content address. +pub fn headerProbeLen( + artifact_byte_len: u64, + source_graph_artifact_id: []const u8, + source_graph_checksum: []const u8, +) !usize { + var required: usize = fixed_header_len; + required = std.math.add(usize, required, source_graph_artifact_id.len) catch return error.GraphMetricSegmentTooLarge; + required = std.math.add(usize, required, source_graph_checksum.len) catch return error.GraphMetricSegmentTooLarge; + const required_u64 = std.math.cast(u64, required) orelse return error.GraphMetricSegmentTooLarge; + return @intCast(@min(artifact_byte_len, required_u64)); +} + +pub fn controlProbeLen( + artifact_byte_len: u64, + source_graph_artifact_id: []const u8, + source_graph_checksum: []const u8, + edge_filter: graph_mod.GraphMetricEdgeFilter, +) !usize { + var required = try headerProbeLen( + std.math.maxInt(u64), + source_graph_artifact_id, + source_graph_checksum, + ); + for (edge_filter.types) |edge_type| { + required = std.math.add(usize, required, @sizeOf(u32) + edge_type.len) catch return error.GraphMetricSegmentTooLarge; + } + required = std.math.add(usize, required, @sizeOf(u32)) catch return error.GraphMetricSegmentTooLarge; + const required_u64 = std.math.cast(u64, required) orelse return error.GraphMetricSegmentTooLarge; + return @intCast(@min(artifact_byte_len, required_u64)); +} + +/// Decodes the bounded control prefix used by point-score readers and checks +/// that the persisted edge filter is exactly the configured set. +pub fn decodeControl(data: []const u8, expected_edge_filter: graph_mod.GraphMetricEdgeFilter) !Control { + const header = try decodeHeader(data); + var pos = fixed_header_len; + pos = std.math.add(usize, pos, header.source_graph_artifact_id.len) catch return error.InvalidGraphMetricSegment; + pos = std.math.add(usize, pos, header.source_graph_checksum.len) catch return error.InvalidGraphMetricSegment; + if (header.edge_type_count != expected_edge_filter.types.len) return error.InvalidGraphMetricSegment; + if ((header.edge_type_count == 0) != (expected_edge_filter.mode == .all)) return error.InvalidGraphMetricSegment; + var previous: ?[]const u8 = null; + for (0..header.edge_type_count) |_| { + const edge_type_len = try readInt(u32, data, &pos); + const edge_type = try take(data, &pos, edge_type_len); + if (edge_type.len == 0 or (previous != null and std.mem.order(u8, previous.?, edge_type) != .lt)) { + return error.InvalidGraphMetricSegment; + } + var found = false; + for (expected_edge_filter.types) |expected| { + if (std.mem.eql(u8, expected, edge_type)) { + found = true; + break; + } + } + if (!found) return error.InvalidGraphMetricSegment; + previous = edge_type; + } + const score_count = try readInt(u32, data, &pos); + return .{ .header = header, .score_count = score_count, .score_data_offset = @intCast(pos) }; +} + +pub fn routingFooterLenFromTrailer(artifact_byte_len: u64, trailer: []const u8) !usize { + if (trailer.len != routing_trailer_len) return error.InvalidGraphMetricSegment; + const footer_len_u64 = std.mem.readInt(u64, trailer[0..routing_trailer_len], .little); + if (footer_len_u64 < routing_header_len + routing_trailer_len or + footer_len_u64 > max_routing_bytes or footer_len_u64 > artifact_byte_len) + { + return error.InvalidGraphMetricSegment; + } + return std.math.cast(usize, footer_len_u64) orelse error.InvalidGraphMetricSegment; +} + +/// Decodes a routing footer whose node-id slices borrow from `footer`. +pub fn decodeRoutingIndexAlloc(alloc: Allocator, footer: []const u8, artifact_byte_len: u64) !RoutingIndex { + return decodeRoutingIndexForVersionWithCancellationAlloc(alloc, footer, artifact_byte_len, wire_version, .none); +} + +pub fn decodeRoutingIndexWithCancellationAlloc( + alloc: Allocator, + footer: []const u8, + artifact_byte_len: u64, + cancellation: CancellationToken, +) !RoutingIndex { + return decodeRoutingIndexForVersionWithCancellationAlloc(alloc, footer, artifact_byte_len, wire_version, cancellation); +} + +pub fn decodeRoutingIndexForVersionWithCancellationAlloc( + alloc: Allocator, + footer: []const u8, + artifact_byte_len: u64, + segment_version: u16, + cancellation: CancellationToken, +) !RoutingIndex { + try cancellation.check(); + if (segment_version != wire_version) return error.UnsupportedGraphMetricSegmentVersion; + if (footer.len < routing_header_len + routing_trailer_len) return error.InvalidGraphMetricSegment; + const footer_len = try routingFooterLenFromTrailer(artifact_byte_len, footer[footer.len - routing_trailer_len ..]); + if (footer_len != footer.len) return error.InvalidGraphMetricSegment; + const footer_offset = artifact_byte_len - footer.len; + const root_len = try routingRootLenFromTrailer(footer); + const point_index = footer[0 .. footer.len - root_len]; + var root = try decodeRoutingRootAlloc(alloc, footer[footer.len - root_len ..], artifact_byte_len, segment_version, cancellation); + errdefer root.deinit(alloc); + var point_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(point_index, &point_checksum, .{}); + if (!std.mem.eql(u8, &point_checksum, &root.point_index_checksum)) return error.InvalidGraphMetricSegment; + var pos: usize = 0; + if (!std.mem.eql(u8, try take(footer, &pos, routing_magic.len), routing_magic)) return error.InvalidGraphMetricSegment; + const entry_count = try readInt(u32, footer, &pos); + if (@as(usize, entry_count) > (footer.len - routing_header_len - routing_trailer_len) / routing_entry_fixed_len) { + return error.InvalidGraphMetricSegment; + } + const entries = try alloc.alloc(RoutingEntry, entry_count); + errdefer alloc.free(entries); + var previous_end: ?u64 = null; + for (entries, 0..) |*entry, entry_index| { + if (entry_index % 256 == 0) try cancellation.check(); + const first_node_id_len = try readInt(u32, footer, &pos); + if (first_node_id_len == 0 or first_node_id_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const offset = try readInt(u64, footer, &pos); + const len_u32 = try readInt(u32, footer, &pos); + var checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + @memcpy(&checksum, try take(footer, &pos, checksum.len)); + if (len_u32 == 0) return error.InvalidGraphMetricSegment; + const first_node_id = try take(footer, &pos, first_node_id_len); + const end = std.math.add(u64, offset, len_u32) catch return error.InvalidGraphMetricSegment; + if (end > footer_offset or (previous_end != null and offset != previous_end.?)) return error.InvalidGraphMetricSegment; + if (entry_index > 0 and std.mem.order(u8, entries[entry_index - 1].first_node_id, first_node_id) != .lt) { + return error.InvalidGraphMetricSegment; + } + entry.* = .{ .block_index = entry_index, .first_node_id = first_node_id, .offset = offset, .len = len_u32, .checksum = checksum }; + previous_end = end; + } + if (pos + root.directory_len != point_index.len or (previous_end orelse root.primary_data_offset) != root.primary_data_end or + (entries.len != 0 and entries[0].offset != root.primary_data_offset)) return error.InvalidGraphMetricSegment; + try validateDirectory(point_index, pos, footer_offset, entry_count, root.directory_checksum, cancellation); + try cancellation.check(); + alloc.free(root.entries); + root.entries = entries; + return root; +} + +/// The root is bounded independently of primary vector cardinality. +/// Its trailer length is included in the manifest-authenticated digest. +pub fn routingRootLen(score_count: usize) usize { + return top_tier_header_len + routing_root_metadata_len + routing_root_trailer_len + + scoreBlockCountForSize(@min(score_count, max_persisted_top_entries), ranked_score_block_entries) * ranked_routing_entry_len; +} + +fn routingRootLenFromTrailer(footer: []const u8) !usize { + if (footer.len < routing_root_trailer_len) return error.InvalidGraphMetricSegment; + const len = std.mem.readInt(u32, footer[footer.len - routing_root_trailer_len ..][0..4], .little); + if (len < routingRootLen(0) or len > routingRootLen(max_persisted_top_entries) or len > footer.len) return error.InvalidGraphMetricSegment; + return len; +} + +pub fn decodeRoutingRootAlloc(alloc: Allocator, root: []const u8, artifact_byte_len: u64, version: u16, cancellation: CancellationToken) !RoutingIndex { + try cancellation.check(); + if (version != wire_version) return error.UnsupportedGraphMetricSegmentVersion; + if (try routingRootLenFromTrailer(root) != root.len) return error.InvalidGraphMetricSegment; + const footer_len = try routingFooterLenFromTrailer(artifact_byte_len, root[root.len - routing_trailer_len ..]); + if (footer_len < root.len + routing_header_len) return error.InvalidGraphMetricSegment; + const footer_offset = artifact_byte_len - footer_len; + var pos: usize = 0; + if (!std.mem.eql(u8, try take(root, &pos, top_tier_magic.len), top_tier_magic)) return error.InvalidGraphMetricSegment; + const count = try readInt(u32, root, &pos); + const blocks = try readInt(u32, root, &pos); + if (count > max_persisted_top_entries or root.len != routingRootLen(count) or blocks != scoreBlockCountForSize(count, ranked_score_block_entries)) return error.InvalidGraphMetricSegment; + const primary_start = try readInt(u64, root, &pos); + const primary_end = try readInt(u64, root, &pos); + if (primary_start > primary_end or primary_end > footer_offset) return error.InvalidGraphMetricSegment; + var point_checksum: [32]u8 = undefined; + @memcpy(&point_checksum, try take(root, &pos, 32)); + const directory_len = std.math.cast(usize, try readInt(u64, root, &pos)) orelse return error.InvalidGraphMetricSegment; + if (directory_len < routing_header_len or directory_len > footer_len - root.len - routing_header_len) return error.InvalidGraphMetricSegment; + var directory_checksum: [32]u8 = undefined; + @memcpy(&directory_checksum, try take(root, &pos, 32)); + const entries = try alloc.alloc(RankedRoutingEntry, blocks); + errdefer alloc.free(entries); + var next_offset = primary_end; + for (entries) |*entry| { + const offset = try readInt(u64, root, &pos); + const len = try readInt(u32, root, &pos); + if (offset != next_offset or len == 0 or len > max_ranked_score_block_bytes or len > footer_offset - next_offset) return error.InvalidGraphMetricSegment; + entry.* = .{ .offset = offset, .len = len, .checksum = undefined }; + @memcpy(&entry.checksum, try take(root, &pos, 32)); + next_offset += len; + } + if (next_offset != footer_offset or pos + routing_root_trailer_len != root.len) return error.InvalidGraphMetricSegment; + return .{ .entries = try alloc.alloc(RoutingEntry, 0), .ranked_entries = entries, .top_score_count = count, .footer_offset = footer_offset, .primary_data_offset = primary_start, .primary_data_end = primary_end, .point_index_checksum = point_checksum, .directory_len = directory_len, .directory_checksum = directory_checksum }; +} + +fn readRoutingEntry(bytes: []const u8, pos: *usize) !RoutingEntry { + const id_len = try readInt(u32, bytes, pos); + if (id_len == 0 or id_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const offset = try readInt(u64, bytes, pos); + const len = try readInt(u32, bytes, pos); + if (len == 0) return error.InvalidGraphMetricSegment; + var checksum: [32]u8 = undefined; + @memcpy(&checksum, try take(bytes, pos, 32)); + return .{ .first_node_id = try take(bytes, pos, id_len), .offset = offset, .len = len, .checksum = checksum }; +} + +/// Directory entries describe authenticated routing pages, not score blocks. +/// Their identifiers borrow bytes; the caller retains the directory payload. +pub fn decodePointDirectoryAlloc(alloc: Allocator, bytes: []const u8, directory_offset: u64, footer_offset: u64, block_count: usize, cancellation: CancellationToken) ![]RoutingEntry { + try cancellation.check(); + var pos: usize = 0; + if (!std.mem.eql(u8, try take(bytes, &pos, directory_magic.len), directory_magic)) return error.InvalidGraphMetricSegment; + const count = try readInt(u32, bytes, &pos); + if (count != scoreBlockCountForSize(block_count, routing_page_entries) or count > bytes.len / routing_entry_fixed_len) return error.InvalidGraphMetricSegment; + const entries = try alloc.alloc(RoutingEntry, count); + errdefer alloc.free(entries); + var next = std.math.add(u64, footer_offset, routing_header_len) catch return error.InvalidGraphMetricSegment; + for (entries, 0..) |*entry, i| { + try cancellation.check(); + entry.* = try readRoutingEntry(bytes, &pos); + entry.block_index = i * routing_page_entries; + if (entry.offset != next or next > directory_offset or entry.len > directory_offset - next or + entry.len > routing_page_entries * (routing_entry_fixed_len + max_score_node_id_bytes) or + (i > 0 and std.mem.order(u8, entries[i - 1].first_node_id, entry.first_node_id) != .lt)) return error.InvalidGraphMetricSegment; + next += entry.len; + } + if (pos != bytes.len or next != directory_offset) return error.InvalidGraphMetricSegment; + return entries; +} + +pub fn decodePointPageAlloc(alloc: Allocator, bytes: []const u8, page: RoutingEntry, block_count: usize, primary_start: u64, primary_end: u64, cancellation: CancellationToken) ![]RoutingEntry { + try cancellation.check(); + if (page.block_index >= block_count or bytes.len != page.len) return error.InvalidGraphMetricSegment; + const count = @min(routing_page_entries, block_count - page.block_index); + const entries = try alloc.alloc(RoutingEntry, count); + errdefer alloc.free(entries); + var pos: usize = 0; + var next: ?u64 = null; + for (entries, 0..) |*entry, i| { + try cancellation.check(); + entry.* = try readRoutingEntry(bytes, &pos); + entry.block_index = page.block_index + i; + if (entry.offset < primary_start or entry.offset > primary_end or entry.len > primary_end - entry.offset or + (next != null and entry.offset != next.?) or + (i > 0 and std.mem.order(u8, entries[i - 1].first_node_id, entry.first_node_id) != .lt)) return error.InvalidGraphMetricSegment; + next = entry.offset + entry.len; + } + if (pos != bytes.len or !std.mem.eql(u8, entries[0].first_node_id, page.first_node_id) or + (page.block_index == 0 and entries[0].offset != primary_start) or + (page.block_index + count == block_count and next.? != primary_end)) return error.InvalidGraphMetricSegment; + return entries; +} + +/// Validate the complete directory against the canonical flat page records. +/// Full artifact validation and warm-start reads use this allocation-free path. +fn validateDirectory(point: []const u8, directory_start: usize, footer_offset: u64, block_count: usize, checksum: [32]u8, cancellation: CancellationToken) !void { + var actual: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(point[directory_start..], &actual, .{}); + if (!std.mem.eql(u8, &actual, &checksum)) return error.InvalidGraphMetricSegment; + var pos = directory_start; + if (!std.mem.eql(u8, try take(point, &pos, directory_magic.len), directory_magic)) return error.InvalidGraphMetricSegment; + const page_count = scoreBlockCountForSize(block_count, routing_page_entries); + if (try readInt(u32, point, &pos) != page_count) return error.InvalidGraphMetricSegment; + var record_pos: usize = routing_header_len; + for (0..page_count) |page_index| { + try cancellation.check(); + const page = try readRoutingEntry(point, &pos); + const start = record_pos; + const first = try readRoutingEntry(point[0..directory_start], &record_pos); + const count = @min(routing_page_entries, block_count - page_index * routing_page_entries); + for (1..count) |_| _ = try readRoutingEntry(point[0..directory_start], &record_pos); + std.crypto.hash.sha2.Sha256.hash(point[start..record_pos], &actual, .{}); + if (page.offset != footer_offset + start or page.len != record_pos - start or + !std.mem.eql(u8, page.first_node_id, first.first_node_id) or !std.mem.eql(u8, &page.checksum, &actual)) return error.InvalidGraphMetricSegment; + } + if (pos != point.len or record_pos != directory_start) return error.InvalidGraphMetricSegment; +} + +pub fn artifactIntegrity(segment: types.Segment, payload: []const u8) !ArtifactIntegrity { + if (payload.len > std.math.maxInt(u32)) return error.GraphMetricSegmentTooLarge; + const control_len = try controlProbeLen( + payload.len, + segment.source_graph_artifact_id, + segment.source_graph_checksum, + segment.edge_filter, + ); + const control = try decodeControl(payload[0..control_len], segment.edge_filter); + if (control.score_data_offset != control_len) return error.InvalidGraphMetricSegment; + if (payload.len < routing_trailer_len) return error.InvalidGraphMetricSegment; + const routing_len = try routingFooterLenFromTrailer(payload.len, payload[payload.len - routing_trailer_len ..]); + var control_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload[0..control_len], &control_checksum, .{}); + var routing_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + const root_len = try routingRootLenFromTrailer(payload); + if (root_len != routingRootLen(control.score_count) or root_len > routing_len) return error.InvalidGraphMetricSegment; + std.crypto.hash.sha2.Sha256.hash(payload[payload.len - root_len ..], &routing_checksum, .{}); + var point_index_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload[payload.len - routing_len .. payload.len - root_len], &point_index_checksum, .{}); + return .{ + .control_len = @intCast(control_len), + .routing_footer_len = @intCast(routing_len), + .control_checksum = control_checksum, + .routing_checksum = routing_checksum, + .point_index_checksum = point_index_checksum, + }; +} + +pub fn decodeScoreBlockWithCancellation(block: []const u8, cancellation: CancellationToken) !DecodedScoreBlock { + var decoded = DecodedScoreBlock{}; + var pos: usize = 0; + const prefix_len = try readInt(u16, block, &pos); + if (prefix_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const prefix = try take(block, &pos, prefix_len); + decoded.node_prefix = prefix; + var previous_suffix: ?[]const u8 = null; + var score_index: usize = 0; + while (pos < block.len) : (score_index += 1) { + if (score_index >= score_block_entries) return error.InvalidGraphMetricSegment; + if (score_index % 256 == 0) try cancellation.check(); + const suffix_len = try readInt(u16, block, &pos); + const node_len = prefix.len + suffix_len; + if (node_len == 0 or node_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const value: f64 = @bitCast(try readInt(u64, block, &pos)); + if (!validMetricScore(value)) return error.InvalidGraphMetricSegment; + const suffix = try take(block, &pos, suffix_len); + if (previous_suffix != null and std.mem.order(u8, previous_suffix.?, suffix) != .lt) return error.InvalidGraphMetricSegment; + decoded.scores[score_index] = .{ .node_suffix = suffix, .value = value }; + previous_suffix = suffix; + } + if (score_index == 0) return error.InvalidGraphMetricSegment; + decoded.len = score_index; + try cancellation.check(); + return decoded; +} + +pub fn decodeRankedScoreBlockWithCancellation(block: []const u8, cancellation: CancellationToken) !DecodedScoreBlock { + var decoded = DecodedScoreBlock{}; + var pos: usize = 0; + const prefix_len = try readInt(u16, block, &pos); + if (prefix_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const prefix = try take(block, &pos, prefix_len); + decoded.node_prefix = prefix; + var score_index: usize = 0; + while (pos < block.len) : (score_index += 1) { + if (score_index >= ranked_score_block_entries) return error.InvalidGraphMetricSegment; + if (score_index % 64 == 0) try cancellation.check(); + const suffix_len = try readInt(u16, block, &pos); + const node_len = prefix.len + suffix_len; + if (node_len == 0 or node_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const value: f64 = @bitCast(try readInt(u64, block, &pos)); + if (!validMetricScore(value)) return error.InvalidGraphMetricSegment; + const suffix = try take(block, &pos, suffix_len); + if (score_index > 0) { + const previous = decoded.scores[score_index - 1]; + if (previous.value < value or + (previous.value == value and std.mem.order(u8, previous.node_suffix, suffix) != .lt)) + { + return error.InvalidGraphMetricSegment; + } + } + decoded.scores[score_index] = .{ .node_suffix = suffix, .value = value }; + } + if (score_index == 0) return error.InvalidGraphMetricSegment; + decoded.len = score_index; + try cancellation.check(); + return decoded; +} + +pub fn scoreFromBlockWithCancellation(block: []const u8, node_id: []const u8, cancellation: CancellationToken) !?f64 { + const decoded = try decodeScoreBlockWithCancellation(block, cancellation); + return decoded.score(node_id); +} + +/// Decodes only provenance and lifecycle metadata from a bounded prefix. Score +/// vectors and edge filters are deliberately not materialized. +pub fn decodeHeader(data: []const u8) !Header { + if (data.len < fixed_header_len) return error.InvalidGraphMetricSegment; + var pos: usize = 0; + if (!std.mem.eql(u8, try take(data, &pos, 4), wire_magic)) return error.InvalidGraphMetricSegment; + const version = try readInt(u16, data, &pos); + if (version != wire_version) return error.UnsupportedGraphMetricSegmentVersion; + if (pos >= data.len) return error.InvalidGraphMetricSegment; + const kind = std.enums.fromInt(@import("../../graph/graph.zig").GraphMetricKind, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const materialization_state = std.enums.fromInt(types.MaterializationState, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const rejection_reason = std.enums.fromInt(types.RejectionReason, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const converged_byte = (try take(data, &pos, 1))[0]; + if (converged_byte > 1) return error.InvalidGraphMetricSegment; + const iterations = try readInt(u32, data, &pos); + const fingerprint = try readInt(u64, data, &pos); + const materializer_fingerprint = try readInt(u64, data, &pos); + const topology_checksum = (try take(data, &pos, 32))[0..32].*; + const delta: f64 = @bitCast(try readInt(u64, data, &pos)); + if (!validMetricScore(delta)) return error.InvalidGraphMetricSegment; + switch (materialization_state) { + .ready => if (rejection_reason != .none) return error.InvalidGraphMetricSegment, + .rejected => if (rejection_reason == .none or converged_byte != 0 or iterations != 0 or delta != 0) return error.InvalidGraphMetricSegment, + } + const artifact_len = try readInt(u32, data, &pos); + const checksum_len = try readInt(u32, data, &pos); + const edge_type_count = try readInt(u32, data, &pos); + const source_graph_artifact_id = try take(data, &pos, artifact_len); + const source_graph_checksum = try take(data, &pos, checksum_len); + return .{ + .kind = kind, + .version = version, + .materialization_state = materialization_state, + .rejection_reason = rejection_reason, + .config_fingerprint = fingerprint, + .materializer_fingerprint = materializer_fingerprint, + .topology_checksum = topology_checksum, + .source_graph_artifact_id = source_graph_artifact_id, + .source_graph_checksum = source_graph_checksum, + .converged = converged_byte == 1, + .iterations_completed = iterations, + .delta = delta, + .edge_type_count = edge_type_count, + }; +} + +pub fn encodedSize(segment: types.Segment) !usize { + return encodedSizeWithCancellation(segment, .none); +} + +pub fn encodedSizeWithCancellation(segment: types.Segment, cancellation: CancellationToken) !usize { + const prepared = try prepareTopTier(segment.scores, cancellation); + return try encodedSizeWithPreparedTopTier(segment, &prepared, cancellation); +} + +fn encodedSizeWithPreparedTopTier(segment: types.Segment, prepared: *const PreparedTopTier, cancellation: CancellationToken) !usize { + try validateSegmentWithCancellation(segment, cancellation); + var size: usize = fixed_header_len; + // Logical index/metric names live in the manifest reference. Keeping them + // out of the content-addressed payload lets aliases and equivalently + // configured metrics share one immutable object. + size = std.math.add(usize, size, segment.source_graph_artifact_id.len) catch return error.GraphMetricSegmentTooLarge; + size = std.math.add(usize, size, segment.source_graph_checksum.len) catch return error.GraphMetricSegmentTooLarge; + for (segment.edge_filter.types) |edge_type| { + size = std.math.add(usize, size, 4 + edge_type.len) catch return error.GraphMetricSegmentTooLarge; + } + size = std.math.add(usize, size, @sizeOf(u32)) catch return error.GraphMetricSegmentTooLarge; + var score_start: usize = 0; + while (score_start < segment.scores.len) : (score_start += score_block_entries) { + try cancellation.check(); + const score_end = @min(segment.scores.len, score_start + score_block_entries); + size = std.math.add(usize, size, try scoreBlockEncodedSize(segment.scores[score_start..score_end])) catch + return error.GraphMetricSegmentTooLarge; + } + size = std.math.add(usize, size, prepared.payload_size) catch return error.GraphMetricSegmentTooLarge; + size = std.math.add(usize, size, try routingEncodedSize(segment.scores, prepared)) catch return error.GraphMetricSegmentTooLarge; + return size; +} + +pub fn encodeAlloc(alloc: Allocator, segment: types.Segment) ![]u8 { + return encodeAllocWithCancellation(alloc, segment, .none); +} + +pub fn encodeAllocWithCancellation(alloc: Allocator, segment: types.Segment, cancellation: CancellationToken) ![]u8 { + return try encodeAllocWithCancellationAndLimit(alloc, segment, cancellation, std.math.maxInt(usize)); +} + +pub fn encodeAllocWithCancellationAndLimit( + alloc: Allocator, + segment: types.Segment, + cancellation: CancellationToken, + max_encoded_bytes: usize, +) ![]u8 { + const plan = try prepareEncoding(segment, cancellation); + return encodePreparedAlloc(alloc, segment, cancellation, &plan, max_encoded_bytes); +} + +/// Prepared against an immutable segment; reuse only with that same segment. +/// Numeric top-tier selection and exact size calculation happen once, before +/// publication reserves output capacity or allocates an encoded payload. +pub const EncodingPlan = struct { top: PreparedTopTier, size: usize }; + +pub fn prepareEncoding(segment: types.Segment, cancellation: CancellationToken) !EncodingPlan { + const prepared = try prepareTopTier(segment.scores, cancellation); + const size = try encodedSizeWithPreparedTopTier(segment, &prepared, cancellation); + return .{ .top = prepared, .size = size }; +} + +pub fn encodePreparedAlloc(alloc: Allocator, segment: types.Segment, cancellation: CancellationToken, plan: *const EncodingPlan, max_encoded_bytes: usize) ![]u8 { + try cancellation.check(); + const prepared = plan.top; + const size = plan.size; + if (size > max_encoded_bytes) return error.GraphMetricSegmentTooLarge; + const data = try alloc.alloc(u8, size); + errdefer alloc.free(data); + var pos: usize = 0; + putBytes(data, &pos, wire_magic); + putInt(u16, data, &pos, wire_version); + data[pos] = @intFromEnum(segment.kind); + pos += 1; + data[pos] = @intFromEnum(segment.materialization_state); + pos += 1; + data[pos] = @intFromEnum(segment.rejection_reason); + pos += 1; + data[pos] = @intFromBool(segment.converged); + pos += 1; + putInt(u32, data, &pos, segment.iterations_completed); + putInt(u64, data, &pos, segment.config_fingerprint); + putInt(u64, data, &pos, segment.materializer_fingerprint); + putBytes(data, &pos, &segment.topology_checksum); + putInt(u64, data, &pos, @bitCast(segment.delta)); + putInt(u32, data, &pos, @intCast(segment.source_graph_artifact_id.len)); + putInt(u32, data, &pos, @intCast(segment.source_graph_checksum.len)); + putInt(u32, data, &pos, @intCast(segment.edge_filter.types.len)); + putBytes(data, &pos, segment.source_graph_artifact_id); + putBytes(data, &pos, segment.source_graph_checksum); + for (segment.edge_filter.types) |edge_type| { + putInt(u32, data, &pos, @intCast(edge_type.len)); + putBytes(data, &pos, edge_type); + } + putInt(u32, data, &pos, @intCast(segment.scores.len)); + const score_data_offset = pos; + var primary_start: usize = 0; + while (primary_start < segment.scores.len) : (primary_start += score_block_entries) { + try cancellation.check(); + const primary_end = @min(segment.scores.len, primary_start + score_block_entries); + const block_scores = segment.scores[primary_start..primary_end]; + const prefix_len = scoreBlockPrefixLen(block_scores); + putInt(u16, data, &pos, @intCast(prefix_len)); + putBytes(data, &pos, block_scores[0].node_id[0..prefix_len]); + for (block_scores) |score| { + const suffix = score.node_id[prefix_len..]; + putInt(u16, data, &pos, @intCast(suffix.len)); + putInt(u64, data, &pos, @bitCast(score.value)); + putBytes(data, &pos, suffix); + } + } + const score_data_end = pos; + var ranked_entries: [max_ranked_score_blocks]RankedRoutingEntry = undefined; + var ranked_block_index: usize = 0; + while (ranked_block_index < prepared.blockCount()) : (ranked_block_index += 1) { + try cancellation.check(); + const ranked_start = ranked_block_index * ranked_score_block_entries; + const ranked_end = @min(prepared.count, ranked_start + ranked_score_block_entries); + const block_offset = pos; + const block_indexes = prepared.indexes[ranked_start..ranked_end]; + const prefix_len = rankedBlockPrefixLen(segment.scores, block_indexes); + putInt(u16, data, &pos, @intCast(prefix_len)); + putBytes(data, &pos, segment.scores[block_indexes[0]].node_id[0..prefix_len]); + for (block_indexes) |top_index| { + const score = segment.scores[top_index]; + const suffix = score.node_id[prefix_len..]; + putInt(u16, data, &pos, @intCast(suffix.len)); + putInt(u64, data, &pos, @bitCast(score.value)); + putBytes(data, &pos, suffix); + } + const block_len = pos - block_offset; + var block_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[block_offset..pos], &block_checksum, .{}); + ranked_entries[ranked_block_index] = .{ + .offset = block_offset, + .len = block_len, + .checksum = block_checksum, + }; + } + + const footer_offset = pos; + putBytes(data, &pos, routing_magic); + const block_count = scoreBlockCount(segment.scores.len); + putInt(u32, data, &pos, @intCast(block_count)); + var block_offset = score_data_offset; + var score_index: usize = 0; + while (score_index < segment.scores.len) : (score_index += score_block_entries) { + try cancellation.check(); + const block_end = @min(segment.scores.len, score_index + score_block_entries); + const block_len = try scoreBlockEncodedSize(segment.scores[score_index..block_end]); + const first_node_id = segment.scores[score_index].node_id; + putInt(u32, data, &pos, @intCast(first_node_id.len)); + putInt(u64, data, &pos, @intCast(block_offset)); + putInt(u32, data, &pos, @intCast(block_len)); + var block_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[block_offset..][0..block_len], &block_checksum, .{}); + putBytes(data, &pos, &block_checksum); + putBytes(data, &pos, first_node_id); + block_offset += block_len; + } + std.debug.assert(block_offset == score_data_end); + const directory_offset = pos; + putBytes(data, &pos, directory_magic); + putInt(u32, data, &pos, @intCast(scoreBlockCountForSize(block_count, routing_page_entries))); + var page_start = footer_offset + routing_header_len; + var first_block: usize = 0; + while (first_block < block_count) : (first_block += routing_page_entries) { + try cancellation.check(); + var page_end = page_start; + for (first_block..@min(block_count, first_block + routing_page_entries)) |_| { + _ = try readRoutingEntry(data[0..directory_offset], &page_end); + } + const first_id = segment.scores[first_block * score_block_entries].node_id; + var checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[page_start..page_end], &checksum, .{}); + putInt(u32, data, &pos, @intCast(first_id.len)); + putInt(u64, data, &pos, @intCast(page_start)); + putInt(u32, data, &pos, @intCast(page_end - page_start)); + putBytes(data, &pos, &checksum); + putBytes(data, &pos, first_id); + page_start = page_end; + } + std.debug.assert(page_start == directory_offset); + const directory_len = pos - directory_offset; + var directory_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[directory_offset..pos], &directory_checksum, .{}); + var point_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[footer_offset..pos], &point_checksum, .{}); + const root_offset = pos; + putBytes(data, &pos, top_tier_magic); + putInt(u32, data, &pos, @intCast(prepared.count)); + putInt(u32, data, &pos, @intCast(prepared.blockCount())); + putInt(u64, data, &pos, @intCast(score_data_offset)); + putInt(u64, data, &pos, @intCast(score_data_end)); + putBytes(data, &pos, &point_checksum); + putInt(u64, data, &pos, @intCast(directory_len)); + putBytes(data, &pos, &directory_checksum); + for (ranked_entries[0..prepared.blockCount()]) |entry| { + putInt(u64, data, &pos, entry.offset); + putInt(u32, data, &pos, @intCast(entry.len)); + putBytes(data, &pos, &entry.checksum); + } + putInt(u32, data, &pos, @intCast(pos + routing_root_trailer_len - root_offset)); + putInt(u64, data, &pos, @intCast(pos + routing_trailer_len - footer_offset)); + std.debug.assert(pos == data.len); + try cancellation.check(); + return data; +} + +pub fn decodeAlloc(alloc: Allocator, data: []const u8) !types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, .{}, .none); +} + +pub fn decodeAllocWithLimits(alloc: Allocator, data: []const u8, limits: bounded_decode.Limits) !types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, limits, .none); +} + +pub fn decodeAllocWithCancellation(alloc: Allocator, data: []const u8, cancellation: CancellationToken) !types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, .{}, cancellation); +} + +pub fn decodeAllocWithLimitsAndCancellation( + alloc: Allocator, + data: []const u8, + limits: bounded_decode.Limits, + cancellation: CancellationToken, +) !types.Segment { + try cancellation.check(); + var budget = try bounded_decode.Budget.init(data.len, limits); + var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_allocation_bytes); + var decoded = decodeBoundedAlloc(limiter.allocator(), data, &budget, cancellation) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.DecodedArtifactTooLarge; + return err; + }; + errdefer decoded.deinit(alloc); + try cancellation.check(); + return decoded; +} + +fn decodeBoundedAlloc(alloc: Allocator, data: []const u8, budget: *bounded_decode.Budget, cancellation: CancellationToken) !types.Segment { + if (data.len < fixed_header_len + @sizeOf(u32)) return error.InvalidGraphMetricSegment; + var pos: usize = 0; + if (!std.mem.eql(u8, take(data, &pos, 4) catch return error.InvalidGraphMetricSegment, wire_magic)) return error.InvalidGraphMetricSegment; + const version = readInt(u16, data, &pos) catch return error.InvalidGraphMetricSegment; + if (version != wire_version) return error.UnsupportedGraphMetricSegmentVersion; + const kind = std.enums.fromInt(@import("../../graph/graph.zig").GraphMetricKind, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const materialization_state = std.enums.fromInt(types.MaterializationState, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const rejection_reason = std.enums.fromInt(types.RejectionReason, data[pos]) orelse return error.InvalidGraphMetricSegment; + pos += 1; + const converged_byte = data[pos]; + pos += 1; + if (converged_byte > 1) return error.InvalidGraphMetricSegment; + const iterations = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + const fingerprint = readInt(u64, data, &pos) catch return error.InvalidGraphMetricSegment; + const materializer_fingerprint = readInt(u64, data, &pos) catch return error.InvalidGraphMetricSegment; + const topology_checksum = (try take(data, &pos, 32))[0..32].*; + const delta: f64 = @bitCast(readInt(u64, data, &pos) catch return error.InvalidGraphMetricSegment); + if (!validMetricScore(delta)) return error.InvalidGraphMetricSegment; + const artifact_len = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + const checksum_len = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + const edge_type_count = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + const identity_len = std.math.add(usize, artifact_len, checksum_len) catch return error.InvalidGraphMetricSegment; + try budget.admitBytes(identity_len); + const artifact_id = try dupeTake(alloc, data, &pos, artifact_len); + errdefer alloc.free(artifact_id); + const checksum = try dupeTake(alloc, data, &pos, checksum_len); + errdefer alloc.free(checksum); + _ = try budget.admitCount([]const u8, edge_type_count, data.len - pos, 4); + const edge_types = try alloc.alloc([]const u8, edge_type_count); + errdefer alloc.free(edge_types); + var initialized_edge_types: usize = 0; + errdefer for (edge_types[0..initialized_edge_types]) |edge_type| alloc.free(edge_type); + for (edge_types, 0..) |*edge_type, edge_type_index| { + if (edge_type_index % 16 == 0) try cancellation.check(); + const edge_type_len = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + try budget.admitBytes(edge_type_len); + edge_type.* = try dupeTake(alloc, data, &pos, edge_type_len); + initialized_edge_types += 1; + } + const score_count = readInt(u32, data, &pos) catch return error.InvalidGraphMetricSegment; + _ = try budget.admitCount(types.Score, score_count, data.len - pos, ranked_score_fixed_len); + const scores = try alloc.alloc(types.Score, score_count); + errdefer alloc.free(scores); + var initialized: usize = 0; + errdefer for (scores[0..initialized]) |*score| score.deinit(alloc); + var block_prefix: []const u8 = &.{}; + for (scores, 0..) |*score, score_index| { + if (score_index % 4096 == 0) try cancellation.check(); + if (score_index % score_block_entries == 0) { + const prefix_len = readInt(u16, data, &pos) catch return error.InvalidGraphMetricSegment; + if (prefix_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + block_prefix = take(data, &pos, prefix_len) catch return error.InvalidGraphMetricSegment; + } + const suffix_len = readInt(u16, data, &pos) catch return error.InvalidGraphMetricSegment; + const node_len = std.math.add(usize, block_prefix.len, suffix_len) catch return error.InvalidGraphMetricSegment; + if (node_len == 0 or node_len > max_score_node_id_bytes) return error.InvalidGraphMetricSegment; + const value: f64 = @bitCast(readInt(u64, data, &pos) catch return error.InvalidGraphMetricSegment); + if (!validMetricScore(value)) return error.InvalidGraphMetricSegment; + try budget.admitBytes(node_len); + const suffix = take(data, &pos, suffix_len) catch return error.InvalidGraphMetricSegment; + const node_id = try alloc.alloc(u8, node_len); + errdefer alloc.free(node_id); + @memcpy(node_id[0..block_prefix.len], block_prefix); + @memcpy(node_id[block_prefix.len..], suffix); + score.* = .{ .node_id = node_id, .value = value }; + initialized += 1; + } + try validateRoutingFooter(data, pos, scores, cancellation); + var segment = types.Segment{ .metadata_version = version, .kind = kind, .source_graph_artifact_id = artifact_id, .source_graph_checksum = checksum, .config_fingerprint = fingerprint, .materializer_fingerprint = materializer_fingerprint, .topology_checksum = topology_checksum, .materialization_state = materialization_state, .rejection_reason = rejection_reason, .edge_filter = .{ .mode = if (edge_type_count == 0) .all else .types, .types = edge_types }, .converged = converged_byte == 1, .iterations_completed = iterations, .delta = delta, .scores = scores }; + errdefer segment.deinit(alloc); + try validateSegmentWithCancellation(segment, cancellation); + return segment; +} + +fn validateSegment(segment: types.Segment) !void { + return validateSegmentWithCancellation(segment, .none); +} + +fn validateSegmentWithCancellation(segment: types.Segment, cancellation: CancellationToken) !void { + try cancellation.check(); + if (segment.source_graph_artifact_id.len == 0 or segment.source_graph_checksum.len == 0) return error.InvalidGraphMetricSegment; + _ = std.math.cast(u32, segment.source_graph_artifact_id.len) orelse return error.GraphMetricSegmentTooLarge; + _ = std.math.cast(u32, segment.source_graph_checksum.len) orelse return error.GraphMetricSegmentTooLarge; + _ = std.math.cast(u32, segment.scores.len) orelse return error.GraphMetricSegmentTooLarge; + if (!validMetricScore(segment.delta)) return error.InvalidGraphMetricSegment; + switch (segment.materialization_state) { + .ready => if (segment.rejection_reason != .none) return error.InvalidGraphMetricSegment, + .rejected => { + if (segment.rejection_reason == .none or segment.scores.len != 0 or segment.converged or segment.iterations_completed != 0 or segment.delta != 0) return error.InvalidGraphMetricSegment; + }, + } + if ((segment.edge_filter.mode == .all) != (segment.edge_filter.types.len == 0)) return error.InvalidGraphMetricSegment; + for (segment.edge_filter.types, 0..) |edge_type, i| { + if (edge_type.len == 0) return error.InvalidGraphMetricSegment; + _ = std.math.cast(u32, edge_type.len) orelse return error.GraphMetricSegmentTooLarge; + if (i > 0 and std.mem.order(u8, segment.edge_filter.types[i - 1], edge_type) != .lt) return error.InvalidGraphMetricSegment; + } + for (segment.scores, 0..) |score, i| { + if (i % 4096 == 0) try cancellation.check(); + if (score.node_id.len == 0 or !validMetricScore(score.value)) return error.InvalidGraphMetricSegment; + if (score.node_id.len > max_score_node_id_bytes) return error.GraphMetricSegmentTooLarge; + _ = std.math.cast(u32, score.node_id.len) orelse return error.GraphMetricSegmentTooLarge; + if (i > 0 and std.mem.order(u8, segment.scores[i - 1].node_id, score.node_id) != .lt) return error.InvalidGraphMetricSegment; + } + try cancellation.check(); +} + +fn scoreBlockCount(score_count: usize) usize { + return scoreBlockCountForSize(score_count, score_block_entries); +} + +fn scoreBlockCountForSize(score_count: usize, block_entries: usize) usize { + return score_count / block_entries + @intFromBool(score_count % block_entries != 0); +} + +fn commonPrefixLen(left: []const u8, right: []const u8) usize { + const end = @min(left.len, right.len); + var index: usize = 0; + while (index < end and left[index] == right[index]) : (index += 1) {} + return index; +} + +fn validMetricScore(value: f64) bool { + // Reject negative zero as well as negative/NaN/infinite values. This keeps + // canonical encodings unique and makes unsigned IEEE-754 radix order match + // numeric order exactly. + const bits: u64 = @bitCast(value); + return std.math.isFinite(value) and bits & (@as(u64, 1) << 63) == 0; +} + +fn scoreBlockPrefixLen(scores: []const types.Score) usize { + if (scores.len == 0) return 0; + // Primary blocks are node-sorted, so the first/last common prefix is the + // prefix shared by the complete block. + return commonPrefixLen(scores[0].node_id, scores[scores.len - 1].node_id); +} + +fn rankedBlockPrefixLen(scores: []const types.Score, indexes: []const usize) usize { + if (indexes.len == 0) return 0; + var prefix_len = scores[indexes[0]].node_id.len; + for (indexes[1..]) |index| { + prefix_len = @min(prefix_len, commonPrefixLen(scores[indexes[0]].node_id[0..prefix_len], scores[index].node_id)); + if (prefix_len == 0) break; + } + return prefix_len; +} + +fn scoreBlockEncodedSize(scores: []const types.Score) !usize { + const prefix_len = scoreBlockPrefixLen(scores); + var size = std.math.add(usize, @sizeOf(u16), prefix_len) catch return error.GraphMetricSegmentTooLarge; + for (scores) |score| { + size = std.math.add(usize, size, ranked_score_fixed_len + score.node_id.len - prefix_len) catch + return error.GraphMetricSegmentTooLarge; + } + return size; +} + +fn rankedBlockEncodedSize(scores: []const types.Score, indexes: []const usize) !usize { + const prefix_len = rankedBlockPrefixLen(scores, indexes); + var size = std.math.add(usize, @sizeOf(u16), prefix_len) catch return error.GraphMetricSegmentTooLarge; + for (indexes) |index| { + size = std.math.add(usize, size, ranked_score_fixed_len + scores[index].node_id.len - prefix_len) catch + return error.GraphMetricSegmentTooLarge; + } + return size; +} + +fn topScoreRanksBefore(a: PersistedTopScore, b: PersistedTopScore) bool { + if (a.value != b.value) return a.value > b.value; + return std.mem.order(u8, a.node_id, b.node_id) == .lt; +} + +fn scoreIndexRanksBefore(scores: []const types.Score, a: usize, b: usize) bool { + return topScoreRanksBefore( + .{ .node_id = scores[a].node_id, .value = scores[a].value }, + .{ .node_id = scores[b].node_id, .value = scores[b].value }, + ); +} + +fn siftWorstScoreUp(scores: []const types.Score, heap: []usize, start: usize) void { + var child = start; + while (child > 0) { + const parent = (child - 1) / 2; + if (!scoreIndexRanksBefore(scores, heap[parent], heap[child])) break; + std.mem.swap(usize, &heap[parent], &heap[child]); + child = parent; + } +} + +fn siftWorstScoreDown(scores: []const types.Score, heap: []usize, start: usize) void { + var parent = start; + while (true) { + const left = parent * 2 + 1; + if (left >= heap.len) return; + const right = left + 1; + var worse_child = left; + if (right < heap.len and scoreIndexRanksBefore(scores, heap[left], heap[right])) worse_child = right; + if (!scoreIndexRanksBefore(scores, heap[parent], heap[worse_child])) return; + std.mem.swap(usize, &heap[parent], &heap[worse_child]); + parent = worse_child; + } +} + +fn selectTopScoreIndexes( + scores: []const types.Score, + storage: *[max_persisted_top_entries]usize, + cancellation: CancellationToken, +) ![]usize { + const target = @min(scores.len, max_persisted_top_entries); + // Express the ratio without multiplication so this admission check stays + // overflow-safe if the persisted tier limit becomes configurable later. + if (target >= 512 and scores.len > target and scores.len - target > target) { + return try selectTopScoreIndexesRadix(scores, storage, target, cancellation); + } + var heap_len: usize = 0; + for (scores, 0..) |_, score_index| { + if (score_index % 4096 == 0) try cancellation.check(); + if (heap_len < target) { + storage[heap_len] = score_index; + siftWorstScoreUp(scores, storage[0 .. heap_len + 1], heap_len); + heap_len += 1; + } else if (target > 0 and scoreIndexRanksBefore(scores, score_index, storage[0])) { + storage[0] = score_index; + siftWorstScoreDown(scores, storage[0..heap_len], 0); + } + } + var remaining = heap_len; + while (remaining > 1) { + if (remaining % 256 == 0) try cancellation.check(); + std.mem.swap(usize, &storage[0], &storage[remaining - 1]); + remaining -= 1; + siftWorstScoreDown(scores, storage[0..remaining], 0); + } + return storage[0..heap_len]; +} + +fn scoreIndexLessThan(scores: []const types.Score, left: usize, right: usize) bool { + return scoreIndexRanksBefore(scores, left, right); +} + +/// Select a large persisted prefix by IEEE-754 radix instead of paying +/// O(N log K) heap comparisons. Graph metric scores are finite and +/// non-negative, so their bit representation preserves numeric ordering. +/// Equal-score membership remains deterministic because the input vector is +/// node-sorted and the final K indexes use the canonical score/node ordering. +fn selectTopScoreIndexesRadix( + scores: []const types.Score, + storage: *[max_persisted_top_entries]usize, + target: usize, + cancellation: CancellationToken, +) ![]usize { + std.debug.assert(target > 0 and target <= storage.len and target < scores.len); + var prefix: u64 = 0; + var prefix_mask: u64 = 0; + var rank = target - 1; + for (0..8) |pass| { + var counts: [256]usize = @splat(0); + const shift: u6 = @intCast(56 - pass * 8); + for (scores, 0..) |score, score_index| { + if (score_index % 4096 == 0) try cancellation.check(); + const bits: u64 = @bitCast(score.value); + if (bits & prefix_mask != prefix) continue; + counts[@as(u8, @truncate(bits >> shift))] += 1; + } + var bucket: usize = counts.len; + while (bucket > 0) { + bucket -= 1; + if (rank < counts[bucket]) break; + rank -= counts[bucket]; + } + prefix |= @as(u64, @intCast(bucket)) << shift; + prefix_mask |= @as(u64, 0xff) << shift; + } + const threshold: f64 = @bitCast(prefix); + var selected: usize = 0; + for (scores, 0..) |score, score_index| { + if (score_index % 4096 == 0) try cancellation.check(); + if (score.value > threshold) { + if (selected >= target) return error.InvalidGraphMetricSegment; + storage[selected] = score_index; + selected += 1; + } + } + for (scores, 0..) |score, score_index| { + if (selected == target) break; + if (score_index % 4096 == 0) try cancellation.check(); + if (score.value == threshold) { + storage[selected] = score_index; + selected += 1; + } + } + if (selected != target) return error.InvalidGraphMetricSegment; + std.mem.sort(usize, storage[0..selected], scores, scoreIndexLessThan); + return storage[0..selected]; +} + +const PreparedTopTier = struct { + indexes: [max_persisted_top_entries]usize = undefined, + count: usize = 0, + payload_size: usize = 0, + + fn blockCount(self: *const PreparedTopTier) usize { + return scoreBlockCountForSize(self.count, ranked_score_block_entries); + } +}; + +fn prepareTopTier(scores: []const types.Score, cancellation: CancellationToken) !PreparedTopTier { + var prepared = PreparedTopTier{}; + const selected = try selectTopScoreIndexes(scores, &prepared.indexes, cancellation); + prepared.count = selected.len; + var ranked_start: usize = 0; + while (ranked_start < selected.len) : (ranked_start += ranked_score_block_entries) { + try cancellation.check(); + const ranked_end = @min(selected.len, ranked_start + ranked_score_block_entries); + prepared.payload_size = std.math.add( + usize, + prepared.payload_size, + try rankedBlockEncodedSize(scores, selected[ranked_start..ranked_end]), + ) catch return error.GraphMetricSegmentTooLarge; + } + return prepared; +} + +fn routingEncodedSize(scores: []const types.Score, prepared: *const PreparedTopTier) !usize { + var size: usize = 2 * routing_header_len + top_tier_header_len + routing_root_metadata_len + routing_root_trailer_len; + var score_index: usize = 0; + while (score_index < scores.len) : (score_index += score_block_entries) { + size = std.math.add(usize, size, routing_entry_fixed_len) catch return error.GraphMetricSegmentTooLarge; + size = std.math.add(usize, size, scores[score_index].node_id.len) catch return error.GraphMetricSegmentTooLarge; + if (score_index / score_block_entries % routing_page_entries == 0) { + size = std.math.add(usize, size, routing_entry_fixed_len + scores[score_index].node_id.len) catch return error.GraphMetricSegmentTooLarge; + } + } + size = std.math.add(usize, size, std.math.mul(usize, prepared.blockCount(), ranked_routing_entry_len) catch return error.GraphMetricSegmentTooLarge) catch return error.GraphMetricSegmentTooLarge; + if (size > max_routing_bytes) return error.GraphMetricSegmentTooLarge; + return size; +} + +fn validateRoutingFooter(data: []const u8, score_data_end: usize, scores: []const types.Score, cancellation: CancellationToken) !void { + try cancellation.check(); + if (score_data_end > data.len or data.len < routing_trailer_len) return error.InvalidGraphMetricSegment; + const footer_len = try routingFooterLenFromTrailer(data.len, data[data.len - routing_trailer_len ..]); + const footer_offset = data.len - footer_len; + if (score_data_end > footer_offset) return error.InvalidGraphMetricSegment; + const footer = data[footer_offset..]; + var pos: usize = 0; + if (!std.mem.eql(u8, try take(footer, &pos, routing_magic.len), routing_magic)) return error.InvalidGraphMetricSegment; + if (try readInt(u32, footer, &pos) != scoreBlockCount(scores.len)) return error.InvalidGraphMetricSegment; + var expected_offset = score_data_end; + for (0..scoreBlockCount(scores.len)) |block_index| { + try cancellation.check(); + const score_start = block_index * score_block_entries; + const score_end = @min(scores.len, score_start + score_block_entries); + const block_len = try scoreBlockEncodedSize(scores[score_start..score_end]); + expected_offset -= block_len; + } + const primary_start = expected_offset; + for (0..scoreBlockCount(scores.len)) |block_index| { + try cancellation.check(); + const score_start = block_index * score_block_entries; + const score_end = @min(scores.len, score_start + score_block_entries); + const block_len = try scoreBlockEncodedSize(scores[score_start..score_end]); + const first_len = try readInt(u32, footer, &pos); + const offset = try readInt(u64, footer, &pos); + const encoded_block_len = try readInt(u32, footer, &pos); + var encoded_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + @memcpy(&encoded_checksum, try take(footer, &pos, encoded_checksum.len)); + const first = try take(footer, &pos, first_len); + if (!std.mem.eql(u8, first, scores[score_start].node_id) or + offset != expected_offset or encoded_block_len != block_len) + { + return error.InvalidGraphMetricSegment; + } + var actual_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + const block_offset = std.math.cast(usize, offset) orelse return error.InvalidGraphMetricSegment; + if (block_offset > data.len or block_len > data.len - block_offset) return error.InvalidGraphMetricSegment; + std.crypto.hash.sha2.Sha256.hash(data[block_offset..][0..block_len], &actual_checksum, .{}); + if (!std.mem.eql(u8, &actual_checksum, &encoded_checksum)) return error.InvalidGraphMetricSegment; + expected_offset += block_len; + } + if (expected_offset != score_data_end) return error.InvalidGraphMetricSegment; + const directory_start = pos; + if (!std.mem.eql(u8, try take(footer, &pos, directory_magic.len), directory_magic)) return error.InvalidGraphMetricSegment; + const page_count = scoreBlockCountForSize(scoreBlockCount(scores.len), routing_page_entries); + if (try readInt(u32, footer, &pos) != page_count) return error.InvalidGraphMetricSegment; + for (0..page_count) |_| _ = try readRoutingEntry(footer, &pos); + const directory_len = pos - directory_start; + var directory_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(footer[directory_start..pos], &directory_checksum, .{}); + try validateDirectory(footer[0..pos], directory_start, footer_offset, scoreBlockCount(scores.len), directory_checksum, cancellation); + { + var point_checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(footer[0..pos], &point_checksum, .{}); + if (!std.mem.eql(u8, try take(footer, &pos, top_tier_magic.len), top_tier_magic)) return error.InvalidGraphMetricSegment; + var top_storage: [max_persisted_top_entries]usize = undefined; + const expected_top = try selectTopScoreIndexes(scores, &top_storage, cancellation); + if (try readInt(u32, footer, &pos) != expected_top.len) return error.InvalidGraphMetricSegment; + const block_count = try readInt(u32, footer, &pos); + if (block_count != scoreBlockCountForSize(expected_top.len, ranked_score_block_entries)) return error.InvalidGraphMetricSegment; + if (try readInt(u64, footer, &pos) != primary_start or try readInt(u64, footer, &pos) != score_data_end or + !std.mem.eql(u8, try take(footer, &pos, 32), &point_checksum)) return error.InvalidGraphMetricSegment; + if (try readInt(u64, footer, &pos) != directory_len or + !std.mem.eql(u8, try take(footer, &pos, 32), &directory_checksum)) return error.InvalidGraphMetricSegment; + var ranked_offset = score_data_end; + for (0..block_count) |block_index| { + try cancellation.check(); + const offset = try readInt(u64, footer, &pos); + const block_len = try readInt(u32, footer, &pos); + var encoded_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + @memcpy(&encoded_checksum, try take(footer, &pos, encoded_checksum.len)); + if (offset != ranked_offset or block_len == 0 or block_len > max_ranked_score_block_bytes) return error.InvalidGraphMetricSegment; + const block_offset = std.math.cast(usize, offset) orelse return error.InvalidGraphMetricSegment; + if (block_offset > footer_offset or block_len > footer_offset - block_offset) return error.InvalidGraphMetricSegment; + const block = data[block_offset..][0..block_len]; + var actual_checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(block, &actual_checksum, .{}); + if (!std.mem.eql(u8, &actual_checksum, &encoded_checksum)) return error.InvalidGraphMetricSegment; + const decoded = try decodeRankedScoreBlockWithCancellation(block, cancellation); + const ranked_start = block_index * ranked_score_block_entries; + const ranked_end = @min(expected_top.len, ranked_start + ranked_score_block_entries); + if (decoded.len != ranked_end - ranked_start) return error.InvalidGraphMetricSegment; + for (decoded.scores[0..decoded.len], expected_top[ranked_start..ranked_end]) |actual, score_index| { + const expected = scores[score_index]; + if (!actual.eqlNode(decoded.node_prefix, expected.node_id) or @as(u64, @bitCast(actual.value)) != @as(u64, @bitCast(expected.value))) { + return error.InvalidGraphMetricSegment; + } + } + ranked_offset = std.math.add(usize, block_offset, block_len) catch return error.InvalidGraphMetricSegment; + } + if (ranked_offset != footer_offset) return error.InvalidGraphMetricSegment; + } + if (try readInt(u32, footer, &pos) != routingRootLen(scores.len) or pos + routing_trailer_len != footer.len) return error.InvalidGraphMetricSegment; + try cancellation.check(); +} + +test "serverless graph metric large top tier uses exact radix selection" { + const alloc = std.testing.allocator; + const score_count = max_persisted_top_entries * 2 + 1; + const scores = try alloc.alloc(types.Score, score_count); + defer alloc.free(scores); + for (scores, 0..) |*score, index| score.* = .{ + .node_id = @constCast("node"), + .value = @floatFromInt(index), + }; + var storage: [max_persisted_top_entries]usize = undefined; + const selected = try selectTopScoreIndexes(scores, &storage, .none); + try std.testing.expectEqual(max_persisted_top_entries, selected.len); + try std.testing.expectEqual(score_count - 1, selected[0]); + try std.testing.expectEqual(score_count - max_persisted_top_entries, selected[selected.len - 1]); +} + +test "serverless graph metric score blocks compress shared node prefixes" { + const scores = [_]types.Score{ + .{ .node_id = @constCast("tenant:west:node:0001"), .value = 1 }, + .{ .node_id = @constCast("tenant:west:node:0002"), .value = 2 }, + .{ .node_id = @constCast("tenant:west:node:0003"), .value = 3 }, + }; + var uncompressed_size: usize = 0; + for (scores) |score| uncompressed_size += @sizeOf(u32) + @sizeOf(u64) + score.node_id.len; + try std.testing.expect(try scoreBlockEncodedSize(&scores) < uncompressed_size); +} + +test "serverless graph metric canonical scores reject negative zero" { + const scores = [_]types.Score{.{ .node_id = @constCast("node"), .value = -0.0 }}; + const segment = types.Segment{ + .kind = .pagerank, + .source_graph_artifact_id = @constCast("sha256:graph"), + .source_graph_checksum = @constCast("sha256:sum"), + .config_fingerprint = 1, + .edge_filter = .{}, + .converged = true, + .iterations_completed = 1, + .delta = 0, + .scores = @constCast(&scores), + }; + try std.testing.expectError(error.InvalidGraphMetricSegment, encodedSize(segment)); + + const valid_scores = [_]types.Score{.{ .node_id = @constCast("node"), .value = 0.0 }}; + var noncanonical_delta = segment; + noncanonical_delta.scores = @constCast(&valid_scores); + noncanonical_delta.delta = -0.0; + try std.testing.expectError(error.InvalidGraphMetricSegment, encodedSize(noncanonical_delta)); +} + +fn putBytes(data: []u8, pos: *usize, value: []const u8) void { + @memcpy(data[pos.*..][0..value.len], value); + pos.* += value.len; +} +fn putInt(comptime T: type, data: []u8, pos: *usize, value: T) void { + std.mem.writeInt(T, data[pos.*..][0..@sizeOf(T)], value, .little); + pos.* += @sizeOf(T); +} +fn readInt(comptime T: type, data: []const u8, pos: *usize) !T { + if (pos.* + @sizeOf(T) > data.len) return error.InvalidGraphMetricSegment; + const value = std.mem.readInt(T, data[pos.*..][0..@sizeOf(T)], .little); + pos.* += @sizeOf(T); + return value; +} +fn take(data: []const u8, pos: *usize, len: usize) ![]const u8 { + if (pos.* > data.len or len > data.len - pos.*) return error.InvalidGraphMetricSegment; + const value = data[pos.*..][0..len]; + pos.* += len; + return value; +} +fn dupeTake(alloc: Allocator, data: []const u8, pos: *usize, len: usize) ![]u8 { + return try alloc.dupe(u8, try take(data, pos, len)); +} + +test "serverless graph metric segment round trips with binary-search lookup" { + const alloc = std.testing.allocator; + var scores = try alloc.alloc(types.Score, 2); + scores[0] = .{ .node_id = try alloc.dupe(u8, "a"), .value = 0.25 }; + scores[1] = .{ .node_id = try alloc.dupe(u8, "b"), .value = 0.75 }; + var segment = types.Segment{ .kind = .pagerank, .source_graph_artifact_id = try alloc.dupe(u8, "sha256:graph"), .source_graph_checksum = try alloc.dupe(u8, "sha256:sum"), .config_fingerprint = 42, .edge_filter = .{}, .converged = true, .iterations_completed = 12, .delta = 0.00001, .scores = scores }; + segment.topology_checksum = @splat(0x55); + defer segment.deinit(alloc); + const encoded = try encodeAlloc(alloc, segment); + defer alloc.free(encoded); + const plan = try prepareEncoding(segment, .none); + const prepared = try encodePreparedAlloc(alloc, segment, .none, &plan, plan.size); + defer alloc.free(prepared); + try std.testing.expectEqualSlices(u8, encoded, prepared); + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.GraphMetricSegmentTooLarge, encodePreparedAlloc(failing.allocator(), segment, .none, &plan, plan.size - 1)); + var canceled = std.atomic.Value(bool).init(true); + try std.testing.expectError(error.Canceled, encodePreparedAlloc(failing.allocator(), segment, CancellationToken.fromAtomic(&canceled), &plan, plan.size)); + try std.testing.expect(!failing.has_induced_failure); + const header_len = try headerProbeLen( + encoded.len, + segment.source_graph_artifact_id, + segment.source_graph_checksum, + ); + try std.testing.expectEqual( + fixed_header_len + segment.source_graph_artifact_id.len + segment.source_graph_checksum.len, + header_len, + ); + const header = try decodeHeader(encoded[0..header_len]); + try std.testing.expectEqual(graph_mod.GraphMetricKind.pagerank, header.kind); + try std.testing.expectEqualStrings(segment.source_graph_artifact_id, header.source_graph_artifact_id); + try std.testing.expectEqualSlices(u8, &segment.topology_checksum, &header.topology_checksum); + var decoded = try decodeAlloc(alloc, encoded); + defer decoded.deinit(alloc); + try std.testing.expectEqual(@as(?f64, 0.75), decoded.score("b")); + try std.testing.expectEqualSlices(u8, &segment.topology_checksum, &decoded.topology_checksum); + try std.testing.expect(decoded.score("missing") == null); + const root_len = routingRootLen(scores.len); + var root = try decodeRoutingRootAlloc(alloc, encoded[encoded.len - root_len ..], encoded.len, wire_version, .none); + defer root.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), root.entries.len); + try std.testing.expectEqual(@as(usize, 1), root.ranked_entries.len); + try std.testing.expectEqual(@as(usize, 1872), routingRootLen(1_000_000)); + try std.testing.expectEqual(routingRootLen(max_persisted_top_entries), routingRootLen(std.math.maxInt(usize))); + try std.testing.expectError(error.InvalidGraphMetricSegment, decodeRoutingRootAlloc(alloc, encoded[encoded.len - root_len + 1 ..], encoded.len, wire_version, .none)); + const integrity = try artifactIntegrity(segment, encoded); + try std.testing.expectEqualSlices(u8, &integrity.point_index_checksum, &root.point_index_checksum); + const footer = encoded[encoded.len - integrity.routing_footer_len ..]; + footer[0] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricSegment, decodeRoutingIndexAlloc(alloc, footer, encoded.len)); + footer[0] ^= 1; +} + +test "serverless graph metric routing resolves exact scores without decoding the vector" { + const alloc = std.testing.allocator; + var scores = try alloc.alloc(types.Score, score_block_entries + 1); + var initialized: usize = 0; + var scores_transferred = false; + errdefer if (!scores_transferred) { + for (scores[0..initialized]) |*score| score.deinit(alloc); + alloc.free(scores); + }; + for (scores, 0..) |*score, i| { + score.* = .{ + .node_id = try std.fmt.allocPrint(alloc, "node:{d:0>4}", .{i}), + .value = @floatFromInt(i), + }; + initialized += 1; + } + var segment = types.Segment{ + .kind = .pagerank, + .source_graph_artifact_id = try alloc.dupe(u8, "sha256:graph"), + .source_graph_checksum = try alloc.dupe(u8, "sha256:sum"), + .config_fingerprint = 42, + .edge_filter = .{}, + .converged = true, + .iterations_completed = 12, + .delta = 0.00001, + .scores = scores, + }; + scores_transferred = true; + defer segment.deinit(alloc); + const encoded = try encodeAlloc(alloc, segment); + defer alloc.free(encoded); + const control_len = try controlProbeLen(encoded.len, "sha256:graph", "sha256:sum", .{}); + const control = try decodeControl(encoded[0..control_len], .{}); + try std.testing.expectEqual(@as(u32, score_block_entries + 1), control.score_count); + + const footer_len = try routingFooterLenFromTrailer(encoded.len, encoded[encoded.len - routing_trailer_len ..]); + const footer_offset = encoded.len - footer_len; + var routing = try decodeRoutingIndexAlloc(alloc, encoded[footer_offset..], encoded.len); + defer routing.deinit(alloc); + try std.testing.expectEqual(@as(usize, 2), routing.entries.len); + try std.testing.expectEqual(score_block_entries + 1, routing.top_score_count); + try std.testing.expectEqual(@as(usize, 5), routing.ranked_entries.len); + const first_ranked_entry = routing.ranked_entries[0]; + const first_ranked = try decodeRankedScoreBlockWithCancellation( + encoded[@intCast(first_ranked_entry.offset)..][0..first_ranked_entry.len], + .none, + ); + try std.testing.expectEqual(@as(usize, ranked_score_block_entries), first_ranked.len); + try std.testing.expect(first_ranked.scores[0].eqlNode(first_ranked.node_prefix, "node:1024")); + try std.testing.expectEqual(@as(f64, 1024), first_ranked.scores[0].value); + try std.testing.expect(first_ranked.scores[first_ranked.len - 1].eqlNode(first_ranked.node_prefix, "node:0769")); + try std.testing.expectEqual(control.score_data_offset, routing.entries[0].offset); + const entry = routing.find("node:1024").?; + try std.testing.expectEqual(@as(?f64, 1024), try scoreFromBlockWithCancellation( + encoded[@intCast(entry.offset)..][0..entry.len], + "node:1024", + .none, + )); + try std.testing.expectEqual(@as(?f64, null), try scoreFromBlockWithCancellation( + encoded[@intCast(entry.offset)..][0..entry.len], + "node:missing", + .none, + )); + + const corrupted = try alloc.dupe(u8, encoded); + defer alloc.free(corrupted); + corrupted[@as(usize, @intCast(entry.offset)) + @sizeOf(u32)] ^= 0x01; + try std.testing.expectError(error.InvalidGraphMetricSegment, decodeAlloc(alloc, corrupted)); + + const corrupted_top = try alloc.dupe(u8, encoded); + defer alloc.free(corrupted_top); + corrupted_top[@as(usize, @intCast(first_ranked_entry.offset)) + @sizeOf(u32)] ^= 0x01; + try std.testing.expectError(error.InvalidGraphMetricSegment, decodeAlloc(alloc, corrupted_top)); +} + +test "serverless graph metric segment round trips terminal materialization rejection" { + const alloc = std.testing.allocator; + var segment = types.Segment{ + .kind = .pagerank, + .source_graph_artifact_id = try alloc.dupe(u8, "sha256:graph"), + .source_graph_checksum = try alloc.dupe(u8, "sha256:sum"), + .config_fingerprint = 42, + .materialization_state = .rejected, + .rejection_reason = .build_budget_exceeded, + .edge_filter = .{}, + .converged = false, + .iterations_completed = 0, + .delta = 0, + .scores = try alloc.alloc(types.Score, 0), + }; + defer segment.deinit(alloc); + const encoded = try encodeAlloc(alloc, segment); + defer alloc.free(encoded); + var decoded = try decodeAlloc(alloc, encoded); + defer decoded.deinit(alloc); + try std.testing.expectEqual(types.MaterializationState.rejected, decoded.materialization_state); + try std.testing.expectEqual(types.RejectionReason.build_budget_exceeded, decoded.rejection_reason); + try std.testing.expectEqual(@as(usize, 0), decoded.scores.len); +} + +test "serverless graph metric segment rejects unpublished legacy wire versions" { + const alloc = std.testing.allocator; + var segment = types.Segment{ + .kind = .degree, + .source_graph_artifact_id = try alloc.dupe(u8, "sha256:graph"), + .source_graph_checksum = try alloc.dupe(u8, "sha256:sum"), + .config_fingerprint = 9, + .edge_filter = .{}, + .converged = true, + .iterations_completed = 1, + .delta = 0, + .scores = try alloc.alloc(types.Score, 0), + }; + defer segment.deinit(alloc); + const current = try encodeAlloc(alloc, segment); + defer alloc.free(current); + const version_one = try alloc.dupe(u8, current); + defer alloc.free(version_one); + std.mem.writeInt(u16, version_one[4..6], 1, .little); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeHeader(version_one)); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeAlloc(alloc, version_one)); + + const version_six = try alloc.dupe(u8, current); + defer alloc.free(version_six); + std.mem.writeInt(u16, version_six[4..6], 6, .little); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeHeader(version_six)); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeAlloc(alloc, version_six)); + + const future = try alloc.dupe(u8, current); + defer alloc.free(future); + std.mem.writeInt(u16, future[4..6], wire_version + 1, .little); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeHeader(future)); + try std.testing.expectError(error.UnsupportedGraphMetricSegmentVersion, decodeAlloc(alloc, future)); +} diff --git a/zig/pkg/antfly/src/serverless/graph_metric_segment/mod.zig b/zig/pkg/antfly/src/serverless/graph_metric_segment/mod.zig new file mode 100644 index 0000000000..b55243cd6e --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_metric_segment/mod.zig @@ -0,0 +1,51 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the Elastic License 2.0 is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See +// the Elastic License 2.0 for the specific language governing permissions and +// limitations. + +pub const types = @import("types.zig"); +pub const codec = @import("codec.zig"); +pub const Score = types.Score; +pub const Segment = types.Segment; +pub const MaterializationState = types.MaterializationState; +pub const RejectionReason = types.RejectionReason; +pub const freeSegment = types.freeSegment; +pub const artifactNameAlloc = types.artifactNameAlloc; +pub const parseArtifactName = types.parseArtifactName; +pub const encodeAlloc = codec.encodeAlloc; +pub const encodeAllocWithCancellation = codec.encodeAllocWithCancellation; +pub const encodeAllocWithCancellationAndLimit = codec.encodeAllocWithCancellationAndLimit; +pub const encodedSize = codec.encodedSize; +pub const encodedSizeWithCancellation = codec.encodedSizeWithCancellation; +pub const decodeAlloc = codec.decodeAlloc; +pub const decodeAllocWithLimits = codec.decodeAllocWithLimits; +pub const decodeAllocWithCancellation = codec.decodeAllocWithCancellation; +pub const decodeAllocWithLimitsAndCancellation = codec.decodeAllocWithLimitsAndCancellation; +pub const decodeHeader = codec.decodeHeader; +pub const decodeControl = codec.decodeControl; +pub const decodeRoutingIndexAlloc = codec.decodeRoutingIndexAlloc; +pub const decodeRoutingIndexWithCancellationAlloc = codec.decodeRoutingIndexWithCancellationAlloc; +pub const decodeRoutingIndexForVersionWithCancellationAlloc = codec.decodeRoutingIndexForVersionWithCancellationAlloc; +pub const artifactIntegrity = codec.artifactIntegrity; +pub const routingFooterLenFromTrailer = codec.routingFooterLenFromTrailer; +pub const decodeScoreBlockWithCancellation = codec.decodeScoreBlockWithCancellation; +pub const scoreFromBlockWithCancellation = codec.scoreFromBlockWithCancellation; +pub const wire_version = codec.wire_version; +pub const headerProbeLen = codec.headerProbeLen; +pub const controlProbeLen = codec.controlProbeLen; +pub const routing_trailer_len = codec.routing_trailer_len; +pub const score_block_entries = codec.score_block_entries; + +test "serverless graph metric segment module compiles" { + _ = types; + _ = codec; +} diff --git a/zig/pkg/antfly/src/serverless/graph_metric_segment/types.zig b/zig/pkg/antfly/src/serverless/graph_metric_segment/types.zig new file mode 100644 index 0000000000..b77196d628 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_metric_segment/types.zig @@ -0,0 +1,151 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the Elastic License 2.0 is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See +// the Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const graph_mod = @import("../../graph/graph.zig"); + +pub const Score = struct { + node_id: []u8, + value: f64, + + pub fn deinit(self: *Score, alloc: Allocator) void { + alloc.free(self.node_id); + self.* = undefined; + } +}; + +pub const MaterializationState = enum(u8) { + ready = 0, + rejected = 1, +}; + +pub const RejectionReason = enum(u8) { + none = 0, + build_budget_exceeded = 1, +}; + +/// An immutable metric vector tied to the exact graph artifact from which it +/// was computed. Scores are sorted by node id for deterministic encoding and +/// binary-search point lookups without a per-request hash table. +pub const Segment = struct { + /// Wire schema observed while decoding. Producers leave this at zero; the + /// codec stamps the emitted version on read. + metadata_version: u16 = 0, + kind: graph_mod.GraphMetricKind, + source_graph_artifact_id: []u8, + source_graph_checksum: []u8, + topology_checksum: [32]u8 = @splat(0), + config_fingerprint: u64, + /// Identifies the implementation and admission policy that produced this + /// artifact. A changed runtime policy invalidates terminal rejections and + /// safely causes a rebuild without user configuration churn. + materializer_fingerprint: u64 = 0, + /// Serving generation that first published this immutable metric. These + /// remain stable when a later manifest reuses the artifact. + published_generation: u64 = 0, + /// Graph/source generation evaluated by the materializer. + edge_generation: u64 = 0, + /// Wall-clock completion time in Unix epoch milliseconds. Zero denotes an + /// older artifact whose provenance predates this field. + computed_at_ms: u64 = 0, + materialization_state: MaterializationState = .ready, + rejection_reason: RejectionReason = .none, + edge_filter: graph_mod.GraphMetricEdgeFilter, + converged: bool, + iterations_completed: u32, + delta: f64, + scores: []Score, + /// Decoded segments own score identifiers. Encoders can instead borrow a + /// canonical projection for the segment's short, scoped lifetime without + /// adding ownership metadata to every score. + owns_score_node_ids: bool = true, + + pub fn deinit(self: *Segment, alloc: Allocator) void { + alloc.free(self.source_graph_artifact_id); + alloc.free(self.source_graph_checksum); + self.edge_filter.deinit(alloc); + if (self.owns_score_node_ids) for (self.scores) |*item| item.deinit(alloc); + alloc.free(self.scores); + self.* = undefined; + } + + pub fn score(self: Segment, node_id: []const u8) ?f64 { + var low: usize = 0; + var high = self.scores.len; + while (low < high) { + const mid = low + (high - low) / 2; + switch (std.mem.order(u8, self.scores[mid].node_id, node_id)) { + .lt => low = mid + 1, + .gt => high = mid, + .eq => return self.scores[mid].value, + } + } + return null; + } +}; + +pub fn freeSegment(alloc: Allocator, segment: *Segment) void { + segment.deinit(alloc); +} + +/// Length-prefix both components so index and metric names cannot alias even +/// when they contain separators used by human-readable artifact names. +pub fn artifactNameAlloc(alloc: Allocator, graph_index_name: []const u8, metric_name: []const u8) ![]u8 { + if (graph_index_name.len == 0 or metric_name.len == 0) return error.InvalidGraphMetricArtifactName; + return try std.fmt.allocPrint(alloc, "{d}:{s}{d}:{s}", .{ graph_index_name.len, graph_index_name, metric_name.len, metric_name }); +} + +pub const ParsedArtifactName = struct { + graph_index_name: []const u8, + metric_name: []const u8, +}; + +/// Parse the length-prefixed artifact name without allocation. Validating both +/// components avoids separator ambiguity and makes manifest recovery safe for +/// arbitrary user-supplied index and metric names. +pub fn parseArtifactName(name: []const u8) !ParsedArtifactName { + const graph_separator = std.mem.indexOfScalar(u8, name, ':') orelse return error.InvalidGraphMetricArtifactName; + if (graph_separator == 0) return error.InvalidGraphMetricArtifactName; + const graph_len = std.fmt.parseInt(usize, name[0..graph_separator], 10) catch return error.InvalidGraphMetricArtifactName; + if (graph_len == 0) return error.InvalidGraphMetricArtifactName; + const graph_start = graph_separator + 1; + const graph_end = std.math.add(usize, graph_start, graph_len) catch return error.InvalidGraphMetricArtifactName; + if (graph_end >= name.len) return error.InvalidGraphMetricArtifactName; + + const metric_separator_relative = std.mem.indexOfScalar(u8, name[graph_end..], ':') orelse return error.InvalidGraphMetricArtifactName; + if (metric_separator_relative == 0) return error.InvalidGraphMetricArtifactName; + const metric_separator = graph_end + metric_separator_relative; + const metric_len = std.fmt.parseInt(usize, name[graph_end..metric_separator], 10) catch return error.InvalidGraphMetricArtifactName; + if (metric_len == 0) return error.InvalidGraphMetricArtifactName; + const metric_start = metric_separator + 1; + const metric_end = std.math.add(usize, metric_start, metric_len) catch return error.InvalidGraphMetricArtifactName; + if (metric_end != name.len) return error.InvalidGraphMetricArtifactName; + + return .{ + .graph_index_name = name[graph_start..graph_end], + .metric_name = name[metric_start..metric_end], + }; +} + +test "serverless graph metric artifact names round trip without separator ambiguity" { + const alloc = std.testing.allocator; + const encoded = try artifactNameAlloc(alloc, "graph:west", "rank:daily"); + defer alloc.free(encoded); + + const parsed = try parseArtifactName(encoded); + try std.testing.expectEqualStrings("graph:west", parsed.graph_index_name); + try std.testing.expectEqualStrings("rank:daily", parsed.metric_name); + try std.testing.expectError(error.InvalidGraphMetricArtifactName, parseArtifactName("5:short4:no")); +} diff --git a/zig/pkg/antfly/src/serverless/graph_segment/adjacency_reader.zig b/zig/pkg/antfly/src/serverless/graph_segment/adjacency_reader.zig new file mode 100644 index 0000000000..ce4f2329a3 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_segment/adjacency_reader.zig @@ -0,0 +1,472 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Authenticated, bounded point reads over the current graph wire. The node +//! dictionary is paged; the ordinal routing array addresses rows without a +//! graph-wide decode or a per-request hash table. +const std = @import("std"); +const Allocator = std.mem.Allocator; +const wire = @import("packed.zig"); +const types = @import("types.zig"); +const topology = @import("topology_reader.zig"); +const artifacts = @import("../artifacts/store.zig"); +const refs = @import("../manifest/artifact_ref.zig"); +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; + +pub const Reader = struct { + alloc: Allocator, + context: topology.Context, + kinds: []const []const u8, + tables: []const []const u8, + table_bytes: []u8, + page_bytes: []u8 = &.{}, + page: ?usize = null, + page_nodes: [wire.node_page_entries][]const u8 = undefined, + page_count: usize = 0, + + pub fn init(alloc: Allocator, store: *artifacts.ArtifactStore, source: refs.ArtifactRef, cancellation: CancellationToken, remaining: *u64) !?Reader { + // Queries alternate dictionary, routing, and adjacency blocks. A small + // request-local working set avoids thrashing these independent ranges; + // streaming topology preparation keeps its one-block configuration. + var context = try topology.Context.initWithCache(alloc, store, source, cancellation, remaining, 8); + errdefer context.deinit(); + const directory = context.directory orelse { + context.deinit(); + return null; + }; + const header = try context.readAlloc(alloc, 0, wire.header_len); + defer alloc.free(header); + if (!std.mem.eql(u8, header[0..4], wire.wire_magic) or std.mem.readInt(u16, header[4..6], .little) != wire.wire_version) + return error.InvalidGraphSegment; + if (std.mem.readInt(u32, header[10..14], .little) != directory.nodes) return error.InvalidGraphSegment; + const table_count = std.mem.readInt(u32, header[6..10], .little); + const kind_count = std.mem.readInt(u32, header[14..18], .little); + if (kind_count > directory.entries.len / 52) return error.InvalidGraphSegment; + const kinds = try alloc.alloc([]const u8, kind_count); + errdefer alloc.free(kinds); + var iterator = directory.iterator(); + for (kinds) |*kind| kind.* = (try iterator.next() orelse return error.InvalidGraphSegment).kind; + if (try iterator.next() != null) return error.InvalidGraphSegment; + const nodes_begin = std.mem.readInt(u64, directory.page_offsets[0..8], .little); + if (nodes_begin < wire.header_len or nodes_begin > context.trailer.body_len) return error.InvalidGraphSegment; + if (table_count > (nodes_begin - wire.header_len) / 4) return error.InvalidGraphSegment; + const table_bytes = try context.readAlloc(alloc, wire.header_len, nodes_begin - wire.header_len); + errdefer alloc.free(table_bytes); + const tables = try alloc.alloc([]const u8, table_count); + errdefer alloc.free(tables); + var pos: usize = 0; + for (tables) |*table| { + table.* = try string(table_bytes, &pos); + if (table.len == 0) return error.InvalidGraphSegment; + } + if (pos != table_bytes.len) return error.InvalidGraphSegment; + return .{ .alloc = alloc, .context = context, .kinds = kinds, .tables = tables, .table_bytes = table_bytes }; + } + + pub fn deinit(self: *Reader) void { + self.alloc.free(self.page_bytes); + self.alloc.free(self.tables); + self.alloc.free(self.table_bytes); + self.alloc.free(self.kinds); + self.context.deinit(); + self.* = undefined; + } + + fn string(bytes: []const u8, pos: *usize) ![]const u8 { + if (bytes.len - pos.* < 4) return error.InvalidGraphSegment; + const len = std.mem.readInt(u32, bytes[pos.*..][0..4], .little); + pos.* += 4; + if (len > bytes.len - pos.*) return error.InvalidGraphSegment; + const result = bytes[pos.*..][0..len]; + pos.* += len; + return result; + } + + fn loadPage(self: *Reader, page: usize) !void { + if (self.page == page) return; + const directory = self.context.directory.?; + const range = try directory.nodePage(page); + if (range.offset < wire.header_len or range.offset > self.context.trailer.body_len or range.len > self.context.trailer.body_len - range.offset) + return error.InvalidGraphSegment; + self.alloc.free(self.page_bytes); + self.page_bytes = &.{}; + self.page = null; + self.page_bytes = try self.context.readAlloc(self.alloc, range.offset, range.len); + self.page_count = @min(wire.node_page_entries, directory.nodes - page * wire.node_page_entries); + var pos: usize = 0; + for (self.page_nodes[0..self.page_count], 0..) |*node, i| { + node.* = try string(self.page_bytes, &pos); + if (i > 0 and std.mem.order(u8, self.page_nodes[i - 1], node.*) != .lt) return error.InvalidGraphSegment; + } + if (pos != self.page_bytes.len) return error.InvalidGraphSegment; + const fence_bytes = directory.page_fences[page * wire.node_page_fence_bytes ..][0..wire.node_page_fence_bytes]; + if (self.page_nodes[0].len != std.mem.readInt(u32, fence_bytes[0..4], .little) or !std.mem.eql(u8, self.fence(page), self.page_nodes[0][0..@min(self.page_nodes[0].len, 64)])) return error.InvalidGraphSegment; + self.page = page; + } + + fn ordinal(self: *Reader, key: []const u8) !?u32 { + const directory = self.context.directory.?; + const pages = directory.page_offsets.len / 8 - 1; + if (pages == 0) return null; + // Authenticated 64-byte fence prefixes usually identify one page with + // no I/O. Long shared prefixes only widen the binary-search interval; + // they never change lookup semantics or require unbounded control data. + const prefix = key[0..@min(key.len, 64)]; + var begin: usize = 0; + var end = pages; + while (begin < end) { + const middle = begin + (end - begin) / 2; + if (std.mem.order(u8, self.fence(middle), prefix) == .lt) begin = middle + 1 else end = middle; + } + const first = begin; + if (first < pages and key.len <= 64) { + const raw = directory.page_fences[first * wire.node_page_fence_bytes ..][0..wire.node_page_fence_bytes]; + if (std.mem.readInt(u32, raw[0..4], .little) == key.len and std.mem.eql(u8, self.fence(first), key)) + return @intCast(first * wire.node_page_entries); + } + end = pages; + while (begin < end) { + const middle = begin + (end - begin) / 2; + if (std.mem.order(u8, self.fence(middle), prefix) != .gt) begin = middle + 1 else end = middle; + } + if (begin == 0) return null; + var lower = first -| 1; + var upper = begin; + while (lower < upper) { + const middle = lower + (upper - lower) / 2; + try self.loadPage(middle); + if (std.mem.order(u8, self.page_nodes[self.page_count - 1], key) == .lt) lower = middle + 1 else upper = middle; + } + if (lower == begin) return null; + try self.loadPage(lower); + const index = std.sort.binarySearch([]const u8, self.page_nodes[0..self.page_count], key, compareString) orelse return null; + return @intCast(lower * wire.node_page_entries + index); + } + + fn fence(self: *Reader, page: usize) []const u8 { + const bytes = self.context.directory.?.page_fences[page * wire.node_page_fence_bytes ..][0..wire.node_page_fence_bytes]; + return bytes[4..][0..@min(std.mem.readInt(u32, bytes[0..4], .little), 64)]; + } + + fn compareString(a: []const u8, b: []const u8) std.math.Order { + return std.mem.order(u8, a, b); + } + + const Row = struct { offset: u64, out: u32, in: u32 }; + pub fn containsNode(self: *Reader, key: []const u8) !bool { + return try self.row(key) != null; + } + fn row(self: *Reader, key: []const u8) !?Row { + const node = try self.ordinal(key) orelse return null; + const routing = self.context.trailer.body_len + self.context.trailer.topology_len; + const raw = try self.context.readAlloc(self.alloc, routing + @as(u64, node) * 8, 8); + defer self.alloc.free(raw); + const offset = std.mem.readInt(u64, raw[0..8], .little); + if (offset == 0) return null; + if (offset < wire.header_len or offset > self.context.trailer.body_len or self.context.trailer.body_len - offset < 12) return error.InvalidGraphSegment; + const header = try self.context.readAlloc(self.alloc, offset, 12); + defer self.alloc.free(header); + if (std.mem.readInt(u32, header[0..4], .little) != node) return error.InvalidGraphSegment; + const result = Row{ .offset = offset + 12, .out = std.mem.readInt(u32, header[4..8], .little), .in = std.mem.readInt(u32, header[8..12], .little) }; + if ((@as(u64, result.out) + result.in) * wire.edge_len > self.context.trailer.body_len - result.offset) return error.InvalidGraphSegment; + return result; + } + + fn edgeAt(self: *Reader, offset: u64, index: usize, work: *usize) !wire.Edge { + if (work.* == 0) return error.GraphTraversalQueryBudgetExceeded; + work.* -= 1; + const bytes = try self.context.readAlloc(self.alloc, offset + index * wire.edge_len, wire.edge_len); + defer self.alloc.free(bytes); + return self.validEdge(bytes, 0); + } + + fn validEdge(self: *Reader, bytes: []const u8, index: usize) !wire.Edge { + const edge = wire.readEdge(bytes, index); + if (edge.node >= self.context.directory.?.nodes or edge.edge_type >= self.kinds.len or !std.math.isFinite(edge.weight)) return error.InvalidGraphSegment; + if (edge.table) |table| if (table >= self.tables.len) return error.InvalidGraphSegment; + return edge; + } + + fn lowerBound(self: *Reader, offset: u64, count: usize, kind: u32, node: u32, work: *usize) !usize { + var lower: usize = 0; + var upper = count; + while (lower < upper) { + const middle = lower + (upper - lower) / 2; + const edge = try self.edgeAt(offset, middle, work); + if (edge.edge_type < kind or (edge.edge_type == kind and edge.node < node)) lower = middle + 1 else upper = middle; + } + return lower; + } + + fn copyEdge(self: *Reader, edge: wire.Edge) !types.Edge { + try self.loadPage(edge.node / wire.node_page_entries); + const neighbor = try self.alloc.dupe(u8, self.page_nodes[edge.node % wire.node_page_entries]); + errdefer self.alloc.free(neighbor); + return .{ .neighbor_id = neighbor, .edge_type = try self.alloc.dupe(u8, self.kinds[edge.edge_type]), .weight = edge.weight, .neighbor_table_id = edge.table }; + } + + /// Work is a shared remaining physical-edge allowance, consumed before I/O. + /// Returned edges own their strings using this reader's admitted allocator. + pub fn probe(self: *Reader, source: []const u8, kind: []const u8, target: []const u8, work: *usize) !?types.Edge { + const kind_id = std.sort.binarySearch([]const u8, self.kinds, kind, compareString) orelse return null; + const target_id = try self.ordinal(target) orelse return null; + const found = try self.row(source) orelse return null; + const index = try self.lowerBound(found.offset, found.out, @intCast(kind_id), target_id, work); + if (index == found.out) return null; + const edge = try self.edgeAt(found.offset, index, work); + if (edge.edge_type != kind_id or edge.node != target_id) return null; + return try self.copyEdge(edge); + } + + fn readEdges(self: *Reader, offset: u64, count: usize, requested: []const []const u8, limit: usize, work: *usize, skip_qualified: bool, skip_node: ?[]const u8) ![]types.Edge { + const Range = struct { begin: usize, end: usize }; + var ranges: std.ArrayListUnmanaged(Range) = .empty; + defer ranges.deinit(self.alloc); + var total: usize = 0; + if (requested.len == 0) { + total = count; + try ranges.append(self.alloc, .{ .begin = 0, .end = count }); + } else for (requested, 0..) |kind, i| { + // Duplicate filters never duplicate physical edges or admission. + const duplicate = for (requested[0..i]) |prior| { + if (std.mem.eql(u8, prior, kind)) break true; + } else false; + if (duplicate) continue; + const id = std.sort.binarySearch([]const u8, self.kinds, kind, compareString) orelse continue; + const begin = try self.lowerBound(offset, count, @intCast(id), 0, work); + const end = try self.lowerBound(offset, count, @intCast(id + 1), 0, work); + total = std.math.add(usize, total, end - begin) catch return error.QueryCandidateBudgetExceeded; + try ranges.append(self.alloc, .{ .begin = begin, .end = end }); + } + if (!skip_qualified and skip_node == null and total > limit) return error.QueryCandidateBudgetExceeded; + if (total > work.*) return error.GraphTraversalQueryBudgetExceeded; + work.* -= total; + // Preserve canonical order independent of filter order. + std.mem.sort(Range, ranges.items, {}, struct { + fn less(_: void, a: Range, b: Range) bool { + return a.begin < b.begin; + } + }.less); + var result: std.ArrayListUnmanaged(types.Edge) = .empty; + errdefer { + for (result.items) |*edge| edge.deinit(self.alloc); + result.deinit(self.alloc); + } + for (ranges.items) |range| { + var begin = range.begin; + while (begin < range.end) { + const n = @min(range.end - begin, 4096); + const bytes = try self.context.readAlloc(self.alloc, offset + begin * wire.edge_len, n * wire.edge_len); + defer self.alloc.free(bytes); + for (0..n) |i| { + const edge = try self.validEdge(bytes, i); + if (skip_qualified and edge.table != null) continue; + if (skip_node) |node| { + try self.loadPage(edge.node / wire.node_page_entries); + if (std.mem.eql(u8, node, self.page_nodes[edge.node % wire.node_page_entries])) continue; + } + if (result.items.len == limit) return error.QueryCandidateBudgetExceeded; + try result.ensureUnusedCapacity(self.alloc, 1); + result.appendAssumeCapacity(try self.copyEdge(edge)); + } + begin += n; + } + } + return result.toOwnedSlice(self.alloc); + } + + pub fn adjacency(self: *Reader, key: []const u8, requested: []const []const u8, direction: anytype, limit: usize, work: *usize) !?types.Adjacency { + return self.adjacencyFiltered(key, requested, direction, limit, work, true, false); + } + + pub fn adjacencyFiltered(self: *Reader, key: []const u8, requested: []const []const u8, direction: anytype, limit: usize, work: *usize, include_qualified: bool, deduplicate_self_loops: bool) !?types.Adjacency { + const found = try self.row(key) orelse return null; + const node = try self.alloc.dupe(u8, key); + errdefer self.alloc.free(node); + const out = try self.readEdges(found.offset, if (direction == .out or direction == .both) found.out else 0, requested, limit, work, !include_qualified, null); + errdefer { + for (out) |*edge| edge.deinit(self.alloc); + self.alloc.free(out); + } + const incoming = try self.readEdges(found.offset + @as(u64, found.out) * wire.edge_len, if (direction == .in or direction == .both) found.in else 0, requested, limit - out.len, work, false, if (deduplicate_self_loops and direction == .both) key else null); + return .{ .node_id = node, .out_edges = out, .in_edges = incoming }; + } +}; + +const TestStore = struct { + payload: []const u8, + calls: usize = 0, + bytes: usize = 0, + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(_: *anyopaque, _: Allocator, _: []const u8) !artifacts.ArtifactMetadata { + return error.Unsupported; + } + fn get(_: *anyopaque, _: Allocator, _: []const u8) ![]u8 { + return error.UnexpectedFullRead; + } + fn range(ptr: *anyopaque, alloc: Allocator, _: []const u8, offset: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + if (offset > self.payload.len or len > self.payload.len - offset) return error.InvalidRange; + self.calls += 1; + self.bytes += len; + return alloc.dupe(u8, self.payload[@intCast(offset)..][0..len]); + } + fn stat(_: *anyopaque, _: Allocator, _: []const u8) !artifacts.ArtifactMetadata { + return error.Unsupported; + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.Unsupported; + } + const vtable = artifacts.ArtifactStore.VTable{ .deinit = deinit, .put = put, .get_alloc = get, .get_range_alloc = range, .stat = stat, .delete = delete }; +}; + +fn exerciseReader(alloc: Allocator, payload: []const u8, source: refs.ArtifactRef) !void { + var memory = TestStore{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &TestStore.vtable }; + var remaining: u64 = 1024 * 1024; + var reader = (try Reader.init(alloc, &store, source, .none, &remaining)).?; + defer reader.deinit(); + var work: usize = 1000; + const Direction = enum { out, in, both }; + var result = (try reader.adjacency("a", &.{ "z", "link", "link" }, Direction.both, 8, &work)).?; + defer result.deinit(alloc); + try std.testing.expectEqual(@as(usize, 3), result.out_edges.len); + try std.testing.expectEqual(@as(usize, 1), result.in_edges.len); + try std.testing.expectEqualStrings("a", result.out_edges[0].neighbor_id); + try std.testing.expectEqualStrings("b", result.out_edges[1].neighbor_id); + try std.testing.expectEqual(@as(f32, 2), result.out_edges[1].weight); + try std.testing.expectEqual(@as(?u32, 0), result.out_edges[2].neighbor_table_id); + try std.testing.expectEqualStrings("elsewhere", reader.tables[0]); + var exact = (try reader.probe("a", "link", "b", &work)).?; + defer exact.deinit(alloc); + try std.testing.expectEqual(@as(f32, 2), exact.weight); + try std.testing.expect(try reader.probe("a", "absent", "b", &work) == null); + try std.testing.expect(try reader.adjacency("absent", &.{}, Direction.out, 8, &work) == null); + var incoming = (try reader.adjacency("b", &.{"link"}, Direction.in, 8, &work)).?; + defer incoming.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), incoming.out_edges.len); + try std.testing.expectEqual(@as(usize, 1), incoming.in_edges.len); + if (reader.adjacency("a", &.{}, Direction.out, 1, &work)) |_| return error.ExpectedBudgetFailure else |err| { + if (err == error.OutOfMemory) return err; + try std.testing.expectEqual(error.QueryCandidateBudgetExceeded, err); + } + work = 0; + if (reader.probe("a", "link", "b", &work)) |_| return error.ExpectedBudgetFailure else |err| { + if (err == error.OutOfMemory) return err; + try std.testing.expectEqual(error.GraphTraversalQueryBudgetExceeded, err); + } + try std.testing.expectEqual(@as(usize, 3), memory.calls); + try std.testing.expectEqual(payload.len, memory.bytes); +} + +test "serverless graph paged adjacency preserves lookup semantics budgets and allocation cleanup" { + const alloc = std.testing.allocator; + const payload = try wire.encodeAlloc(alloc, .{ + .neighbor_tables = @constCast(&[_][]u8{@constCast("elsewhere")}), + .adjacencies = @constCast(&[_]types.Adjacency{ + types.Adjacency{ .node_id = @constCast("a"), .out_edges = @constCast(&[_]types.Edge{ + types.Edge{ .neighbor_id = @constCast("a"), .edge_type = @constCast("link"), .weight = 1 }, + types.Edge{ .neighbor_id = @constCast("b"), .edge_type = @constCast("link"), .weight = 2 }, + types.Edge{ .neighbor_id = @constCast("b"), .edge_type = @constCast("z"), .weight = 3, .neighbor_table_id = 0 }, + }), .in_edges = @constCast(&[_]types.Edge{types.Edge{ .neighbor_id = @constCast("a"), .edge_type = @constCast("link"), .weight = 1 }}) }, + types.Adjacency{ .node_id = @constCast("b"), .out_edges = &.{}, .in_edges = @constCast(&[_]types.Edge{types.Edge{ .neighbor_id = @constCast("a"), .edge_type = @constCast("link"), .weight = 2 }}) }, + }), + }); + defer alloc.free(payload); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &digest, .{}); + const checksum = std.fmt.bytesToHex(digest, .lower); + const id = "sha256:" ++ checksum; + var source = refs.ArtifactRef{ .kind = .graph_segment, .name = "g", .artifact_id = id, .checksum = &checksum, .byte_len = payload.len }; + try wire.bindTopologyControl(&source, payload); + try exerciseReader(alloc, payload, source); + try std.testing.checkAllAllocationFailures(alloc, exerciseReader, .{ payload, source }); + + var memory = TestStore{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &TestStore.vtable }; + var remaining: u64 = 1024 * 1024; + // The routing array is covered by the same manifest-authenticated block + // hashes as dictionary and adjacency data. + const trailer = try wire.decodeTopologyTrailer(payload[payload.len - wire.topology_trailer_len ..], payload.len); + payload[@intCast(trailer.body_len + trailer.topology_len)] ^= 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, Reader.init(alloc, &store, source, .none, &remaining)); + payload[@intCast(trailer.body_len + trailer.topology_len)] ^= 1; + remaining = wire.topology_trailer_len - 1; + try std.testing.expectError(error.GraphMetricBuildBudgetExceeded, Reader.init(alloc, &store, source, .none, &remaining)); +} + +test "serverless graph paged preparation coalesces thousands of small type runs" { + const alloc = std.testing.allocator; + var fixture = std.heap.ArenaAllocator.init(alloc); + defer fixture.deinit(); + var builder = @import("builder.zig").Builder{ .alloc = fixture.allocator() }; + defer builder.deinit(); + for (0..10000) |i| try builder.addEdge("a", "b", try std.fmt.allocPrint(fixture.allocator(), "kind{d:0>5}", .{i}), 1, null); + const payload = try builder.encodeAlloc(4 * 1024 * 1024, .none); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &digest, .{}); + const checksum = std.fmt.bytesToHex(digest, .lower); + const id = try std.fmt.allocPrint(fixture.allocator(), "sha256:{s}", .{checksum}); + var source = refs.ArtifactRef{ .kind = .graph_segment, .name = "g", .artifact_id = id, .checksum = &checksum, .byte_len = payload.len }; + try wire.bindTopologyControl(&source, payload); + var memory = TestStore{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &TestStore.vtable }; + var remaining: u64 = 2 * 1024 * 1024; + const Config = struct { edge_filter: struct { mode: enum { all, types } = .all, types: []const []const u8 = &.{} } = .{} }; + var prepared = (try topology.readAlloc(alloc, &store, source, &[_]Config{.{}}, .{ .max_nodes = 2, .max_edges = 10000 }, .none, &remaining)).?; + defer prepared.deinit(alloc); + try std.testing.expectEqual(@as(usize, 10000), prepared.edges.len); + try std.testing.expectEqual(@as(usize, 10000), prepared.edge_types.len); + try std.testing.expect(memory.calls <= 8); + try std.testing.expect(memory.bytes <= payload.len); +} + +test "serverless graph paged dictionary fences handle long shared prefixes and page boundaries" { + const alloc = std.testing.allocator; + var fixture = std.heap.ArenaAllocator.init(alloc); + defer fixture.deinit(); + const a = fixture.allocator(); + const count = 1025; + const ids = try a.alloc([]const u8, count); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(a, "{s}/{d:0>8}", .{ &([_]u8{'x'} ** 100), i }); + var builder = @import("builder.zig").Builder{ .alloc = a }; + defer builder.deinit(); + for (ids, 0..) |id, i| try builder.addEdge(id, ids[(i + 1) % count], "link", 1, null); + const payload = try builder.encodeAlloc(4 * 1024 * 1024, .none); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &digest, .{}); + const checksum = std.fmt.bytesToHex(digest, .lower); + var source = refs.ArtifactRef{ .kind = .graph_segment, .name = "g", .artifact_id = try std.fmt.allocPrint(a, "sha256:{s}", .{checksum}), .checksum = &checksum, .byte_len = payload.len }; + try wire.bindTopologyControl(&source, payload); + var memory = TestStore{ .payload = payload }; + var store = artifacts.ArtifactStore{ .allocator = alloc, .ptr = &memory, .vtable = &TestStore.vtable }; + var remaining: u64 = 8 * 1024 * 1024; + var reader = (try Reader.init(alloc, &store, source, .none, &remaining)).?; + defer reader.deinit(); + var work: usize = 100; + for ([_]usize{ 0, 255, 256, 511, 512, 1023, 1024 }) |i| { + var row = (try reader.adjacency(ids[i], &.{}, enum { out, in, both }.out, 1, &work)).?; + defer row.deinit(alloc); + try std.testing.expectEqualStrings(ids[(i + 1) % count], row.out_edges[0].neighbor_id); + } + try std.testing.expect(!try reader.containsNode("")); + try std.testing.expect(!try reader.containsNode("z")); + const missing = try std.fmt.allocPrint(a, "{s}/00001025", .{&([_]u8{'x'} ** 100)}); + try std.testing.expect(!try reader.containsNode(missing)); + const calls = memory.calls; + for ([_]usize{ 0, 255, 256, 511, 512, 1023, 1024 }) |i| try std.testing.expect(try reader.containsNode(ids[i])); + // Routing, row headers, and dictionary pages coexist in the bounded + // request cache instead of evicting one another on every hop. + try std.testing.expectEqual(calls, memory.calls); +} diff --git a/zig/pkg/antfly/src/serverless/graph_segment/builder.zig b/zig/pkg/antfly/src/serverless/graph_segment/builder.zig new file mode 100644 index 0000000000..fcafdf03d9 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_segment/builder.zig @@ -0,0 +1,302 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Construction-time ordinal graph. Identifiers are owned once, independent +//! of degree; only integer edges survive between input documents/batches. +const std = @import("std"); +const wire = @import("packed.zig"); +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +const edge_type = @import("../../graph/edge_type.zig"); +const Allocator = std.mem.Allocator; + +const Dictionary = struct { + values: std.StringArrayHashMapUnmanaged(bool) = .empty, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.values.keys()) |key| alloc.free(key); + self.values.deinit(alloc); + } + + fn intern(self: *@This(), alloc: Allocator, key: []const u8, local: bool) !u32 { + if (self.values.getIndex(key)) |i| { + self.values.values()[i] = self.values.values()[i] or local; + return @intCast(i); + } + const ordinal = std.math.cast(u32, self.values.count()) orelse return error.GraphSegmentTooLarge; + const owned = try alloc.dupe(u8, key); + errdefer alloc.free(owned); + try self.values.put(alloc, owned, local); + return ordinal; + } + + fn orderAlloc(self: *const @This(), alloc: Allocator) ![]u32 { + const order = try alloc.alloc(u32, self.values.count()); + for (order, 0..) |*value, i| value.* = @intCast(i); + std.mem.sort(u32, order, self.values.keys(), struct { + fn less(keys: []const []const u8, a: u32, b: u32) bool { + return std.mem.order(u8, keys[a], keys[b]) == .lt; + } + }.less); + return order; + } +}; + +pub const Edge = struct { + source: u32, + target: u32, + kind: u32, + table: u32, + weight: f32, +}; + +pub const Builder = struct { + alloc: Allocator, + nodes: Dictionary = .{}, + kinds: Dictionary = .{}, + tables: Dictionary = .{}, + edges: std.ArrayListUnmanaged(Edge) = .empty, + local_nodes: usize = 0, + + pub fn deinit(self: *@This()) void { + self.nodes.deinit(self.alloc); + self.kinds.deinit(self.alloc); + self.tables.deinit(self.alloc); + self.edges.deinit(self.alloc); + self.* = undefined; + } + + pub fn nodeCount(self: *const @This()) usize { + return self.local_nodes; + } + + pub fn addNode(self: *@This(), node: []const u8) !void { + _ = try self.internNode(node, true); + } + + fn internNode(self: *@This(), node: []const u8, local: bool) !u32 { + const was_local = self.nodes.values.get(node) orelse false; + const id = try self.nodes.intern(self.alloc, node, local); + if (local and !was_local) self.local_nodes += 1; + return id; + } + + pub fn addEdge(self: *@This(), source: []const u8, target: []const u8, kind: []const u8, weight: f32, table: ?[]const u8) !void { + if (!std.math.isFinite(weight)) return error.InvalidGraphSegment; + try edge_type.validateStored(kind); + if (table) |name| if (name.len == 0) return error.InvalidGraphSegment; + const src = try self.internNode(source, true); + const dst = try self.internNode(target, table == null); + const typ = try self.kinds.intern(self.alloc, kind, false); + const tbl = if (table) |name| try self.tables.intern(self.alloc, name, false) else wire.no_table; + try self.edges.append(self.alloc, .{ .source = src, .target = dst, .kind = typ, .table = tbl, .weight = weight }); + } + + /// Does not consume the builder. Dictionary/edge ordering is canonical, + /// including table-qualified duplicate endpoints and input permutations. + pub fn encodeAlloc(self: *const @This(), max_bytes: usize, cancellation: CancellationToken) ![]u8 { + try cancellation.check(); + const node_order = try self.nodes.orderAlloc(self.alloc); + defer self.alloc.free(node_order); + const type_order = try self.kinds.orderAlloc(self.alloc); + defer self.alloc.free(type_order); + const table_order = try self.tables.orderAlloc(self.alloc); + defer self.alloc.free(table_order); + const node_map = try invert(self.alloc, node_order); + defer self.alloc.free(node_map); + const type_map = try invert(self.alloc, type_order); + defer self.alloc.free(type_map); + const table_map = try invert(self.alloc, table_order); + defer self.alloc.free(table_map); + var local_edges: usize = 0; + for (self.edges.items) |edge| local_edges += @intFromBool(edge.table == wire.no_table); + var size: usize = wire.header_len; + for ([_]*const Dictionary{ &self.nodes, &self.kinds, &self.tables }) |dict| { + for (dict.values.keys()) |key| { + _ = std.math.cast(u32, key.len) orelse return error.GraphSegmentTooLarge; + size = std.math.add(usize, size, std.math.add(usize, key.len, 4) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + } + } + const record_count = std.math.add(usize, self.edges.items.len, local_edges) catch return error.GraphSegmentTooLarge; + size = std.math.add(usize, size, std.math.mul(usize, record_count, wire.edge_len) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + size = std.math.add(usize, size, std.math.mul(usize, self.nodeCount(), 12) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + const body_len = size; + size = std.math.add(usize, size, try wire.topologyExtensionSize(self.kinds.values.keys(), self.nodes.values.count(), local_edges, body_len)) catch return error.GraphSegmentTooLarge; + if (size > max_bytes) return error.GraphSegmentTooLarge; + // Count and scatter directly into final adjacency storage. No mapped + // forward/reverse Edge arrays coexist with the immutable wire payload. + const counts = try self.alloc.alloc([2]u32, node_order.len); + defer self.alloc.free(counts); + @memset(counts, .{ 0, 0 }); + for (self.edges.items, 0..) |edge, i| { + if (i % 4096 == 0) try cancellation.check(); + const src = node_map[edge.source]; + counts[src][0] = std.math.add(u32, counts[src][0], 1) catch return error.GraphSegmentTooLarge; + if (edge.table == wire.no_table) { + const dst = node_map[edge.target]; + counts[dst][1] = std.math.add(u32, counts[dst][1], 1) catch return error.GraphSegmentTooLarge; + } + } + const positions = try self.alloc.alloc([2]usize, node_order.len); + defer self.alloc.free(positions); + const bytes = try self.alloc.alloc(u8, size); + errdefer self.alloc.free(bytes); + @memcpy(bytes[0..4], wire.wire_magic); + std.mem.writeInt(u16, bytes[4..6], wire.wire_version, .little); + var pos: usize = 6; + put(bytes, &pos, @intCast(table_order.len)); + put(bytes, &pos, @intCast(node_order.len)); + put(bytes, &pos, @intCast(type_order.len)); + put(bytes, &pos, @intCast(self.nodeCount())); + inline for (.{ .{ &self.tables, table_order }, .{ &self.nodes, node_order }, .{ &self.kinds, type_order } }) |pair| { + for (pair[1]) |i| { + const key = pair[0].values.keys()[i]; + put(bytes, &pos, @intCast(key.len)); + @memcpy(bytes[pos..][0..key.len], key); + pos += key.len; + } + } + for (node_order, 0..) |old_node, node| { + if (node % 256 == 0) try cancellation.check(); + if (!self.nodes.values.values()[old_node]) continue; + put(bytes, &pos, @intCast(node)); + put(bytes, &pos, counts[node][0]); + put(bytes, &pos, counts[node][1]); + for (0..2) |direction| { + positions[node][direction] = pos; + pos += @as(usize, counts[node][direction]) * wire.edge_len; + } + } + std.debug.assert(pos == body_len); + for (self.edges.items, 0..) |edge, i| { + if (i % 4096 == 0) try cancellation.check(); + const src = node_map[edge.source]; + const dst = node_map[edge.target]; + writeEdge(bytes, &positions[src][0], dst, type_map[edge.kind], edge.weight, if (edge.table == wire.no_table) wire.no_table else table_map[edge.table]); + if (edge.table == wire.no_table) writeEdge(bytes, &positions[dst][1], src, type_map[edge.kind], edge.weight, wire.no_table); + } + for (node_order, 0..) |old_node, node| { + if (!self.nodes.values.values()[old_node]) continue; + for (0..2) |direction| { + try cancellation.check(); + const end = positions[node][direction]; + const start = end - @as(usize, counts[node][direction]) * wire.edge_len; + const records = std.mem.bytesAsSlice([wire.edge_len]u8, bytes[start..end]); + std.mem.sort([wire.edge_len]u8, records, {}, wireEdgeLess); + } + } + try wire.finishEncoding(self.alloc, bytes, body_len, wire.topologyDirectorySize(self.kinds.values.keys(), self.nodes.values.count(), body_len + local_edges * 8, 0), cancellation); + return bytes; + } +}; + +fn writeEdge(bytes: []u8, pos: *usize, target: u32, kind: u32, weight: f32, table: u32) void { + put(bytes, pos, target); + put(bytes, pos, kind); + put(bytes, pos, @bitCast(weight)); + put(bytes, pos, table); +} + +fn wireEdgeLess(_: void, a: [wire.edge_len]u8, b: [wire.edge_len]u8) bool { + const kind_a = std.mem.readInt(u32, a[4..8], .little); + const kind_b = std.mem.readInt(u32, b[4..8], .little); + if (kind_a != kind_b) return kind_a < kind_b; + const target_a = std.mem.readInt(u32, a[0..4], .little); + const target_b = std.mem.readInt(u32, b[0..4], .little); + if (target_a != target_b) return target_a < target_b; + const weight_a: f32 = @bitCast(std.mem.readInt(u32, a[8..12], .little)); + const weight_b: f32 = @bitCast(std.mem.readInt(u32, b[8..12], .little)); + if (weight_a != weight_b) return weight_a < weight_b; + return std.mem.readInt(u32, a[12..16], .little) < std.mem.readInt(u32, b[12..16], .little); +} + +fn invert(alloc: Allocator, order: []const u32) ![]u32 { + const result = try alloc.alloc(u32, order.len); + for (order, 0..) |old, new| result[old] = @intCast(new); + return result; +} + +fn put(bytes: []u8, pos: *usize, value: u32) void { + std.mem.writeInt(u32, bytes[pos.*..][0..4], value, .little); + pos.* += 4; +} + +test "serverless ordinal graph builder owns identifiers and canonicalizes directions and qualified endpoints" { + const alloc = std.testing.allocator; + var first = Builder{ .alloc = alloc }; + defer first.deinit(); + var second = Builder{ .alloc = alloc }; + defer second.deinit(); + try first.addEdge("b", "a", "link", 1, null); + try first.addEdge("a", "remote", "link", 2, "other"); + try first.addNode("isolated"); + try second.addNode("isolated"); + try second.addEdge("a", "remote", "link", 2, "other"); + try second.addEdge("b", "a", "link", 1, null); + const a = try first.encodeAlloc(4096, .none); + defer alloc.free(a); + const b = try second.encodeAlloc(4096, .none); + defer alloc.free(b); + try std.testing.expectEqualSlices(u8, a, b); + var view = try wire.viewAlloc(alloc, a, .{}, .none); + defer view.deinit(alloc); + try std.testing.expectEqual(@as(usize, 3), view.adjacencies.len); + try std.testing.expectEqual(@as(usize, 4), view.nodes.len); + try std.testing.expectEqual(@as(usize, wire.edge_len), view.adjacencies[0].in.len); + try std.testing.expectError(error.GraphSegmentTooLarge, first.encodeAlloc(a.len - 1, .none)); +} + +test "serverless ordinal graph builder packs skewed duplicate and qualified adjacency canonically" { + const alloc = std.testing.allocator; + var forward = Builder{ .alloc = alloc }; + defer forward.deinit(); + var reverse = Builder{ .alloc = alloc }; + defer reverse.deinit(); + const nodes = [_][]const u8{ "hub", "a", "b", "remote" }; + const kinds = [_][]const u8{ "z", "a" }; + for (0..512) |i| { + const j = 511 - i; + // A high-degree hub, self-loops, repeated edges, negative weights, + // multiple types and qualified endpoints exercise both orientations. + try forward.addEdge("hub", nodes[i % nodes.len], kinds[i % kinds.len], @as(f32, @floatFromInt(i % 7)) - 3, if (i % 5 == 0) "other" else null); + try reverse.addEdge("hub", nodes[j % nodes.len], kinds[j % kinds.len], @as(f32, @floatFromInt(j % 7)) - 3, if (j % 5 == 0) "other" else null); + } + const a = try forward.encodeAlloc(64 * 1024, .none); + defer alloc.free(a); + const b = try reverse.encodeAlloc(64 * 1024, .none); + defer alloc.free(b); + try std.testing.expectEqualSlices(u8, a, b); + var view = try wire.viewAlloc(alloc, a, .{}, .none); + defer view.deinit(alloc); + var outgoing: usize = 0; + var incoming: usize = 0; + for (view.adjacencies) |adjacency| { + outgoing += adjacency.out.len / wire.edge_len; + incoming += adjacency.in.len / wire.edge_len; + } + try std.testing.expectEqual(@as(usize, 512), outgoing); + try std.testing.expectEqual(@as(usize, 409), incoming); +} + +test "serverless ordinal graph builder allocation failure is recoverable by destruction" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, struct { + fn run(alloc: Allocator) !void { + var builder = Builder{ .alloc = alloc }; + defer builder.deinit(); + try builder.addEdge("source", "target", "link", 1, null); + try builder.addEdge("source", "remote", "link", 2, "table"); + const bytes = try builder.encodeAlloc(4096, .none); + defer alloc.free(bytes); + } + }.run, .{}); +} diff --git a/zig/pkg/antfly/src/serverless/graph_segment/codec.zig b/zig/pkg/antfly/src/serverless/graph_segment/codec.zig index 0af3931846..3d7596bbfb 100644 --- a/zig/pkg/antfly/src/serverless/graph_segment/codec.zig +++ b/zig/pkg/antfly/src/serverless/graph_segment/codec.zig @@ -14,92 +14,26 @@ const std = @import("std"); const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const graph_types = @import("types.zig"); const graph_edge_type = @import("../../graph/edge_type.zig"); -const bounded_decode = @import("../bounded_decode.zig"); - -pub const DecodeLimits = bounded_decode.Limits; - -pub const wire_magic = "AFSG"; -pub const wire_version: u16 = 2; - -const header_len = 4 + 2 + 4 + 4; -const no_neighbor_table = std.math.maxInt(u32); - -fn edgeEncodedSize(edge: graph_types.Edge) !usize { - _ = std.math.cast(u32, edge.neighbor_id.len) orelse return error.GraphSegmentTooLarge; - graph_edge_type.validateStored(edge.edge_type) catch return error.InvalidGraphSegment; - _ = std.math.cast(u32, edge.edge_type.len) orelse return error.GraphSegmentTooLarge; - var size: usize = 16; - size = std.math.add(usize, size, edge.neighbor_id.len) catch return error.GraphSegmentTooLarge; - return std.math.add(usize, size, edge.edge_type.len) catch error.GraphSegmentTooLarge; +pub const compact = @import("packed.zig"); +pub const DecodeLimits = @import("../bounded_decode.zig").Limits; +pub const wire_magic = compact.wire_magic; +pub const wire_version = compact.wire_version; +const header_len = compact.header_len; +pub const encodeAlloc = compact.encodeAlloc; +pub const encodedSize = compact.encodedSize; +pub const decodedRetainedBytes = compact.decodedRetainedBytes; +pub const decodeAllocWithLimitsAndCancellation = compact.decodeAllocWithLimitsAndCancellation; +pub fn decodeAlloc(alloc: Allocator, data: []const u8) !graph_types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, .{}, .none); } - -pub fn encodeAlloc(alloc: Allocator, segment: graph_types.Segment) ![]u8 { - const size = try encodedSize(segment); - - const buf = try alloc.alloc(u8, size); - errdefer alloc.free(buf); - - var pos: usize = 0; - @memcpy(buf[pos..][0..4], wire_magic); - pos += 4; - std.mem.writeInt(u16, buf[pos..][0..2], wire_version, .little); - pos += 2; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(segment.neighbor_tables.len), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(segment.adjacencies.len), .little); - pos += 4; - - for (segment.neighbor_tables) |table| { - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(table.len), .little); - pos += 4; - @memcpy(buf[pos..][0..table.len], table); - pos += table.len; - } - - for (segment.adjacencies) |adjacency| { - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(adjacency.node_id.len), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(adjacency.out_edges.len), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(adjacency.in_edges.len), .little); - pos += 4; - @memcpy(buf[pos..][0..adjacency.node_id.len], adjacency.node_id); - pos += adjacency.node_id.len; - for (adjacency.out_edges) |edge| pos += encodeEdge(buf[pos..], edge); - for (adjacency.in_edges) |edge| pos += encodeEdge(buf[pos..], edge); - } - - std.debug.assert(pos == buf.len); - return buf; +pub fn decodeAllocWithLimits(alloc: Allocator, data: []const u8, limits: DecodeLimits) !graph_types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, limits, .none); } - -pub fn encodedSize(segment: graph_types.Segment) !usize { - _ = std.math.cast(u32, segment.neighbor_tables.len) orelse return error.GraphSegmentTooLarge; - _ = std.math.cast(u32, segment.adjacencies.len) orelse return error.GraphSegmentTooLarge; - var size: usize = header_len; - for (segment.neighbor_tables) |table| { - _ = std.math.cast(u32, table.len) orelse return error.GraphSegmentTooLarge; - size = std.math.add(usize, size, 4) catch return error.GraphSegmentTooLarge; - size = std.math.add(usize, size, table.len) catch return error.GraphSegmentTooLarge; - } - for (segment.adjacencies) |adjacency| { - _ = std.math.cast(u32, adjacency.node_id.len) orelse return error.GraphSegmentTooLarge; - _ = std.math.cast(u32, adjacency.out_edges.len) orelse return error.GraphSegmentTooLarge; - _ = std.math.cast(u32, adjacency.in_edges.len) orelse return error.GraphSegmentTooLarge; - size = std.math.add(usize, size, 12) catch return error.GraphSegmentTooLarge; - size = std.math.add(usize, size, adjacency.node_id.len) catch return error.GraphSegmentTooLarge; - for (adjacency.out_edges) |edge| { - if (edge.neighbor_table_id) |id| if (id >= segment.neighbor_tables.len) return error.InvalidGraphSegment; - size = std.math.add(usize, size, try edgeEncodedSize(edge)) catch return error.GraphSegmentTooLarge; - } - for (adjacency.in_edges) |edge| { - if (edge.neighbor_table_id) |id| if (id >= segment.neighbor_tables.len) return error.InvalidGraphSegment; - size = std.math.add(usize, size, try edgeEncodedSize(edge)) catch return error.GraphSegmentTooLarge; - } - } - return size; +pub fn decodeAllocWithCancellation(alloc: Allocator, data: []const u8, cancellation: CancellationToken) !graph_types.Segment { + return decodeAllocWithLimitsAndCancellation(alloc, data, .{}, cancellation); } test "lake graph segment codec rejects forged adjacency counts before allocation" { @@ -107,259 +41,33 @@ test "lake graph segment codec rejects forged adjacency counts before allocation @memcpy(payload[0..4], wire_magic); std.mem.writeInt(u16, payload[4..6], wire_version, .little); std.mem.writeInt(u32, payload[6..10], 0, .little); - std.mem.writeInt(u32, payload[10..14], std.math.maxInt(u32), .little); + std.mem.writeInt(u32, payload[18..22], std.math.maxInt(u32), .little); try std.testing.expectError(error.InvalidGraphSegment, decodeAlloc(std.testing.allocator, &payload)); } -pub fn decodeAlloc(alloc: Allocator, data: []const u8) !graph_types.Segment { - return try decodeAllocWithLimits(alloc, data, .{}); -} - -/// Exact owned bytes requested by decodeAlloc for a canonical v2 artifact. -/// This allocation-free preflight lets request budgets reject a segment before -/// any decoded state reaches the backing allocator. -pub fn decodedRetainedBytes(data: []const u8) !usize { - if (data.len < header_len or !std.mem.eql(u8, data[0..4], wire_magic)) - return error.InvalidGraphSegment; - if (std.mem.readInt(u16, data[4..6], .little) != wire_version) - return error.UnsupportedGraphSegmentVersion; - var pos: usize = 6; - const table_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - const adjacency_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - var retained = std.math.mul(usize, table_count, @sizeOf([]u8)) catch - return error.InvalidGraphSegment; - for (0..table_count) |_| { - if (pos > data.len or data.len - pos < 4) return error.InvalidGraphSegment; - const len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (len == 0 or len > data.len - pos) return error.InvalidGraphSegment; - retained = std.math.add(usize, retained, len) catch return error.InvalidGraphSegment; - pos += len; - } - retained = std.math.add( - usize, - retained, - std.math.mul(usize, adjacency_count, @sizeOf(graph_types.Adjacency)) catch - return error.InvalidGraphSegment, - ) catch return error.InvalidGraphSegment; - for (0..adjacency_count) |_| { - if (pos > data.len or data.len - pos < 12) return error.InvalidGraphSegment; - const node_id_len = std.mem.readInt(u32, data[pos..][0..4], .little); - const out_count = std.mem.readInt(u32, data[pos + 4 ..][0..4], .little); - const in_count = std.mem.readInt(u32, data[pos + 8 ..][0..4], .little); - pos += 12; - if (node_id_len > data.len - pos) return error.InvalidGraphSegment; - retained = std.math.add(usize, retained, node_id_len) catch return error.InvalidGraphSegment; - pos += node_id_len; - for ([_]u32{ out_count, in_count }) |edge_count| { - retained = std.math.add( - usize, - retained, - std.math.mul(usize, edge_count, @sizeOf(graph_types.Edge)) catch - return error.InvalidGraphSegment, - ) catch return error.InvalidGraphSegment; - for (0..edge_count) |_| { - if (pos > data.len or data.len - pos < 16) return error.InvalidGraphSegment; - const neighbor_len = std.mem.readInt(u32, data[pos..][0..4], .little); - const type_len = std.mem.readInt(u32, data[pos + 4 ..][0..4], .little); - const table_id = std.mem.readInt(u32, data[pos + 12 ..][0..4], .little); - pos += 16; - if (type_len == 0 or type_len > graph_edge_type.max_bytes or - (table_id != no_neighbor_table and table_id >= table_count)) - return error.InvalidGraphSegment; - const names_len = std.math.add(usize, neighbor_len, type_len) catch - return error.InvalidGraphSegment; - if (names_len > data.len - pos) return error.InvalidGraphSegment; - retained = std.math.add(usize, retained, names_len) catch - return error.InvalidGraphSegment; - pos += names_len; - } - } - } - if (pos != data.len) return error.InvalidGraphSegment; - return retained; -} - -pub fn decodeAllocWithLimits(alloc: Allocator, data: []const u8, limits: DecodeLimits) !graph_types.Segment { - var budget = try bounded_decode.Budget.init(data.len, limits); - var limiter = try bounded_decode.AllocationLimiter.init(alloc, limits.max_allocation_bytes); - return decodeBoundedAlloc(limiter.allocator(), data, &budget) catch |err| { - if (err == error.OutOfMemory and limiter.limit_exceeded) return error.DecodedArtifactTooLarge; - return err; +test "serverless packed graph decode observes cancellation inside dictionary scans" { + const alloc = std.testing.allocator; + var segment = graph_types.Segment{ .adjacencies = try alloc.alloc(graph_types.Adjacency, 257) }; + for (segment.adjacencies, 0..) |*adjacency, i| adjacency.* = .{ + .node_id = try std.fmt.allocPrint(alloc, "node-{d:0>4}", .{i}), + .out_edges = try alloc.alloc(graph_types.Edge, 0), + .in_edges = try alloc.alloc(graph_types.Edge, 0), }; -} - -fn decodeBoundedAlloc(alloc: Allocator, data: []const u8, budget: *bounded_decode.Budget) !graph_types.Segment { - if (data.len < header_len) return error.InvalidGraphSegment; - var pos: usize = 0; - if (!std.mem.eql(u8, data[pos..][0..4], wire_magic)) return error.InvalidGraphSegment; - pos += 4; - const version = std.mem.readInt(u16, data[pos..][0..2], .little); - pos += 2; - if (version != wire_version) return error.UnsupportedGraphSegmentVersion; - const neighbor_table_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + 4 > data.len) return error.InvalidGraphSegment; - const adjacency_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - - const neighbor_tables = try decodeNeighborTablesAlloc(alloc, data, &pos, neighbor_table_count, budget); - errdefer { - for (neighbor_tables) |table| alloc.free(table); - if (neighbor_tables.len > 0) alloc.free(neighbor_tables); - } - if (@as(usize, adjacency_count) > (data.len - pos) / 12) return error.InvalidGraphSegment; - _ = try budget.admitCount(graph_types.Adjacency, adjacency_count, data.len - pos, 12); - - const adjacencies = try alloc.alloc(graph_types.Adjacency, adjacency_count); - errdefer alloc.free(adjacencies); - var initialized: usize = 0; - errdefer { - for (adjacencies[0..initialized]) |*adjacency| adjacency.deinit(alloc); - } - - for (0..adjacency_count) |idx| { - if (pos + 12 > data.len) return error.InvalidGraphSegment; - const node_id_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - const out_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - const in_count = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (node_id_len > data.len - pos) return error.InvalidGraphSegment; - try budget.admitBytes(node_id_len); - const node_id = try alloc.dupe(u8, data[pos .. pos + node_id_len]); - pos += node_id_len; - errdefer alloc.free(node_id); - - const out_edges = try decodeEdgesAlloc(alloc, data, &pos, out_count, neighbor_tables.len, budget); - errdefer { - for (out_edges) |*edge| edge.deinit(alloc); - alloc.free(out_edges); - } - const in_edges = try decodeEdgesAlloc(alloc, data, &pos, in_count, neighbor_tables.len, budget); - errdefer { - for (in_edges) |*edge| edge.deinit(alloc); - alloc.free(in_edges); + defer segment.deinit(alloc); + const payload = try encodeAlloc(alloc, segment); + defer alloc.free(payload); + const State = struct { + calls: usize = 0, + fn cancelled(ptr: *const anyopaque) bool { + const self: *@This() = @ptrCast(@alignCast(@constCast(ptr))); + self.calls += 1; + return self.calls >= 3; } - // Exact public probes use binary lookup. Reject non-canonical artifacts - // at the trust boundary instead of risking an exact-looking false miss. - if (!graph_types.edgesHaveCanonicalLookupOrder(out_edges) or - !graph_types.edgesHaveCanonicalLookupOrder(in_edges)) - return error.InvalidGraphSegment; - - adjacencies[idx] = .{ - .node_id = node_id, - .out_edges = out_edges, - .in_edges = in_edges, - }; - initialized += 1; - } - - if (pos != data.len) return error.InvalidGraphSegment; - return .{ .neighbor_tables = neighbor_tables, .adjacencies = adjacencies }; -} - -fn decodeNeighborTablesAlloc( - alloc: Allocator, - data: []const u8, - pos: *usize, - table_count: u32, - budget: *bounded_decode.Budget, -) ![][]u8 { - if (pos.* > data.len or @as(usize, table_count) > (data.len - pos.*) / 4) return error.InvalidGraphSegment; - _ = try budget.admitCount([]u8, table_count, data.len - pos.*, 4); - const tables = try alloc.alloc([]u8, table_count); - errdefer if (tables.len > 0) alloc.free(tables); - var initialized: usize = 0; - errdefer for (tables[0..initialized]) |table| alloc.free(table); - for (0..table_count) |idx| { - if (pos.* + 4 > data.len) return error.InvalidGraphSegment; - const table_len = std.mem.readInt(u32, data[pos.*..][0..4], .little); - pos.* += 4; - if (table_len == 0 or table_len > data.len - pos.*) return error.InvalidGraphSegment; - try budget.admitBytes(table_len); - tables[idx] = try alloc.dupe(u8, data[pos.* .. pos.* + table_len]); - pos.* += table_len; - initialized += 1; - } - return tables; -} - -fn encodeEdge(buf: []u8, edge: graph_types.Edge) usize { - var pos: usize = 0; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(edge.neighbor_id.len), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], @intCast(edge.edge_type.len), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], @bitCast(edge.weight), .little); - pos += 4; - std.mem.writeInt(u32, buf[pos..][0..4], edge.neighbor_table_id orelse no_neighbor_table, .little); - pos += 4; - @memcpy(buf[pos..][0..edge.neighbor_id.len], edge.neighbor_id); - pos += edge.neighbor_id.len; - @memcpy(buf[pos..][0..edge.edge_type.len], edge.edge_type); - pos += edge.edge_type.len; - return pos; -} - -fn decodeEdgesAlloc( - alloc: Allocator, - data: []const u8, - pos: *usize, - edge_count: u32, - neighbor_table_count: usize, - budget: *bounded_decode.Budget, -) ![]graph_types.Edge { - const fixed_edge_len: usize = 16; - if (pos.* > data.len or @as(usize, edge_count) > (data.len - pos.*) / fixed_edge_len) return error.InvalidGraphSegment; - _ = try budget.admitCount(graph_types.Edge, edge_count, data.len - pos.*, fixed_edge_len); - const edges = try alloc.alloc(graph_types.Edge, edge_count); - errdefer alloc.free(edges); - var initialized: usize = 0; - errdefer { - for (edges[0..initialized]) |*edge| edge.deinit(alloc); - } - - for (0..edge_count) |idx| { - if (pos.* + fixed_edge_len > data.len) return error.InvalidGraphSegment; - const neighbor_id_len = std.mem.readInt(u32, data[pos.*..][0..4], .little); - pos.* += 4; - const edge_type_len = std.mem.readInt(u32, data[pos.*..][0..4], .little); - pos.* += 4; - if (edge_type_len == 0 or edge_type_len > graph_edge_type.max_bytes) - return error.InvalidGraphSegment; - const weight_bits = std.mem.readInt(u32, data[pos.*..][0..4], .little); - pos.* += 4; - const raw_id = std.mem.readInt(u32, data[pos.*..][0..4], .little); - pos.* += 4; - const neighbor_table_id: ?u32 = if (raw_id == no_neighbor_table) - null - else if (raw_id < neighbor_table_count) - raw_id - else - return error.InvalidGraphSegment; - const names_len = std.math.add(usize, neighbor_id_len, edge_type_len) catch return error.InvalidGraphSegment; - if (pos.* > data.len or names_len > data.len - pos.*) return error.InvalidGraphSegment; - try budget.admitBytes(names_len); - const neighbor_id = try alloc.dupe(u8, data[pos.* .. pos.* + neighbor_id_len]); - pos.* += neighbor_id_len; - errdefer alloc.free(neighbor_id); - const edge_type_bytes = data[pos.* .. pos.* + edge_type_len]; - if (!graph_edge_type.isValid(edge_type_bytes)) return error.InvalidGraphSegment; - const edge_type = try alloc.dupe(u8, edge_type_bytes); - pos.* += edge_type_len; - edges[idx] = .{ - .neighbor_id = neighbor_id, - .edge_type = edge_type, - .weight = @bitCast(weight_bits), - .neighbor_table_id = neighbor_table_id, - }; - initialized += 1; - } - return edges; + }; + var state = State{}; + const cancellation = CancellationToken{ .ptr = &state, .is_cancelled_fn = State.cancelled }; + try std.testing.expectError(error.Canceled, decodeAllocWithCancellation(alloc, payload, cancellation)); + try std.testing.expectEqual(@as(usize, 3), state.calls); } test "serverless graph segment codec round-trips" { @@ -398,7 +106,7 @@ test "serverless graph segment codec round-trips" { const expected_retained = @sizeOf([]u8) + "entities".len + 2 * @sizeOf(graph_types.Adjacency) + "doc-a".len + "doc-b".len + 2 * @sizeOf(graph_types.Edge) + 2 * ("doc-a".len + "cites".len); - try std.testing.expectEqual(expected_retained, try decodedRetainedBytes(encoded)); + try std.testing.expectEqual(expected_retained, try decodedRetainedBytes(alloc, encoded)); try std.testing.expectEqual(wire_version, std.mem.readInt(u16, encoded[4..6], .little)); var decoded = try decodeAlloc(alloc, encoded); defer graph_types.freeSegment(alloc, &decoded); @@ -439,13 +147,13 @@ test "serverless graph segment codec rejects invalid edge types" { segment.adjacencies[0].out_edges[0].edge_type = try alloc.dupe(u8, "x"); const encoded = try encodeAlloc(alloc, segment); defer alloc.free(encoded); - const edge_type_len_offset = header_len + 12 + segment.adjacencies[0].node_id.len + 4; + const edge_type_len_offset = header_len + 4 + "doc-a".len + 4 + "doc-b".len; std.mem.writeInt(u32, encoded[edge_type_len_offset..][0..4], 0, .little); try std.testing.expectError(error.InvalidGraphSegment, decodeAlloc(alloc, encoded)); std.mem.writeInt(u32, encoded[edge_type_len_offset..][0..4], graph_edge_type.max_bytes + 1, .little); try std.testing.expectError(error.InvalidGraphSegment, decodeAlloc(alloc, encoded)); std.mem.writeInt(u32, encoded[edge_type_len_offset..][0..4], 1, .little); - encoded[encoded.len - 1] = 0xff; + encoded[edge_type_len_offset + 4] = 0xff; try std.testing.expectError(error.InvalidGraphSegment, decodeAlloc(alloc, encoded)); } @@ -470,12 +178,18 @@ test "serverless graph segment codec rejects non-canonical edge ordering" { .edge_type = try alloc.dupe(u8, "cites"), .weight = 1, }; + try std.testing.expectError(error.InvalidGraphSegment, encodeAlloc(alloc, segment)); + std.mem.swap(graph_types.Edge, &segment.adjacencies[0].out_edges[0], &segment.adjacencies[0].out_edges[1]); const encoded = try encodeAlloc(alloc, segment); defer alloc.free(encoded); + var view = try compact.viewAlloc(alloc, encoded, .{}, .none); + defer view.deinit(alloc); + const records = std.mem.bytesAsSlice([compact.edge_len]u8, @constCast(view.adjacencies[0].out)); + std.mem.swap([compact.edge_len]u8, &records[0], &records[1]); try std.testing.expectError(error.InvalidGraphSegment, decodeAlloc(alloc, encoded)); } -test "serverless graph segment codec encodes local artifacts as v2" { +test "serverless graph segment codec encodes local artifacts as packed v6" { const alloc = std.testing.allocator; var segment = graph_types.Segment{ .adjacencies = try alloc.alloc(graph_types.Adjacency, 1), diff --git a/zig/pkg/antfly/src/serverless/graph_segment/mod.zig b/zig/pkg/antfly/src/serverless/graph_segment/mod.zig index 6670a6e49a..4ac0bd1a1f 100644 --- a/zig/pkg/antfly/src/serverless/graph_segment/mod.zig +++ b/zig/pkg/antfly/src/serverless/graph_segment/mod.zig @@ -14,6 +14,8 @@ pub const types = @import("types.zig"); pub const codec = @import("codec.zig"); +pub const Builder = @import("builder.zig").Builder; +pub const AdjacencyReader = @import("adjacency_reader.zig").Reader; pub const Edge = types.Edge; pub const EdgeLookup = types.EdgeLookup; @@ -28,11 +30,15 @@ pub const encodeAlloc = codec.encodeAlloc; pub const encodedSize = codec.encodedSize; pub const decodeAlloc = codec.decodeAlloc; pub const decodeAllocWithLimits = codec.decodeAllocWithLimits; +pub const decodeAllocWithCancellation = codec.decodeAllocWithCancellation; +pub const decodeAllocWithLimitsAndCancellation = codec.decodeAllocWithLimitsAndCancellation; pub const decodedRetainedBytes = codec.decodedRetainedBytes; test "serverless graph segment module compiles" { _ = types; _ = codec; + _ = Builder; + _ = AdjacencyReader; _ = Edge; _ = Adjacency; _ = Segment; @@ -42,5 +48,7 @@ test "serverless graph segment module compiles" { _ = encodedSize; _ = decodeAlloc; _ = decodeAllocWithLimits; + _ = decodeAllocWithCancellation; + _ = decodeAllocWithLimitsAndCancellation; _ = decodedRetainedBytes; } diff --git a/zig/pkg/antfly/src/serverless/graph_segment/packed.zig b/zig/pkg/antfly/src/serverless/graph_segment/packed.zig new file mode 100644 index 0000000000..c6df350a3d --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_segment/packed.zig @@ -0,0 +1,792 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Current graph wire: sorted node/type dictionaries and fixed ordinal edges. +//! Views borrow authenticated bytes; decoding does not allocate per edge. +const std = @import("std"); +const Allocator = std.mem.Allocator; +const types = @import("types.zig"); +const edge_type = @import("../../graph/edge_type.zig"); +const bounded = @import("../bounded_decode.zig"); +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +pub const wire_magic = "AFSG"; +pub const wire_version: u16 = 7; +pub const header_len = 22; +pub const edge_len = 16; +pub const no_table = std.math.maxInt(u32); +pub const topology_trailer_len = 80; +pub const node_page_entries = 256; +pub const node_page_fence_bytes = 68; +pub const authentication_block_bytes = 64 * 1024; +pub const max_topology_directory_bytes = 1024 * 1024; +const absent_directory = std.math.maxInt(u32); + +/// Type descriptors and dictionary page offsets have a bounded control size. +/// The limit depends on directory bytes, not graph-wide hashing scratch; a +/// million-node graph with a small type dictionary retains the accelerator. +pub fn topologyDirectorySize(kinds: []const []const u8, nodes: usize, covered_bytes: usize, tables: usize) usize { + _ = tables; + const pages = nodes / node_page_entries + @intFromBool(nodes % node_page_entries != 0); + var size: usize = 16 +| ((pages + 1) *| 8) +| (pages *| node_page_fence_bytes); + const covered = covered_bytes +| (nodes *| 8); + size +|= ((covered / authentication_block_bytes + @intFromBool(covered % authentication_block_bytes != 0)) *| 32); + for (kinds) |kind| size +|= 52 +| kind.len; + return if (size <= max_topology_directory_bytes) size else 4; +} + +pub fn topologyExtensionSize(kinds: []const []const u8, nodes: usize, edges: usize, body_len: usize) !usize { + const covered = std.math.add(usize, body_len, std.math.mul(usize, edges, 8) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + const directory = topologyDirectorySize(kinds, nodes, covered, 0); + const bytes = if (directory == 4) 0 else std.math.mul(usize, std.math.add(usize, edges, nodes) catch return error.GraphSegmentTooLarge, 8) catch return error.GraphSegmentTooLarge; + return std.math.add(usize, bytes, directory + topology_trailer_len) catch error.GraphSegmentTooLarge; +} + +pub const TopologyTrailer = struct { + body_len: u64, + directory_len: u32, + checksum: [32]u8, + source_nodes: u32, + source_edges: u64, + topology_len: u64, + adjacency_index_len: u64, + + pub fn directoryOffset(self: @This()) u64 { + return self.body_len + self.topology_len + self.adjacency_index_len; + } +}; + +pub fn bindTopologyControl(ref: anytype, payload: []const u8) !void { + if (payload.len < header_len + topology_trailer_len or + !std.mem.eql(u8, payload[0..4], wire_magic) or + std.mem.readInt(u16, payload[4..6], .little) != wire_version) return error.InvalidGraphSegment; + const footer = payload[payload.len - topology_trailer_len ..]; + _ = try decodeTopologyTrailer(footer, payload.len); + std.crypto.hash.sha2.Sha256.hash(footer, &ref.graph_topology_control_checksum, .{}); +} + +pub fn decodeTopologyTrailer(raw: []const u8, payload_len: u64) !TopologyTrailer { + if (raw.len != topology_trailer_len or !std.mem.eql(u8, raw[0..4], "GTD3")) return error.InvalidGraphSegment; + const dir_len = std.mem.readInt(u32, raw[4..8], .little); + const body_len = std.mem.readInt(u64, raw[8..16], .little); + const topology_len = std.mem.readInt(u64, raw[64..72], .little); + const adjacency_index_len = std.mem.readInt(u64, raw[72..80], .little); + if (dir_len < 4 or dir_len > max_topology_directory_bytes or body_len < header_len or + body_len > payload_len or topology_len > payload_len - body_len or topology_len % 8 != 0 or + adjacency_index_len > payload_len - body_len - topology_len or adjacency_index_len % 8 != 0 or + payload_len - body_len - topology_len - adjacency_index_len != @as(u64, dir_len) + topology_trailer_len) return error.InvalidGraphSegment; + if (!std.mem.eql(u8, raw[60..64], &.{ 0, 0, 0, 0 })) return error.InvalidGraphSegment; + return .{ .body_len = body_len, .topology_len = topology_len, .adjacency_index_len = adjacency_index_len, .directory_len = dir_len, .checksum = raw[16..48].*, .source_nodes = std.mem.readInt(u32, raw[48..52], .little), .source_edges = std.mem.readInt(u64, raw[52..60], .little) }; +} + +test "serverless graph topology directory is bounded authenticated and distinguishes empty from unavailable" { + const alloc = std.testing.allocator; + const Filter = struct { mode: enum { all, types } = .all, types: []const []const u8 = &.{} }; + const payload = try encodeAlloc(alloc, .{ .adjacencies = &.{} }); + defer alloc.free(payload); + const trailer = try decodeTopologyTrailer(payload[payload.len - topology_trailer_len ..], payload.len); + const raw = payload[@intCast(trailer.directoryOffset())..][0..trailer.directory_len]; + const digest = (try selectedDirectoryChecksum(raw, trailer.checksum, Filter{})).?; + const empty = (try selectedDirectoryChecksum(raw, trailer.checksum, Filter{ .mode = .types, .types = &.{"absent"} })).?; + try std.testing.expectEqualSlices(u8, &digest, &empty); + raw[0] ^= 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, selectedDirectoryChecksum(raw, trailer.checksum, Filter{})); + std.mem.writeInt(u32, raw[0..4], absent_directory, .little); + @memset(raw[4..], 0); + var checksum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(raw, &checksum, .{}); + try std.testing.expect((try selectedDirectoryChecksum(raw, checksum, Filter{})) == null); + try std.testing.expect(topologyDirectorySize(&.{"link"}, 1024 * 1024, 1024 * 1024, 0) > 4); + try std.testing.expect(topologyDirectorySize(&.{"link"}, 1, 1, 8 * 1024 * 1024) > 4); + const long_kind = try alloc.alloc(u8, max_topology_directory_bytes); + defer alloc.free(long_kind); + try std.testing.expectEqual(@as(usize, 4), topologyDirectorySize(&.{long_kind}, 1, 1, 0)); + // Forged sizes are rejected before any range allocation or request. + const footer = payload[payload.len - topology_trailer_len ..]; + std.mem.writeInt(u32, footer[4..8], max_topology_directory_bytes + 1, .little); + try std.testing.expectError(error.InvalidGraphSegment, decodeTopologyTrailer(footer, payload.len)); +} + +/// Authenticated type runs and addressable pages of the original node +/// dictionary. The directory is small; edge data is never copied into it. +pub const TypeEntry = struct { kind: []const u8, edges: u64, digest: [32]u8, offset: u64 }; +pub const TypeIterator = struct { + bytes: []const u8, + pos: usize = 0, + pub fn next(self: *@This()) !?TypeEntry { + if (self.pos == self.bytes.len) return null; + const tail = self.bytes[self.pos..]; + if (tail.len < 52) return error.InvalidGraphSegment; + const len = std.mem.readInt(u32, tail[0..4], .little); + if (len > tail.len - 52) return error.InvalidGraphSegment; + const meta = tail[4 + len ..]; + self.pos += 52 + len; + return .{ .kind = tail[4..][0..len], .edges = std.mem.readInt(u64, meta[0..8], .little), .digest = meta[8..40].*, .offset = std.mem.readInt(u64, meta[40..48], .little) }; + } +}; +pub const TopologyDirectory = struct { + nodes: u32, + page_offsets: []const u8, + page_fences: []const u8, + block_checksums: []const u8, + entries: []const u8, + pub fn iterator(self: @This()) TypeIterator { + return .{ .bytes = self.entries }; + } + pub fn nodePage(self: @This(), page: usize) !struct { offset: u64, len: u64 } { + if (page + 1 >= self.page_offsets.len / 8) return error.InvalidGraphSegment; + const begin = std.mem.readInt(u64, self.page_offsets[page * 8 ..][0..8], .little); + const end = std.mem.readInt(u64, self.page_offsets[(page + 1) * 8 ..][0..8], .little); + if (end < begin) return error.InvalidGraphSegment; + return .{ .offset = begin, .len = end - begin }; + } + pub fn init(raw: []const u8, expected: [32]u8) !?@This() { + if (raw.len < 4 or raw.len > max_topology_directory_bytes) return error.InvalidGraphSegment; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(raw, &digest, .{}); + if (!std.mem.eql(u8, &digest, &expected)) return error.ArtifactIntegrityMismatch; + const count = std.mem.readInt(u32, raw[0..4], .little); + if (count == absent_directory) { + for (raw[4..]) |byte| if (byte != 0) return error.InvalidGraphSegment; + return null; + } + if (raw.len < 24) return error.InvalidGraphSegment; + const nodes = std.mem.readInt(u32, raw[4..8], .little); + const pages = std.mem.readInt(u32, raw[8..12], .little); + if (pages != nodes / node_page_entries + @intFromBool(nodes % node_page_entries != 0) or + @as(u64, pages) + 1 > (raw.len - 16) / 8) return error.InvalidGraphSegment; + const offsets_end = 16 + (@as(usize, pages) + 1) * 8; + if (pages > (raw.len - offsets_end) / node_page_fence_bytes) return error.InvalidGraphSegment; + const end = offsets_end + @as(usize, pages) * node_page_fence_bytes; + const blocks = std.mem.readInt(u32, raw[12..16], .little); + if (blocks > (raw.len - end) / 32) return error.InvalidGraphSegment; + const checksums_end = end + @as(usize, blocks) * 32; + const result = @This(){ .nodes = nodes, .page_offsets = raw[16..offsets_end], .page_fences = raw[offsets_end..end], .block_checksums = raw[end..checksums_end], .entries = raw[checksums_end..] }; + var previous_fence: ?[]const u8 = null; + for (0..pages) |page| { + _ = try result.nodePage(page); + const fence = result.page_fences[page * node_page_fence_bytes ..][0..node_page_fence_bytes]; + const len = @min(std.mem.readInt(u32, fence[0..4], .little), 64); + const prefix = fence[4..][0..len]; + if (previous_fence) |prior| if (std.mem.order(u8, prior, prefix) == .gt) return error.InvalidGraphSegment; + previous_fence = prefix; + for (fence[4 + len ..]) |byte| if (byte != 0) return error.InvalidGraphSegment; + } + var entries = result.iterator(); + var previous: ?[]const u8 = null; + var previous_end: ?u64 = null; + var seen: u32 = 0; + while (try entries.next()) |entry| { + if (!edge_type.isValid(entry.kind)) return error.InvalidGraphSegment; + if (previous) |name| if (std.mem.order(u8, name, entry.kind) != .lt) return error.InvalidGraphSegment; + if (previous_end) |offset| if (entry.offset != offset) return error.InvalidGraphSegment; + previous_end = std.math.add(u64, entry.offset, std.math.mul(u64, entry.edges, 8) catch return error.InvalidGraphSegment) catch return error.InvalidGraphSegment; + previous = entry.kind; + seen = std.math.add(u32, seen, 1) catch return error.InvalidGraphSegment; + } + if (seen != count) return error.InvalidGraphSegment; + return result; + } +}; + +pub fn selectedDirectoryChecksum(raw: []const u8, expected: [32]u8, filter: anytype) !?[32]u8 { + const directory = (try TopologyDirectory.init(raw, expected)) orelse return null; + var entries = directory.iterator(); + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update("antfly:selected-unweighted-topology:v1"); + while (try entries.next()) |entry| { + const selected = filter.mode == .all or for (filter.types) |name| { + if (std.mem.eql(u8, name, entry.kind)) break true; + } else false; + if (entry.edges > 0 and selected) hash.update(&entry.digest); + } + return hash.finalResult(); +} + +/// Stream adjacency twice using compact node offsets and a bounded hash cache. +/// No adjacency view or graph-wide digest array coexists with the encoder. +/// Topology edges scatter directly into their final immutable type runs. +pub fn finishEncoding(alloc: Allocator, payload: []u8, body_len: usize, directory_len: usize, cancellation: CancellationToken) !void { + if (payload.len < topology_trailer_len or directory_len > payload.len - topology_trailer_len or body_len < header_len or body_len > payload.len - topology_trailer_len - directory_len) return error.InvalidGraphSegment; + const directory_start = payload.len - topology_trailer_len - directory_len; + const directory = payload[directory_start..][0..directory_len]; + const adjacency_index_len: usize = if (directory_len == 4) 0 else @as(usize, std.mem.readInt(u32, payload[10..14], .little)) * 8; + if (adjacency_index_len > directory_start - body_len) return error.InvalidGraphSegment; + const routing_start = directory_start - adjacency_index_len; + @memset(payload[routing_start..directory_start], 0); + const topology_len = routing_start - body_len; + const type_count = std.mem.readInt(u32, payload[14..18], .little); + var source_edges: u64 = 0; + if (directory.len == 4) { + std.mem.writeInt(u32, directory[0..4], absent_directory, .little); + } else { + const node_count = std.mem.readInt(u32, payload[10..14], .little); + const table_count = std.mem.readInt(u32, payload[6..10], .little); + const row_count = std.mem.readInt(u32, payload[18..22], .little); + const offsets = try alloc.alloc(usize, node_count); + defer alloc.free(offsets); + const local = try alloc.alloc(bool, node_count); + defer alloc.free(local); + @memset(local, false); + const State = struct { kind: []const u8, count: u64 = 0, hash: std.crypto.hash.sha2.Sha256 = undefined, start: usize = 0, cursor: usize = 0 }; + const states = try alloc.alloc(State, type_count); + defer alloc.free(states); + var cursor = Cursor{ .bytes = payload[0..body_len] }; + for (0..table_count) |_| { + const name = try cursor.take(try cursor.int()); + if (name.len == 0) return error.InvalidGraphSegment; + } + const pages = node_count / node_page_entries + @intFromBool(node_count % node_page_entries != 0); + const fences_start = 16 + (@as(usize, pages) + 1) * 8; + const fences = directory[fences_start..][0 .. @as(usize, pages) * node_page_fence_bytes]; + @memset(fences, 0); + var pos: usize = 0; + put(directory, &pos, type_count); + put(directory, &pos, node_count); + put(directory, &pos, pages); + const block_count = directory_start / authentication_block_bytes + @intFromBool(directory_start % authentication_block_bytes != 0); + put(directory, &pos, @intCast(block_count)); + var prior_node: ?[]const u8 = null; + for (offsets, 0..) |*offset, i| { + if (i % node_page_entries == 0) { + try cancellation.check(); + std.mem.writeInt(u64, directory[pos..][0..8], cursor.pos, .little); + pos += 8; + } + offset.* = cursor.pos; + const node = try cursor.take(try cursor.int()); + if (i % node_page_entries == 0) { + const fence = fences[i / node_page_entries * node_page_fence_bytes ..][0..node_page_fence_bytes]; + std.mem.writeInt(u32, fence[0..4], @intCast(node.len), .little); + @memcpy(fence[4..][0..@min(node.len, 64)], node[0..@min(node.len, 64)]); + } + if (prior_node) |previous| if (std.mem.order(u8, previous, node) != .lt) return error.InvalidGraphSegment; + prior_node = node; + } + std.mem.writeInt(u64, directory[pos..][0..8], cursor.pos, .little); + pos += 8; + pos += fences.len; + const block_checksums_start = pos; + pos += block_count * 32; + for (states, 0..) |*state, i| { + const kind = try cursor.take(try cursor.int()); + if (!edge_type.isValid(kind) or (i > 0 and std.mem.order(u8, states[i - 1].kind, kind) != .lt)) return error.InvalidGraphSegment; + state.* = .{ .kind = kind }; + } + const rows_start = cursor.pos; + var complete = true; + var previous_row: ?u32 = null; + for (0..row_count) |_| { + try cancellation.check(); + const row_offset = cursor.pos; + const node = try cursor.int(); + const outgoing = try cursor.int(); + const incoming = try cursor.int(); + if (node >= node_count) return error.InvalidGraphSegment; + std.mem.writeInt(u64, payload[routing_start + @as(usize, node) * 8 ..][0..8], row_offset, .little); + if (previous_row) |previous| { + if (node <= previous) complete = false; + } + previous_row = node; + if (local[node]) complete = false; + local[node] = true; + source_edges += outgoing; + for ([_]u32{ outgoing, incoming }, 0..) |count, direction| { + const bytes = try cursor.take(std.math.mul(usize, count, edge_len) catch return error.InvalidGraphSegment); + var previous: ?Edge = null; + for (0..count) |i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = readEdge(bytes, i); + if (edge.node >= node_count or edge.edge_type >= type_count or !std.math.isFinite(edge.weight)) return error.InvalidGraphSegment; + if (edge.table) |id| if (id >= table_count) return error.InvalidGraphSegment; + if (previous) |last| if (last.edge_type > edge.edge_type or (last.edge_type == edge.edge_type and + (last.node > edge.node or (last.node == edge.node and last.weight > edge.weight)))) return error.InvalidGraphSegment; + previous = edge; + if (direction == 0 and edge.table == null) states[edge.edge_type].count += 1; + } + } + } + if (cursor.pos != body_len) return error.InvalidGraphSegment; + var start = body_len; + for (states) |*state| { + state.start = start; + state.cursor = start; + start = std.math.add(usize, start, std.math.mul(usize, @intCast(state.count), 8) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + state.hash = std.crypto.hash.sha2.Sha256.init(.{}); + state.hash.update("antfly:unweighted-type:v1"); + var value: [8]u8 = undefined; + std.mem.writeInt(u64, &value, state.kind.len, .little); + state.hash.update(&value); + state.hash.update(state.kind); + std.mem.writeInt(u64, &value, state.count, .little); + state.hash.update(&value); + } + if (start != routing_start) return error.InvalidGraphSegment; + const Cache = struct { + const Entry = struct { ordinal: u32 = no_table, digest: [32]u8 = undefined }; + entries: []Entry, + fn hash(self: @This(), bytes: []const u8, positions: []const usize, ordinal: u32) [32]u8 { + const entry = &self.entries[ordinal % self.entries.len]; + if (entry.ordinal != ordinal) { + const offset = positions[ordinal]; + const len = std.mem.readInt(u32, bytes[offset..][0..4], .little); + std.crypto.hash.sha2.Sha256.hash(bytes[offset + 4 ..][0..len], &entry.digest, .{}); + entry.ordinal = ordinal; + } + return entry.digest; + } + }; + const cache = Cache{ .entries = try alloc.alloc(Cache.Entry, @max(1, @min(node_count, 65536))) }; + defer alloc.free(cache.entries); + @memset(cache.entries, .{}); + cursor.pos = rows_start; + for (0..row_count) |_| { + const node = try cursor.int(); + const outgoing = try cursor.int(); + const incoming = try cursor.int(); + const bytes = try cursor.take(@as(usize, outgoing) * edge_len); + const source_hash = cache.hash(payload, offsets, node); + for (0..outgoing) |i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = readEdge(bytes, i); + if (edge.table != null) continue; + if (!local[edge.node]) complete = false; + const state = &states[edge.edge_type]; + state.hash.update(&source_hash); + state.hash.update(&cache.hash(payload, offsets, edge.node)); + put(payload, &state.cursor, node); + put(payload, &state.cursor, edge.node); + } + _ = try cursor.take(@as(usize, incoming) * edge_len); + } + for (states) |*state| { + putString(directory, &pos, state.kind); + std.mem.writeInt(u64, directory[pos..][0..8], state.count, .little); + state.hash.final(directory[pos + 8 ..][0..32]); + std.mem.writeInt(u64, directory[pos + 40 ..][0..8], state.start, .little); + pos += 48; + } + if (pos != directory.len) return error.InvalidGraphSegment; + for (0..block_count) |block| { + try cancellation.check(); + const begin = block * authentication_block_bytes; + std.crypto.hash.sha2.Sha256.hash(payload[begin..@min(directory_start, begin + authentication_block_bytes)], directory[block_checksums_start + block * 32 ..][0..32], .{}); + } + if (!complete) { + @memset(directory, 0); + std.mem.writeInt(u32, directory[0..4], absent_directory, .little); + } + } + const trailer = payload[payload.len - topology_trailer_len ..]; + @memset(trailer, 0); + @memcpy(trailer[0..4], "GTD3"); + std.mem.writeInt(u32, trailer[4..8], @intCast(directory.len), .little); + std.mem.writeInt(u64, trailer[8..16], body_len, .little); + std.crypto.hash.sha2.Sha256.hash(directory, trailer[16..48], .{}); + @memcpy(trailer[48..52], payload[18..22]); + std.mem.writeInt(u64, trailer[52..60], source_edges, .little); + std.mem.writeInt(u64, trailer[64..72], topology_len, .little); + std.mem.writeInt(u64, trailer[72..80], adjacency_index_len, .little); +} + +pub fn viewRetainedBytes(data: []const u8) !usize { + if (data.len < header_len or !std.mem.eql(u8, data[0..4], wire_magic)) return error.InvalidGraphSegment; + if (std.mem.readInt(u16, data[4..6], .little) != wire_version) return error.UnsupportedGraphSegmentVersion; + const strings = @as(u64, std.mem.readInt(u32, data[6..10], .little)) + std.mem.readInt(u32, data[10..14], .little) + std.mem.readInt(u32, data[14..18], .little); + const adjacency_count = std.mem.readInt(u32, data[18..22], .little); + return std.math.cast(usize, strings * @sizeOf([]const u8) + @as(u64, adjacency_count) * @sizeOf(Adjacency)) orelse error.InvalidGraphSegment; +} + +const Dictionary = struct { + map: std.StringHashMapUnmanaged(u32) = .empty, + values: std.ArrayListUnmanaged([]const u8) = .empty, + fn deinit(self: *@This(), alloc: Allocator) void { + self.map.deinit(alloc); + self.values.deinit(alloc); + } + fn add(self: *@This(), alloc: Allocator, value: []const u8) !void { + const entry = try self.map.getOrPut(alloc, value); + if (entry.found_existing) return; + entry.value_ptr.* = std.math.cast(u32, self.values.items.len) orelse return error.GraphSegmentTooLarge; + try self.values.append(alloc, value); + } + fn finish(self: *@This()) void { + std.mem.sort([]const u8, self.values.items, {}, struct { + fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; + } + }.less); + for (self.values.items, 0..) |value, i| self.map.getPtr(value).?.* = @intCast(i); + } +}; + +const Encoding = struct { + nodes: Dictionary = .{}, + edge_types: Dictionary = .{}, + size: usize = header_len, + local_edges: usize = 0, + fn deinit(self: *@This(), alloc: Allocator) void { + self.nodes.deinit(alloc); + self.edge_types.deinit(alloc); + } + fn init(alloc: Allocator, segment: types.Segment, cancellation: CancellationToken) !Encoding { + var plan = Encoding{}; + errdefer plan.deinit(alloc); + _ = std.math.cast(u32, segment.neighbor_tables.len) orelse return error.GraphSegmentTooLarge; + _ = std.math.cast(u32, segment.adjacencies.len) orelse return error.GraphSegmentTooLarge; + for (segment.adjacencies, 0..) |adjacency, ordinal| { + if (ordinal % 256 == 0) try cancellation.check(); + try plan.nodes.add(alloc, adjacency.node_id); + for (adjacency.out_edges) |edge| if (edge.neighbor_table_id == null) { + plan.local_edges = std.math.add(usize, plan.local_edges, 1) catch return error.GraphSegmentTooLarge; + }; + for ([_][]const types.Edge{ adjacency.out_edges, adjacency.in_edges }) |edges| { + _ = std.math.cast(u32, edges.len) orelse return error.GraphSegmentTooLarge; + for (edges, 0..) |edge, i| { + if (i % 4096 == 0) try cancellation.check(); + if (edge.neighbor_table_id) |id| if (id >= segment.neighbor_tables.len) return error.InvalidGraphSegment; + edge_type.validateStored(edge.edge_type) catch return error.InvalidGraphSegment; + try plan.nodes.add(alloc, edge.neighbor_id); + try plan.edge_types.add(alloc, edge.edge_type); + } + plan.size = std.math.add(usize, plan.size, std.math.mul(usize, edges.len, edge_len) catch return error.GraphSegmentTooLarge) catch return error.GraphSegmentTooLarge; + } + plan.size = std.math.add(usize, plan.size, 12) catch return error.GraphSegmentTooLarge; + } + try cancellation.check(); + plan.nodes.finish(); + plan.edge_types.finish(); + try cancellation.check(); + for (segment.neighbor_tables) |table| { + if (table.len == 0) return error.InvalidGraphSegment; + try plan.addStringSize(table); + } + for (plan.nodes.values.items) |value| try plan.addStringSize(value); + for (plan.edge_types.values.items) |value| try plan.addStringSize(value); + return plan; + } + fn addStringSize(self: *@This(), value: []const u8) !void { + _ = std.math.cast(u32, value.len) orelse return error.GraphSegmentTooLarge; + self.size = std.math.add(usize, self.size, 4) catch return error.GraphSegmentTooLarge; + self.size = std.math.add(usize, self.size, value.len) catch return error.GraphSegmentTooLarge; + } +}; + +pub fn encodedSize(alloc: Allocator, segment: types.Segment) !usize { + var plan = try Encoding.init(alloc, segment, .none); + defer plan.deinit(alloc); + return std.math.add(usize, plan.size, try topologyExtensionSize(plan.edge_types.values.items, plan.nodes.values.items.len, plan.local_edges, plan.size)) catch error.GraphSegmentTooLarge; +} + +fn put(buf: []u8, pos: *usize, value: u32) void { + std.mem.writeInt(u32, buf[pos.*..][0..4], value, .little); + pos.* += 4; +} +fn putString(buf: []u8, pos: *usize, value: []const u8) void { + put(buf, pos, @intCast(value.len)); + @memcpy(buf[pos.*..][0..value.len], value); + pos.* += value.len; +} + +pub fn encodeAlloc(alloc: Allocator, segment: types.Segment) ![]u8 { + return encodeAllocWithLimit(alloc, segment, std.math.maxInt(usize), .none); +} + +/// Build dictionaries once and enforce the output cap before allocating bytes. +pub fn encodeAllocWithLimit(alloc: Allocator, segment: types.Segment, max_bytes: usize, cancellation: CancellationToken) ![]u8 { + var plan = try Encoding.init(alloc, segment, cancellation); + defer plan.deinit(alloc); + const size = std.math.add(usize, plan.size, try topologyExtensionSize(plan.edge_types.values.items, plan.nodes.values.items.len, plan.local_edges, plan.size)) catch return error.GraphSegmentTooLarge; + if (size > max_bytes) return error.GraphSegmentTooLarge; + const buf = try alloc.alloc(u8, size); + errdefer alloc.free(buf); + @memcpy(buf[0..4], wire_magic); + std.mem.writeInt(u16, buf[4..6], wire_version, .little); + var pos: usize = 6; + put(buf, &pos, @intCast(segment.neighbor_tables.len)); + put(buf, &pos, @intCast(plan.nodes.values.items.len)); + put(buf, &pos, @intCast(plan.edge_types.values.items.len)); + put(buf, &pos, @intCast(segment.adjacencies.len)); + for (segment.neighbor_tables) |table| putString(buf, &pos, table); + for (plan.nodes.values.items) |node| putString(buf, &pos, node); + for (plan.edge_types.values.items) |value| putString(buf, &pos, value); + for (segment.adjacencies, 0..) |adjacency, ordinal| { + if (ordinal % 256 == 0) try cancellation.check(); + put(buf, &pos, plan.nodes.map.get(adjacency.node_id).?); + put(buf, &pos, @intCast(adjacency.out_edges.len)); + put(buf, &pos, @intCast(adjacency.in_edges.len)); + for ([_][]const types.Edge{ adjacency.out_edges, adjacency.in_edges }) |edges| for (edges, 0..) |edge, i| { + if (i % 4096 == 0) try cancellation.check(); + put(buf, &pos, plan.nodes.map.get(edge.neighbor_id).?); + put(buf, &pos, plan.edge_types.map.get(edge.edge_type).?); + put(buf, &pos, @bitCast(edge.weight)); + put(buf, &pos, edge.neighbor_table_id orelse no_table); + }; + } + std.debug.assert(pos == plan.size); + try finishEncoding(alloc, buf, plan.size, topologyDirectorySize(plan.edge_types.values.items, plan.nodes.values.items.len, plan.size + plan.local_edges * 8, 0), cancellation); + return buf; +} + +pub const Edge = struct { + node: u32, + edge_type: u32, + weight: f32, + table: ?u32, +}; +pub fn readEdge(bytes: []const u8, index: usize) Edge { + const row = bytes[index * edge_len ..][0..edge_len]; + const table = std.mem.readInt(u32, row[12..16], .little); + return .{ .node = std.mem.readInt(u32, row[0..4], .little), .edge_type = std.mem.readInt(u32, row[4..8], .little), .weight = @bitCast(std.mem.readInt(u32, row[8..12], .little)), .table = if (table == no_table) null else table }; +} +pub const Adjacency = struct { node: u32, out: []const u8, in: []const u8 }; +pub const View = struct { + tables: []const []const u8, + nodes: []const []const u8, + edge_types: []const []const u8, + adjacencies: []Adjacency, + pub fn deinit(self: *View, alloc: Allocator) void { + alloc.free(self.tables); + alloc.free(self.nodes); + alloc.free(self.edge_types); + alloc.free(self.adjacencies); + self.* = undefined; + } + pub fn retainedBytes(self: View) usize { + return (self.tables.len + self.nodes.len + self.edge_types.len) * @sizeOf([]const u8) + self.adjacencies.len * @sizeOf(Adjacency); + } + pub fn decodedBytes(self: View) !usize { + var size = self.tables.len * @sizeOf([]u8) + self.adjacencies.len * @sizeOf(types.Adjacency); + for (self.tables) |table| size = try std.math.add(usize, size, table.len); + for (self.adjacencies) |adjacency| { + size = try std.math.add(usize, size, self.nodes[adjacency.node].len); + for ([_][]const u8{ adjacency.out, adjacency.in }) |edges| { + size = try std.math.add(usize, size, try std.math.mul(usize, edges.len / edge_len, @sizeOf(types.Edge))); + for (0..edges.len / edge_len) |i| { + const edge = readEdge(edges, i); + size = try std.math.add(usize, size, self.nodes[edge.node].len + self.edge_types[edge.edge_type].len); + } + } + } + return size; + } +}; + +const Cursor = struct { + bytes: []const u8, + pos: usize = header_len, + fn take(self: *@This(), len: usize) ![]const u8 { + if (len > self.bytes.len - self.pos) return error.InvalidGraphSegment; + defer self.pos += len; + return self.bytes[self.pos..][0..len]; + } + fn int(self: *@This()) !u32 { + return std.mem.readInt(u32, (try self.take(4))[0..4], .little); + } + fn strings(self: *@This(), alloc: Allocator, count: u32, sorted: bool, is_type: bool, cancellation: CancellationToken) ![][]const u8 { + if (count > (self.bytes.len - self.pos) / 4) return error.InvalidGraphSegment; + const values = try alloc.alloc([]const u8, count); + errdefer alloc.free(values); + for (values, 0..) |*value, i| { + if (i % 256 == 0) try cancellation.check(); + value.* = try self.take(try self.int()); + if (is_type and !edge_type.isValid(value.*)) return error.InvalidGraphSegment; + if (sorted and i > 0 and std.mem.order(u8, values[i - 1], value.*) != .lt) return error.InvalidGraphSegment; + } + return values; + } +}; + +pub fn viewAlloc(alloc: Allocator, data: []const u8, limits: bounded.Limits, cancellation: CancellationToken) !View { + try cancellation.check(); + _ = try bounded.Budget.init(data.len, limits); + var limiter = try bounded.AllocationLimiter.init(alloc, limits.max_allocation_bytes); + // Check the version before inspecting the current-only extension layout. + if (data.len < 6) return error.InvalidGraphSegment; + if (std.mem.readInt(u16, data[4..6], .little) != wire_version) return error.UnsupportedGraphSegmentVersion; + if (data.len < topology_trailer_len) return error.InvalidGraphSegment; + const trailer = try decodeTopologyTrailer(data[data.len - topology_trailer_len ..], data.len); + const body_len: usize = @intCast(trailer.body_len); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data[@intCast(trailer.directoryOffset()) .. data.len - topology_trailer_len], &digest, .{}); + if (!std.mem.eql(u8, &digest, &trailer.checksum)) return error.InvalidGraphSegment; + return readView(limiter.allocator(), data[0..body_len], limits.max_elements, cancellation) catch |err| { + if (err == error.OutOfMemory and limiter.limit_exceeded) return error.DecodedArtifactTooLarge; + return err; + }; +} + +fn readView(alloc: Allocator, data: []const u8, max_elements: usize, cancellation: CancellationToken) !View { + if (data.len < 6 or !std.mem.eql(u8, data[0..4], wire_magic)) return error.InvalidGraphSegment; + if (std.mem.readInt(u16, data[4..6], .little) != wire_version) return error.UnsupportedGraphSegmentVersion; + if (data.len < header_len) return error.InvalidGraphSegment; + const table_count = std.mem.readInt(u32, data[6..10], .little); + const node_count = std.mem.readInt(u32, data[10..14], .little); + const type_count = std.mem.readInt(u32, data[14..18], .little); + const adjacency_count = std.mem.readInt(u32, data[18..22], .little); + if (@as(u64, table_count) + node_count + type_count > (data.len - header_len) / 4 or + adjacency_count > (data.len - header_len) / 12) return error.InvalidGraphSegment; + var elements: u64 = @as(u64, table_count) + node_count + type_count + adjacency_count; + if (elements > max_elements) return error.DecodedArtifactTooLarge; + var cursor = Cursor{ .bytes = data }; + const tables = try cursor.strings(alloc, table_count, false, false, cancellation); + errdefer alloc.free(tables); + for (tables) |table| if (table.len == 0) return error.InvalidGraphSegment; + const nodes = try cursor.strings(alloc, node_count, true, false, cancellation); + errdefer alloc.free(nodes); + const edge_types = try cursor.strings(alloc, type_count, true, true, cancellation); + errdefer alloc.free(edge_types); + if (adjacency_count > (data.len - cursor.pos) / 12) return error.InvalidGraphSegment; + const adjacencies = try alloc.alloc(Adjacency, adjacency_count); + errdefer alloc.free(adjacencies); + for (adjacencies, 0..) |*adjacency, i| { + if (i % 256 == 0) try cancellation.check(); + const node = try cursor.int(); + const out_count = try cursor.int(); + const in_count = try cursor.int(); + if (node >= nodes.len) return error.InvalidGraphSegment; + elements += @as(u64, out_count) + in_count; + if (elements > max_elements) return error.DecodedArtifactTooLarge; + const out = try cursor.take(std.math.mul(usize, out_count, edge_len) catch return error.InvalidGraphSegment); + const in = try cursor.take(std.math.mul(usize, in_count, edge_len) catch return error.InvalidGraphSegment); + for ([_][]const u8{ out, in }) |edges| { + var previous: ?Edge = null; + for (0..edges.len / edge_len) |e| { + if (e % 4096 == 0) try cancellation.check(); + const edge = readEdge(edges, e); + if (edge.node >= nodes.len or edge.edge_type >= edge_types.len or !std.math.isFinite(edge.weight)) return error.InvalidGraphSegment; + if (edge.table) |id| if (id >= tables.len) return error.InvalidGraphSegment; + if (previous) |prior| { + // Dictionaries are sorted, so canonical order is numeric. + const order = std.math.order(prior.edge_type, edge.edge_type); + const node_order = std.math.order(prior.node, edge.node); + if (order == .gt or (order == .eq and (node_order == .gt or (node_order == .eq and prior.weight > edge.weight)))) return error.InvalidGraphSegment; + } + previous = edge; + } + } + adjacency.* = .{ .node = node, .out = out, .in = in }; + } + if (cursor.pos != data.len) return error.InvalidGraphSegment; + return .{ .tables = tables, .nodes = nodes, .edge_types = edge_types, .adjacencies = adjacencies }; +} + +pub fn decodedRetainedBytes(alloc: Allocator, data: []const u8) !usize { + var view = try viewAlloc(alloc, data, .{}, .none); + defer view.deinit(alloc); + return view.decodedBytes(); +} + +pub fn decodeAllocWithLimitsAndCancellation(alloc: Allocator, data: []const u8, limits: bounded.Limits, cancellation: CancellationToken) !types.Segment { + var view = try viewAlloc(alloc, data, limits, cancellation); + defer view.deinit(alloc); + const owned_bytes = try view.decodedBytes(); + if (owned_bytes > limits.max_allocation_bytes -| view.retainedBytes()) return error.DecodedArtifactTooLarge; + return decodeViewAlloc(alloc, view, cancellation); +} + +/// Materialize an already validated view. The caller admits decodedBytes() +/// before this allocation; borrowed view storage remains live until return. +pub fn decodeViewAlloc(alloc: Allocator, view: View, cancellation: CancellationToken) !types.Segment { + const tables = try alloc.alloc([]u8, view.tables.len); + var count: usize = 0; + errdefer { + for (tables[0..count]) |table| alloc.free(table); + alloc.free(tables); + } + for (view.tables, tables) |table, *copy| { + copy.* = try alloc.dupe(u8, table); + count += 1; + } + const adjacencies = try alloc.alloc(types.Adjacency, view.adjacencies.len); + var initialized: usize = 0; + errdefer { + for (adjacencies[0..initialized]) |*adjacency| adjacency.deinit(alloc); + alloc.free(adjacencies); + } + for (view.adjacencies, adjacencies, 0..) |adjacency, *copy, i| { + if (i % 256 == 0) try cancellation.check(); + const node = try alloc.dupe(u8, view.nodes[adjacency.node]); + errdefer alloc.free(node); + const out = try copyEdges(alloc, view, adjacency.out, cancellation); + errdefer { + for (out) |*edge| edge.deinit(alloc); + alloc.free(out); + } + copy.* = .{ .node_id = node, .out_edges = out, .in_edges = try copyEdges(alloc, view, adjacency.in, cancellation) }; + initialized += 1; + } + return .{ .neighbor_tables = tables, .adjacencies = adjacencies }; +} + +fn copyEdges(alloc: Allocator, view: View, bytes: []const u8, cancellation: CancellationToken) ![]types.Edge { + const edges = try alloc.alloc(types.Edge, bytes.len / edge_len); + var initialized: usize = 0; + errdefer { + for (edges[0..initialized]) |*edge| edge.deinit(alloc); + alloc.free(edges); + } + for (edges, 0..) |*copy, i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = readEdge(bytes, i); + const node = try alloc.dupe(u8, view.nodes[edge.node]); + errdefer alloc.free(node); + copy.* = .{ .neighbor_id = node, .edge_type = try alloc.dupe(u8, view.edge_types[edge.edge_type]), .weight = edge.weight, .neighbor_table_id = edge.table }; + initialized += 1; + } + return edges; +} + +test "serverless packed graph ownership and ordinal validation are failure safe" { + const alloc = std.testing.allocator; + const edge = types.Edge{ .neighbor_id = @constCast("b"), .edge_type = @constCast("follows"), .weight = 1 }; + const segment = types.Segment{ .adjacencies = @constCast(&[_]types.Adjacency{ + .{ .node_id = @constCast("a"), .out_edges = @constCast(&[_]types.Edge{edge}), .in_edges = &.{} }, + .{ .node_id = @constCast("b"), .out_edges = &.{}, .in_edges = &.{} }, + }) }; + const Runner = struct { + fn run(failing: Allocator, fixture: types.Segment) !void { + const payload = try encodeAlloc(failing, fixture); + defer failing.free(payload); + var view = try viewAlloc(failing, payload, .{}, .none); + defer view.deinit(failing); + var decoded = try decodeAllocWithLimitsAndCancellation(failing, payload, .{}, .none); + defer decoded.deinit(failing); + try std.testing.expectEqualStrings("b", decoded.adjacencies[0].out_edges[0].neighbor_id); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{segment}); + const payload = try encodeAlloc(alloc, segment); + defer alloc.free(payload); + for (0..payload.len) |len| { + if (viewAlloc(alloc, payload[0..len], .{}, .none)) |valid| { + var owned = valid; + owned.deinit(alloc); + return error.AcceptedTruncatedGraph; + } else |_| {} + } + var view = try viewAlloc(alloc, payload, .{}, .none); + const edge_offset = @intFromPtr(view.adjacencies[0].out.ptr) - @intFromPtr(payload.ptr); + view.deinit(alloc); + for ([_]usize{ 0, 4, 8, 12 }) |field| { + const saved = std.mem.readInt(u32, payload[edge_offset + field ..][0..4], .little); + // NaN for weight; out-of-range node/type/table ordinals otherwise. + const corrupt: u32 = if (field == 8) 0x7fc00000 else std.math.maxInt(u32) - 1; + std.mem.writeInt(u32, payload[edge_offset + field ..][0..4], corrupt, .little); + try std.testing.expectError(error.InvalidGraphSegment, viewAlloc(alloc, payload, .{}, .none)); + std.mem.writeInt(u32, payload[edge_offset + field ..][0..4], saved, .little); + } + try std.testing.expectError(error.DecodedArtifactTooLarge, viewAlloc(alloc, payload, .{ .max_allocation_bytes = 1 }, .none)); + std.mem.writeInt(u16, payload[4..6], 2, .little); + try std.testing.expectError(error.UnsupportedGraphSegmentVersion, viewAlloc(alloc, payload, .{}, .none)); +} diff --git a/zig/pkg/antfly/src/serverless/graph_segment/topology_reader.zig b/zig/pkg/antfly/src/serverless/graph_segment/topology_reader.zig new file mode 100644 index 0000000000..85bf01a90f --- /dev/null +++ b/zig/pkg/antfly/src/serverless/graph_segment/topology_reader.zig @@ -0,0 +1,401 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Selected topology preparation from authenticated current-wire ranges. +//! Scratch and retained data scale with selected edges/endpoints. Unrelated +//! node strings are visited only within the touched dictionary pages. +const std = @import("std"); +const Allocator = std.mem.Allocator; +const wire = @import("packed.zig"); +const artifacts = @import("../artifacts/store.zig"); +const refs = @import("../manifest/artifact_ref.zig"); +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; + +pub const Edge = struct { source: u32, target: u32 }; +pub const Topology = struct { + node_ids: []const []const u8, + edge_types: []const []const u8, + string_bytes: []u8, + edge_type_offsets: []const u32, + edges: []const Edge, + source_node_count: usize, + source_edge_count: usize, + retained_bytes: usize, + type_checksums: []const [32]u8 = &.{}, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.node_ids); + alloc.free(self.edge_types); + alloc.free(self.string_bytes); + alloc.free(self.edge_type_offsets); + alloc.free(self.edges); + alloc.free(self.type_checksums); + self.* = undefined; + } +}; + +const Reader = struct { + alloc: Allocator, + store: *artifacts.ArtifactStore, + source: refs.ArtifactRef, + cancellation: CancellationToken, + remaining: *u64, + + fn raw(self: @This(), offset: u64, len: usize) ![]u8 { + if (len > self.remaining.*) return error.GraphMetricBuildBudgetExceeded; + self.remaining.* -= len; + const bytes = try self.store.getRangeAllocWithCancellationUsingAllocator(self.alloc, self.source.artifact_id, offset, len, self.cancellation); + errdefer self.alloc.free(bytes); + if (bytes.len != len) return error.ArtifactIntegrityMismatch; + return bytes; + } + + fn read(self: @This(), offset: u64, len: u64) ![]u8 { + try self.cancellation.check(); + if (offset > self.source.byte_len or len > self.source.byte_len - offset) return error.InvalidGraphSegment; + return self.store.getVerifiedRangeAllocWithBudget(self.alloc, self.source.artifact_id, self.source.byte_len, self.source.checksum, offset, std.math.cast(usize, len) orelse return error.GraphMetricBuildBudgetExceeded, self.cancellation, self.remaining) catch |err| switch (err) { + error.ArtifactReadBudgetExceeded => error.GraphMetricBuildBudgetExceeded, + else => err, + }; + } +}; + +/// One immutable source control shared by bounded preparation groups. A +/// manifest-bound footer authenticates the directory; the directory binds all +/// data blocks and semantic type identities. No data-range response is trusted. +pub const Context = struct { + reader: Reader, + trailer: wire.TopologyTrailer, + bytes: []u8, + directory: ?wire.TopologyDirectory, + // One authenticated tail block survives adjacent type runs and preparation + // groups. Large ranges remain one GET; only their boundary block is retained. + block_bytes: []u8, + block_offsets: [8]u64 = @splat(0), + block_lens: [8]usize = @splat(0), + cache_slots: usize, + next_slot: usize = 0, + + pub fn init(alloc: Allocator, store: *artifacts.ArtifactStore, source: refs.ArtifactRef, cancellation: CancellationToken, remaining: *u64) !Context { + return initWithCache(alloc, store, source, cancellation, remaining, 1); + } + + pub fn initWithCache(alloc: Allocator, store: *artifacts.ArtifactStore, source: refs.ArtifactRef, cancellation: CancellationToken, remaining: *u64, requested_slots: usize) !Context { + if (requested_slots == 0 or requested_slots > 8) return error.InvalidGraphSegment; + if (source.byte_len < wire.topology_trailer_len) return error.InvalidGraphSegment; + try artifacts.validateSha256ArtifactIdentity(source.artifact_id, source.checksum); + const reader = Reader{ .alloc = alloc, .store = store, .source = source, .cancellation = cancellation, .remaining = remaining }; + const bound = !std.mem.eql(u8, &source.graph_topology_control_checksum, &@as([32]u8, @splat(0))); + const footer = if (bound) try reader.raw(source.byte_len - wire.topology_trailer_len, wire.topology_trailer_len) else try reader.read(source.byte_len - wire.topology_trailer_len, wire.topology_trailer_len); + defer alloc.free(footer); + if (bound) try verify(footer, source.graph_topology_control_checksum); + const trailer = try wire.decodeTopologyTrailer(footer, source.byte_len); + const raw = try reader.raw(trailer.directoryOffset(), trailer.directory_len); + errdefer alloc.free(raw); + const directory = try wire.TopologyDirectory.init(raw, trailer.checksum); + if (directory) |dir| { + const covered = trailer.directoryOffset(); + const blocks = covered / wire.authentication_block_bytes + @intFromBool(covered % wire.authentication_block_bytes != 0); + if (dir.block_checksums.len / 32 != blocks) return error.InvalidGraphSegment; + if (trailer.adjacency_index_len != @as(u64, dir.nodes) * 8) return error.InvalidGraphSegment; + } + const covered = trailer.directoryOffset(); + const slots: usize = if (directory != null) @intCast(@min(requested_slots, (covered + wire.authentication_block_bytes - 1) / wire.authentication_block_bytes)) else 0; + const capacity: usize = if (slots == 1) @intCast(@min(wire.authentication_block_bytes, covered)) else slots * wire.authentication_block_bytes; + const block_bytes = try alloc.alloc(u8, capacity); + return .{ .reader = reader, .trailer = trailer, .bytes = raw, .directory = directory, .block_bytes = block_bytes, .cache_slots = slots }; + } + + pub fn deinit(self: *Context) void { + self.reader.alloc.free(self.bytes); + self.reader.alloc.free(self.block_bytes); + self.* = undefined; + } + + fn verify(bytes: []const u8, checksum: [32]u8) !void { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{}); + if (!std.mem.eql(u8, &digest, &checksum)) return error.ArtifactIntegrityMismatch; + } + + pub fn retainedBytes(self: Context) usize { + return self.bytes.len + self.block_bytes.len; + } + + fn cachedSlot(self: *Context, offset: u64) ?usize { + for (self.block_offsets[0..self.cache_slots], self.block_lens[0..self.cache_slots], 0..) |at, size, i| { + if (at == offset and size != 0) return i; + } + return null; + } + + /// Return an owned exact range, authenticating/coalescing only missing + /// blocks. Cache memory belongs to the context; response memory belongs to + /// the caller's allocator (and therefore its live-memory admission). + pub fn readAlloc(self: *Context, alloc: Allocator, offset: u64, len: u64) ![]u8 { + try self.reader.cancellation.check(); + const covered = self.trailer.directoryOffset(); + if (offset > covered or len > covered - offset) return error.InvalidGraphSegment; + if (len == 0) return alloc.alloc(u8, 0); + if (self.directory == null) return error.InvalidGraphSegment; + const block_bytes = wire.authentication_block_bytes; + var begin = offset / block_bytes * block_bytes; + const end = @min(covered, (offset + len + block_bytes - 1) / block_bytes * block_bytes); + var at = begin; + const all_cached = while (at < offset + len) : (at += block_bytes) { + if (self.cachedSlot(at) == null) break false; + } else true; + if (all_cached) { + const result = try alloc.alloc(u8, @intCast(len)); + at = offset; + var copied: usize = 0; + while (copied < result.len) { + const base = at / block_bytes * block_bytes; + const i = self.cachedSlot(base).?; + const skip: usize = @intCast(at - base); + const count = @min(result.len - copied, self.block_lens[i] - skip); + @memcpy(result[copied..][0..count], self.block_bytes[i * block_bytes + skip ..][0..count]); + copied += count; + at += count; + } + return result; + } + const slot = self.cachedSlot(begin); + const cached: usize = if (slot) |i| @intCast(@min(len, self.block_lens[i] - (offset - begin))) else 0; + const prefix_start: usize = if (slot) |i| i * block_bytes + @as(usize, @intCast(offset - begin)) else 0; + if (cached == len) return alloc.dupe(u8, self.block_bytes[prefix_start..][0..cached]); + const prefix = if (cached != 0) try alloc.dupe(u8, self.block_bytes[prefix_start..][0..cached]) else &.{}; + defer alloc.free(prefix); + if (cached != 0) begin += self.block_lens[slot.?]; + var reader = self.reader; + reader.alloc = alloc; + const bytes = try reader.raw(begin, @intCast(end - begin)); + errdefer alloc.free(bytes); + var pos: usize = 0; + while (pos < bytes.len) : (pos += block_bytes) { + try self.reader.cancellation.check(); + const block: usize = @intCast(begin / block_bytes + pos / block_bytes); + try verify(bytes[pos..@min(bytes.len, pos + block_bytes)], self.directory.?.block_checksums[block * 32 ..][0..32].*); + } + const fetched_blocks = (bytes.len + block_bytes - 1) / block_bytes; + for (fetched_blocks - @min(fetched_blocks, self.cache_slots)..fetched_blocks) |block| { + const start = block * block_bytes; + const size = @min(block_bytes, bytes.len - start); + const tail_slot = self.cachedSlot(begin + start) orelse blk: { + const next = self.next_slot; + self.next_slot = (next + 1) % self.cache_slots; + break :blk next; + }; + @memcpy(self.block_bytes[tail_slot * block_bytes ..][0..size], bytes[start..][0..size]); + self.block_offsets[tail_slot] = begin + start; + self.block_lens[tail_slot] = size; + } + if (cached != 0) { + const result = try alloc.alloc(u8, @intCast(len)); + @memcpy(result[0..cached], prefix); + @memcpy(result[cached..], bytes[0..@intCast(len - cached)]); + alloc.free(bytes); + return result; + } + const start: usize = @intCast(offset - begin); + std.mem.copyForwards(u8, bytes[0..@intCast(len)], bytes[start..][0..@intCast(len)]); + return alloc.realloc(bytes, @intCast(len)); + } +}; + +fn selected(kind: []const u8, configs: anytype) bool { + for (configs) |config| { + if (config.edge_filter.mode == .all) return true; + for (config.edge_filter.types) |name| if (std.mem.eql(u8, name, kind)) return true; + } + return false; +} + +/// Caller supplies a peak-limited allocator and a shared, byte-accounted read +/// allowance. Manifest-bound sources authenticate only the touched blocks; +/// unbound current-wire sources additionally charge full-source verification. +pub fn readAlloc(alloc: Allocator, store: *artifacts.ArtifactStore, source: refs.ArtifactRef, configs: anytype, limits: anytype, cancellation: CancellationToken, remaining: *u64) !?Topology { + var context = try Context.init(alloc, store, source, cancellation, remaining); + defer context.deinit(); + return readPreparedAlloc(alloc, &context, configs, limits, cancellation); +} + +pub fn readPreparedAlloc(alloc: Allocator, context: *Context, configs: anytype, limits: anytype, cancellation: CancellationToken) !?Topology { + const reader = context; + const trailer = reader.trailer; + if (trailer.source_nodes > limits.max_nodes or trailer.source_edges > limits.max_edges) return error.GraphMetricBuildBudgetExceeded; + const directory = reader.directory orelse return null; + if (trailer.source_nodes > directory.nodes) return error.InvalidGraphSegment; + for (0..directory.page_offsets.len / 8) |i| { + const offset = std.mem.readInt(u64, directory.page_offsets[i * 8 ..][0..8], .little); + if (offset < wire.header_len or offset > trailer.body_len) return error.InvalidGraphSegment; + } + var iterator = directory.iterator(); + var selected_types: usize = 0; + var selected_edges: usize = 0; + var expected_offset = trailer.body_len; + while (try iterator.next()) |entry| { + if (entry.offset != expected_offset) return error.InvalidGraphSegment; + expected_offset = std.math.add(u64, expected_offset, std.math.mul(u64, entry.edges, 8) catch return error.InvalidGraphSegment) catch return error.InvalidGraphSegment; + if (!selected(entry.kind, configs)) continue; + selected_types += 1; + selected_edges = std.math.add(usize, selected_edges, std.math.cast(usize, entry.edges) orelse return error.GraphMetricBuildBudgetExceeded) catch return error.GraphMetricBuildBudgetExceeded; + } + if (expected_offset != trailer.body_len + trailer.topology_len or trailer.topology_len / 8 > trailer.source_edges) return error.InvalidGraphSegment; + const edges = try alloc.alloc(Edge, selected_edges); + errdefer alloc.free(edges); + const dense = selected_edges > directory.nodes / 64; + const mapping = try alloc.alloc(u32, if (dense) directory.nodes else 0); + defer alloc.free(mapping); + @memset(mapping, wire.no_table); + const endpoints = try alloc.alloc(u32, if (dense) directory.nodes else try std.math.mul(usize, selected_edges, 2)); + defer alloc.free(endpoints); + const type_offsets = try alloc.alloc(u32, selected_types + 1); + errdefer alloc.free(type_offsets); + const kinds = try alloc.alloc([]const u8, selected_types); + errdefer alloc.free(kinds); + const checksums = try alloc.alloc([32]u8, selected_types); + errdefer alloc.free(checksums); + var strings = std.ArrayListUnmanaged(u8).empty; + defer strings.deinit(alloc); + iterator = directory.iterator(); + var edge_index: usize = 0; + var type_index: usize = 0; + while (try iterator.next()) |entry| { + if (!selected(entry.kind, configs)) continue; + kinds[type_index] = entry.kind; + checksums[type_index] = entry.digest; + type_offsets[type_index] = @intCast(edge_index); + type_index += 1; + var read_edges: u64 = 0; + var previous: ?Edge = null; + while (read_edges < entry.edges) { + const count: usize = @intCast(@min(entry.edges - read_edges, 128 * 1024)); + const bytes = try reader.readAlloc(alloc, entry.offset + read_edges * 8, count * 8); + defer alloc.free(bytes); + for (0..count) |i| { + if (i % 4096 == 0) try cancellation.check(); + const edge = Edge{ .source = std.mem.readInt(u32, bytes[i * 8 ..][0..4], .little), .target = std.mem.readInt(u32, bytes[i * 8 + 4 ..][0..4], .little) }; + if (edge.source >= directory.nodes or edge.target >= directory.nodes) return error.InvalidGraphSegment; + if (previous) |prior| if (prior.source > edge.source or (prior.source == edge.source and prior.target > edge.target)) return error.InvalidGraphSegment; + previous = edge; + edges[edge_index] = edge; + if (dense) { + mapping[edge.source] = 0; + mapping[edge.target] = 0; + } else { + endpoints[edge_index * 2] = edge.source; + endpoints[edge_index * 2 + 1] = edge.target; + } + edge_index += 1; + } + read_edges += count; + } + } + type_offsets[selected_types] = @intCast(edge_index); + var unique: usize = 0; + if (dense) { + for (mapping, 0..) |*slot, ordinal| { + if (ordinal % 4096 == 0) try cancellation.check(); + if (slot.* == wire.no_table) continue; + slot.* = @intCast(unique); + endpoints[unique] = @intCast(ordinal); + unique += 1; + } + } else { + std.mem.sort(u32, endpoints, {}, std.sort.asc(u32)); + try cancellation.check(); + for (endpoints) |ordinal| { + if (unique != 0 and endpoints[unique - 1] == ordinal) continue; + endpoints[unique] = ordinal; + unique += 1; + } + } + const ordinals = endpoints[0..unique]; + const nodes = try alloc.alloc([]const u8, unique); + errdefer alloc.free(nodes); + const lengths = try alloc.alloc(usize, unique); + defer alloc.free(lengths); + var selected_node: usize = 0; + while (selected_node < unique) { + const page = ordinals[selected_node] / wire.node_page_entries; + var range = try directory.nodePage(page); + var last_page = page; + // Merge nearby selected pages into bounded reads. Sparse selection + // must not turn one source GET into thousands of tiny cloud requests. + for (ordinals[selected_node + 1 ..]) |ordinal| { + const next_page = ordinal / wire.node_page_entries; + if (next_page == last_page) continue; + const next = try directory.nodePage(next_page); + const end = std.math.add(u64, next.offset, next.len) catch return error.InvalidGraphSegment; + if (next.offset < range.offset + range.len) return error.InvalidGraphSegment; + if (next.offset - (range.offset + range.len) > 64 * 1024 or end - range.offset > 1024 * 1024) break; + range.len = end - range.offset; + last_page = next_page; + } + if (range.offset < wire.header_len or range.offset > trailer.body_len or range.len > trailer.body_len - range.offset) return error.InvalidGraphSegment; + const bytes = try reader.readAlloc(alloc, range.offset, range.len); + defer alloc.free(bytes); + const first = page * wire.node_page_entries; + const count = @min((last_page - page + 1) * wire.node_page_entries, directory.nodes - first); + var pos: usize = 0; + var previous: ?[]const u8 = null; + for (0..count) |i| { + if (bytes.len - pos < 4) return error.InvalidGraphSegment; + const len = std.mem.readInt(u32, bytes[pos..][0..4], .little); + pos += 4; + if (len > bytes.len - pos) return error.InvalidGraphSegment; + const node = bytes[pos..][0..len]; + if (previous) |prior| if (std.mem.order(u8, prior, node) != .lt) return error.InvalidGraphSegment; + previous = node; + if (selected_node < unique and ordinals[selected_node] == first + i) { + lengths[selected_node] = len; + try strings.appendSlice(alloc, node); + selected_node += 1; + } + pos += len; + } + if (pos != bytes.len) return error.InvalidGraphSegment; + } + for (kinds) |kind| try strings.appendSlice(alloc, kind); + const string_bytes = try strings.toOwnedSlice(alloc); + errdefer alloc.free(string_bytes); + var pos: usize = 0; + for (nodes, lengths, 0..) |*node, len, i| { + node.* = string_bytes[pos..][0..len]; + if (i > 0 and std.mem.order(u8, nodes[i - 1], node.*) != .lt) return error.InvalidGraphSegment; + pos += len; + } + for (kinds) |*kind| { + const len = kind.len; + kind.* = string_bytes[pos..][0..len]; + pos += len; + } + for (edges, 0..) |*edge, i| { + if (i % 4096 == 0) try cancellation.check(); + edge.source = if (dense) mapping[edge.source] else ordinalIndex(ordinals, edge.source); + edge.target = if (dense) mapping[edge.target] else ordinalIndex(ordinals, edge.target); + } + return .{ .node_ids = nodes, .edge_types = kinds, .string_bytes = string_bytes, .edge_type_offsets = type_offsets, .edges = edges, .type_checksums = checksums, .source_node_count = trailer.source_nodes, .source_edge_count = @intCast(trailer.source_edges), .retained_bytes = (nodes.len + kinds.len) * @sizeOf([]const u8) + string_bytes.len + type_offsets.len * 4 + edges.len * @sizeOf(Edge) + checksums.len * 32 }; +} + +fn ordinalIndex(ordinals: []const u32, ordinal: u32) u32 { + const index = std.sort.lowerBound(u32, ordinals, ordinal, struct { + fn order(a: u32, b: u32) std.math.Order { + return std.math.order(a, b); + } + }.order); + std.debug.assert(index < ordinals.len and ordinals[index] == ordinal); + return @intCast(index); +} diff --git a/zig/pkg/antfly/src/serverless/maintenance_cancellation.zig b/zig/pkg/antfly/src/serverless/maintenance_cancellation.zig index 4f0c936859..2bff6a6d66 100644 --- a/zig/pkg/antfly/src/serverless/maintenance_cancellation.zig +++ b/zig/pkg/antfly/src/serverless/maintenance_cancellation.zig @@ -1,7 +1,19 @@ // Copyright 2026 Antfly, Inc. -// SPDX-License-Identifier: Elastic-2.0 +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. const std = @import("std"); +const CancellationToken = @import("../common/cancellation.zig").CancellationToken; /// Borrowed cooperative cancellation for one synchronous maintenance pass. /// The atomic flag covers graceful shutdown before Future cancellation is @@ -9,10 +21,12 @@ const std = @import("std"); pub const Token = struct { io: std.Io, requested: ?*const std.atomic.Value(bool) = null, + cooperative: CancellationToken = .none, checkpoint_ptr: ?*anyopaque = null, checkpoint_fn: ?*const fn (*anyopaque) anyerror!void = null, pub fn check(self: Token) !void { + try self.cooperative.check(); if (self.requested) |requested| { if (requested.load(.acquire)) return error.Canceled; } @@ -37,3 +51,77 @@ pub const Token = struct { pub fn check(token: ?Token) !void { if (token) |value| try value.check(); } + +/// Borrowed bridge for synchronous graph preparation and its joined parallel +/// kernels. Lease checkpoints mutate renewal state, so concurrent workers must +/// serialize them rather than race on the enclosing HeldLease. +pub const GraphBridge = struct { + maintenance: ?Token, + mutex: std.Io.Mutex = .init, + failure: ?anyerror = null, + + pub fn token(self: *GraphBridge) CancellationToken { + if (self.maintenance == null) return .none; + return .{ .ptr = self, .check_fn = checkpoint, .is_cancelled_fn = isCancelled }; + } + + fn isCancelled(ptr: *const anyopaque) bool { + checkpoint(ptr) catch return true; + return false; + } + + fn checkpoint(ptr: *const anyopaque) !void { + const self: *GraphBridge = @ptrCast(@alignCast(@constCast(ptr))); + const maintenance = self.maintenance.?; + self.mutex.lockUncancelable(maintenance.io); + defer self.mutex.unlock(maintenance.io); + if (self.failure) |err| return err; + maintenance.check() catch |err| { + self.failure = err; + return err; + }; + } +}; + +test "serverless graph maintenance bridge serializes renewal and preserves lease failures" { + var threaded = std.Io.Threaded.init(std.testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + const State = struct { + checks: usize = 0, + lost: bool = false, + fn checkpoint(ptr: *anyopaque) !void { + const self: *@This() = @ptrCast(@alignCast(ptr)); + if (self.lost) return error.WorkLeaseLost; + self.checks += 1; + } + fn worker(token: CancellationToken) anyerror!void { + for (0..1024) |_| try token.check(); + } + }; + var state = State{}; + var bridge = GraphBridge{ .maintenance = (Token{ .io = io }).withCheckpoint(&state, State.checkpoint) }; + var futures: [4]std.Io.Future(anyerror!void) = undefined; + var active: usize = 0; + defer while (active > 0) { + active -= 1; + _ = futures[active].cancel(io) catch {}; + }; + for (&futures) |*future| { + future.* = try io.concurrent(State.worker, .{bridge.token()}); + active += 1; + } + while (active > 0) { + active -= 1; + try futures[active].await(io); + } + try std.testing.expectEqual(@as(usize, 4096), state.checks); + state.lost = true; + try std.testing.expectError(error.WorkLeaseLost, bridge.token().check()); + try std.testing.expect(bridge.token().isCancelled()); + // Object-store transports borrow only the boolean callback. + const transport_token = bridge.token(); + try std.testing.expect(transport_token.is_cancelled_fn.?(transport_token.ptr.?)); + state.lost = false; + try std.testing.expectError(error.WorkLeaseLost, bridge.token().check()); +} diff --git a/zig/pkg/antfly/src/serverless/manifest/artifact_ref.zig b/zig/pkg/antfly/src/serverless/manifest/artifact_ref.zig index dfa637379f..2fd4f348f2 100644 --- a/zig/pkg/antfly/src/serverless/manifest/artifact_ref.zig +++ b/zig/pkg/antfly/src/serverless/manifest/artifact_ref.zig @@ -15,6 +15,23 @@ //! Dependency-light manifest artifact references shared by manifest codecs and //! lake-native publication planning. +/// First manifest version carrying artifact materializer provenance. +/// The only manifest wire that may publish graph-metric artifacts. Serverless +/// has not shipped, so partial pre-release graph-metric layouts are rejected +/// instead of becoming a permanent compatibility surface. +pub const graph_metric_manifest_wire_version: u16 = 22; +pub const graph_metric_segment_wire_version: u16 = 10; + +pub const GraphMetricMaterializationState = enum(u8) { + ready = 0, + rejected = 1, +}; + +pub const GraphMetricRejectionReason = enum(u8) { + none = 0, + build_budget_exceeded = 1, +}; + pub const ArtifactKind = enum(u8) { text_segment = 1, vector_segment = 2, @@ -28,6 +45,7 @@ pub const ArtifactKind = enum(u8) { row_fragment_stats = 10, algebraic_segment = 11, external_base_source = 12, + graph_metric_segment = 13, }; pub const ArtifactRef = struct { @@ -36,9 +54,86 @@ pub const ArtifactRef = struct { artifact_id: []const u8, byte_len: u64, checksum: []const u8, + /// Optional artifact-specific metadata schema version. Zero means the + /// producing manifest predates persisted provenance. + metadata_version: u16 = 0, + published_generation: u64 = 0, + edge_generation: u64 = 0, + computed_at_ms: u64 = 0, + /// Artifact-producing policy identity. Graph-metric refs persist this so + /// catalog scheduling can detect stale materializations without fetching + /// the object payload. Zero denotes a pre-v15 manifest. + materializer_fingerprint: u64 = 0, + /// Manifest-authenticated graph trailer; authenticates the directory and + /// its fixed-size data-block checksums without whole-object cold reads. + graph_topology_control_checksum: [32]u8 = @splat(0), + /// Authenticated range metadata for the current graph-metric wire. Fixed-size + /// digests avoid per-reference allocations and let point/status reads stay + /// bounded without trusting object-store range responses. + graph_metric_control_len: u32 = 0, + graph_metric_routing_footer_len: u32 = 0, + graph_metric_control_checksum: [32]u8 = @splat(0), + // Authenticates the bounded routing root, which in turn authenticates the + // primary point index. Top-K readers never need to fetch that point index. + graph_metric_routing_checksum: [32]u8 = @splat(0), + graph_metric_point_index_checksum: [32]u8 = @splat(0), + graph_metric_config_fingerprint: u64 = 0, + graph_metric_source_checksum: [32]u8 = @splat(0), + /// Canonical selected unweighted connectivity, independent of source + /// payload layout/weights. The metric control authenticates this identity; + /// source_checksum binds the current publication to its graph artifact. + graph_metric_topology_checksum: [32]u8 = @splat(0), + graph_metric_materialization_state: GraphMetricMaterializationState = .ready, + graph_metric_rejection_reason: GraphMetricRejectionReason = .none, }; +/// Distinct logical graphs/metrics may share one immutable payload. Every +/// physical/integrity field must agree; only names and per-reference provenance +/// may differ. Comparing the normalized whole struct also checks future fields. +pub fn areGraphArtifactAliases(a: ArtifactRef, b: ArtifactRef) bool { + const std = @import("std"); + if (a.kind != b.kind or (a.kind != .graph_segment and a.kind != .graph_metric_segment) or + (a.kind == .graph_metric_segment and a.metadata_version != graph_metric_segment_wire_version) or + a.name.len == 0 or b.name.len == 0 or std.mem.eql(u8, a.name, b.name) or + !std.mem.eql(u8, a.artifact_id, b.artifact_id) or !std.mem.eql(u8, a.checksum, b.checksum)) return false; + var normalized = a; + normalized.name = b.name; + normalized.artifact_id = b.artifact_id; + normalized.checksum = b.checksum; + normalized.published_generation = b.published_generation; + normalized.edge_generation = b.edge_generation; + normalized.computed_at_ms = b.computed_at_ms; + return std.meta.eql(normalized, b); +} + +test "serverless graph metric aliases require identical immutable metadata" { + const std = @import("std"); + const original = ArtifactRef{ .kind = .graph_metric_segment, .name = "1:a1:x", .artifact_id = "metric", .checksum = "checksum", .byte_len = 128, .metadata_version = graph_metric_segment_wire_version }; + var alias = original; + alias.name = "1:b1:y"; + alias.published_generation = 3; + alias.edge_generation = 2; + alias.computed_at_ms = 1; + try std.testing.expect(areGraphArtifactAliases(original, alias)); + try std.testing.expect(!areGraphArtifactAliases(original, original)); + alias.graph_metric_routing_checksum[0] = 1; + try std.testing.expect(!areGraphArtifactAliases(original, alias)); + alias.graph_metric_routing_checksum[0] = 0; + alias.graph_metric_config_fingerprint = 1; + try std.testing.expect(!areGraphArtifactAliases(original, alias)); + alias.graph_metric_config_fingerprint = 0; + alias.byte_len += 1; + try std.testing.expect(!areGraphArtifactAliases(original, alias)); + alias.byte_len -= 1; + alias.kind = .graph_segment; + try std.testing.expect(!areGraphArtifactAliases(original, alias)); + var graph = original; + graph.kind = .graph_segment; + try std.testing.expect(areGraphArtifactAliases(graph, alias)); +} + test "manifest artifact kinds include lake-native artifacts" { try @import("std").testing.expectEqual(@as(u8, 9), @intFromEnum(ArtifactKind.row_fragment)); try @import("std").testing.expectEqual(@as(u8, 11), @intFromEnum(ArtifactKind.algebraic_segment)); + try @import("std").testing.expectEqual(@as(u8, 13), @intFromEnum(ArtifactKind.graph_metric_segment)); } diff --git a/zig/pkg/antfly/src/serverless/manifest/codec.zig b/zig/pkg/antfly/src/serverless/manifest/codec.zig index 8950cd8f12..8661ccfc63 100644 --- a/zig/pkg/antfly/src/serverless/manifest/codec.zig +++ b/zig/pkg/antfly/src/serverless/manifest/codec.zig @@ -18,22 +18,15 @@ const bounded_decode = @import("../bounded_decode.zig"); const catalog_types = @import("../catalog/types.zig"); const manifest_base_source = @import("base_source.zig"); const manifest_types = @import("types.zig"); +const artifact_ref = @import("artifact_ref.zig"); const search_sources = @import("../search_sources.zig"); pub const wire_magic = "AFSM"; -pub const wire_version: u16 = 13; +pub const wire_version = artifact_ref.graph_metric_manifest_wire_version; -const header_size_v2 = 4 + 2 + 4 + 8 + 8 + 8 + 8 + 8 + 4 + 4 + 4 + 4 + 4; -const header_size_v3 = header_size_v2 + 1 + 1; -const header_size_v5 = header_size_v3 + 4; -const header_size = header_size_v2 + 4 + 4; -const header_size_v7 = header_size + 4 + 4 + 4; const policy_size = 98; -const header_size_v8 = header_size_v7 + policy_size; -const header_size_v10 = header_size_v8 + 8; -const header_size_v11 = header_size_v10 + 1; -const header_size_v12 = header_size_v11 + 4; -const header_size_v13 = header_size_v12 + 1 + 8; +const header_size = 4 + 2 + 4 + 8 + 8 + 8 + 8 + 8 + 4 + 4 + 4 + 4 + 4 + + 4 + 4 + 4 + 4 + 4 + policy_size + 8 + 1 + 4 + 1 + 8; fn encodePolicy(buf: []u8, policy: catalog_types.NamespacePolicy) void { var pos: usize = 0; @@ -146,7 +139,9 @@ fn decodePolicy(data: []const u8, pos_ptr: *usize) !catalog_types.NamespacePolic } fn artifactEncodedSize(artifact: manifest_types.ArtifactRef) usize { - return 1 + 4 + 4 + 8 + 4 + artifact.name.len + artifact.artifact_id.len + artifact.checksum.len; + const provenance_bytes = 2 + 8 + 8 + 8 + 8; + const integrity_bytes: usize = if (artifact.kind == .graph_segment) 32 else if (artifact.kind == .graph_metric_segment) 4 + 4 + 32 + 32 + 32 + 8 + 32 + 32 + 1 + 1 else 0; + return 1 + 4 + 4 + 8 + 4 + provenance_bytes + integrity_bytes + artifact.name.len + artifact.artifact_id.len + artifact.checksum.len; } fn publishedSearchSourceEncodedSize(source: search_sources.SearchSourceDescriptor) usize { @@ -198,13 +193,30 @@ fn baseSourceEncodedSize(base_source: manifest_types.BaseSourceDescriptor) usize } pub fn encodeAlloc(alloc: Allocator, manifest: manifest_types.Manifest) ![]u8 { + return try encodeForVersionAlloc(alloc, manifest, wire_version); +} + +pub fn encodeForVersionAlloc(alloc: Allocator, manifest: manifest_types.Manifest, target_version: u16) ![]u8 { + if (target_version != wire_version) { + return error.UnsupportedManifestWriteVersion; + } + for (manifest.artifacts) |artifact| { + if (artifact.kind != .graph_metric_segment) continue; + if (artifact.metadata_version != artifact_ref.graph_metric_segment_wire_version or + artifact.graph_metric_control_len == 0 or artifact.graph_metric_routing_footer_len == 0 or + (artifact.graph_metric_materialization_state == .ready and artifact.graph_metric_rejection_reason != .none) or + (artifact.graph_metric_materialization_state == .rejected and artifact.graph_metric_rejection_reason == .none)) + { + return error.InvalidManifest; + } + } const derived_output_items: []const search_sources.DerivedOutputDescriptor = manifest.stats.derived_outputs.items orelse &.{}; const published_source_items: []const search_sources.SearchSourceDescriptor = manifest.stats.published_search_sources.items orelse &.{}; const base_source_len: u32 = if (manifest.base_source) |base_source| @intCast(baseSourceEncodedSize(base_source)) else 0; - var size: usize = header_size_v13 + manifest.namespace.len + + var size: usize = header_size + manifest.namespace.len + manifest.stats.schema_json.len + manifest.stats.read_schema_json.len + manifest.stats.indexes_json.len + @@ -219,7 +231,7 @@ pub fn encodeAlloc(alloc: Allocator, manifest: manifest_types.Manifest) ![]u8 { var pos: usize = 0; @memcpy(buf[pos..][0..4], wire_magic); pos += 4; - std.mem.writeInt(u16, buf[pos..][0..2], wire_version, .little); + std.mem.writeInt(u16, buf[pos..][0..2], target_version, .little); pos += 2; std.mem.writeInt(u32, buf[pos..][0..4], @intCast(manifest.namespace.len), .little); pos += 4; @@ -340,6 +352,42 @@ pub fn encodeAlloc(alloc: Allocator, manifest: manifest_types.Manifest) ![]u8 { pos += 8; std.mem.writeInt(u32, buf[pos..][0..4], @intCast(artifact.checksum.len), .little); pos += 4; + std.mem.writeInt(u16, buf[pos..][0..2], artifact.metadata_version, .little); + pos += 2; + std.mem.writeInt(u64, buf[pos..][0..8], artifact.published_generation, .little); + pos += 8; + std.mem.writeInt(u64, buf[pos..][0..8], artifact.edge_generation, .little); + pos += 8; + std.mem.writeInt(u64, buf[pos..][0..8], artifact.computed_at_ms, .little); + pos += 8; + std.mem.writeInt(u64, buf[pos..][0..8], artifact.materializer_fingerprint, .little); + pos += 8; + if (artifact.kind == .graph_segment) { + @memcpy(buf[pos..][0..32], &artifact.graph_topology_control_checksum); + pos += 32; + } + if (artifact.kind == .graph_metric_segment) { + std.mem.writeInt(u32, buf[pos..][0..4], artifact.graph_metric_control_len, .little); + pos += 4; + std.mem.writeInt(u32, buf[pos..][0..4], artifact.graph_metric_routing_footer_len, .little); + pos += 4; + @memcpy(buf[pos..][0..32], &artifact.graph_metric_control_checksum); + pos += 32; + @memcpy(buf[pos..][0..32], &artifact.graph_metric_routing_checksum); + pos += 32; + @memcpy(buf[pos..][0..32], &artifact.graph_metric_point_index_checksum); + pos += 32; + std.mem.writeInt(u64, buf[pos..][0..8], artifact.graph_metric_config_fingerprint, .little); + pos += 8; + @memcpy(buf[pos..][0..32], &artifact.graph_metric_source_checksum); + pos += 32; + @memcpy(buf[pos..][0..32], &artifact.graph_metric_topology_checksum); + pos += 32; + buf[pos] = @intFromEnum(artifact.graph_metric_materialization_state); + pos += 1; + buf[pos] = @intFromEnum(artifact.graph_metric_rejection_reason); + pos += 1; + } @memcpy(buf[pos..][0..artifact.name.len], artifact.name); pos += artifact.name.len; @memcpy(buf[pos..][0..artifact.artifact_id.len], artifact.artifact_id); @@ -407,7 +455,7 @@ fn encodeBaseSource(buf: []u8, descriptor: manifest_types.BaseSourceDescriptor) } pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest { - if (data.len < header_size_v2) return error.InvalidManifest; + if (data.len < 6) return error.InvalidManifest; var pos: usize = 0; if (!std.mem.eql(u8, data[pos..][0..4], wire_magic)) return error.InvalidManifest; @@ -415,7 +463,8 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest const version = std.mem.readInt(u16, data[pos..][0..2], .little); pos += 2; - if (version != 2 and version != 3 and version != 4 and version != 5 and version != 6 and version != 7 and version != 8 and version != 10 and version != 11 and version != 12 and version != wire_version) return error.UnsupportedManifestVersion; + if (version != wire_version) return error.UnsupportedManifestVersion; + if (data.len < header_size) return error.InvalidManifest; const namespace_len = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; @@ -429,12 +478,12 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest pos += 8; const document_count = std.mem.readInt(u64, data[pos..][0..8], .little); pos += 8; - const document_base_version = if (version >= 10) blk: { + const document_base_version = blk: { const value = std.mem.readInt(u64, data[pos..][0..8], .little); pos += 8; break :blk value; - } else 0; - const document_publish_mode = if (version >= 11) blk: { + }; + const document_publish_mode = blk: { const value: catalog_types.DocumentPublishMode = switch (data[pos]) { 1 => .append_mutation_tail, 2 => .inline_rebase, @@ -443,10 +492,7 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest }; pos += 1; break :blk value; - } else if (wal_start_lsn == wal_end_lsn) - catalog_types.DocumentPublishMode.inline_rebase - else - catalog_types.DocumentPublishMode.append_mutation_tail; + }; const text_segment_count = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; const vector_segment_count = std.mem.readInt(u32, data[pos..][0..4], .little); @@ -457,59 +503,39 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest pos += 4; const artifact_count = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; - const has_vector_source = if (version >= 3 and version <= 5) blk: { - const value = data[pos] != 0; - pos += 1; - break :blk value; - } else false; - const has_sparse_source = if (version >= 3 and version <= 5) blk: { - const value = data[pos] != 0; - pos += 1; - break :blk value; - } else false; - const published_search_source_count = if (version >= 6) blk: { + const published_search_source_count = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const has_chunk_preview_output = if (version == 4) blk: { - const value = data[pos] != 0; - pos += 1; - break :blk value; - } else false; - const has_rerank_terms_output = if (version == 4) blk: { - const value = data[pos] != 0; - pos += 1; - break :blk value; - } else false; - const derived_output_count = if (version >= 5) blk: { + }; + const derived_output_count = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const schema_len = if (version >= 7) blk: { + }; + const schema_len = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const read_schema_len = if (version >= 7) blk: { + }; + const read_schema_len = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const indexes_len = if (version >= 7) blk: { + }; + const indexes_len = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const policy = if (version >= 8) try decodePolicy(data, &pos) else catalog_types.NamespacePolicy{}; - const base_source_len = if (version >= 12) blk: { + }; + const policy = try decodePolicy(data, &pos); + const base_source_len = blk: { if (pos + 4 > data.len) return error.InvalidManifest; const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; - const publication_lineage_tracked = if (version >= 13) blk: { + }; + const publication_lineage_tracked = blk: { if (pos + 1 > data.len) return error.InvalidManifest; const value = switch (data[pos]) { 0 => false, @@ -518,13 +544,13 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest }; pos += 1; break :blk value; - } else false; - const publication_parent_version = if (version >= 13) blk: { + }; + const publication_parent_version = blk: { if (pos + 8 > data.len) return error.InvalidManifest; const value = std.mem.readInt(u64, data[pos..][0..8], .little); pos += 8; break :blk if (value == 0) null else value; - } else null; + }; if (!publication_lineage_tracked and publication_parent_version != null) return error.InvalidManifest; if (publication_parent_version) |parent| { if (parent >= manifest_version) return error.InvalidManifest; @@ -541,91 +567,38 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest errdefer if (read_schema_json.len > 0) alloc.free(read_schema_json); var indexes_json: []u8 = &.{}; errdefer if (indexes_json.len > 0) alloc.free(indexes_json); - if (version >= 7) { - if (pos + schema_len + read_schema_len + indexes_len > data.len) return error.InvalidManifest; - if (schema_len > 0) schema_json = try alloc.dupe(u8, data[pos .. pos + schema_len]); - pos += schema_len; - if (read_schema_len > 0) read_schema_json = try alloc.dupe(u8, data[pos .. pos + read_schema_len]); - pos += read_schema_len; - if (indexes_len > 0) indexes_json = try alloc.dupe(u8, data[pos .. pos + indexes_len]); - pos += indexes_len; - } + if (pos + schema_len + read_schema_len + indexes_len > data.len) return error.InvalidManifest; + if (schema_len > 0) schema_json = try alloc.dupe(u8, data[pos .. pos + schema_len]); + pos += schema_len; + if (read_schema_len > 0) read_schema_json = try alloc.dupe(u8, data[pos .. pos + read_schema_len]); + pos += read_schema_len; + if (indexes_len > 0) indexes_json = try alloc.dupe(u8, data[pos .. pos + indexes_len]); + pos += indexes_len; var published_search_sources: search_sources.PublishedSearchSources = .{}; errdefer search_sources.deinitPublishedSearchSources(alloc, &published_search_sources); - if (version >= 6) { - if (published_search_source_count > 0) { - const items = try alloc.alloc(search_sources.SearchSourceDescriptor, published_search_source_count); - errdefer alloc.free(items); - var initialized_sources: usize = 0; - errdefer { - for (items[0..initialized_sources]) |*item| search_sources.deinitSearchSourceDescriptor(alloc, item); - } - for (0..published_search_source_count) |idx| { - if (pos + 1 + 1 + 4 > data.len) return error.InvalidManifest; - const source_kind = data[pos]; - pos += 1; - const kind = data[pos]; - pos += 1; - const index_name_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + index_name_len > data.len) return error.InvalidManifest; - const index_name = try alloc.dupe(u8, data[pos .. pos + index_name_len]); - pos += index_name_len; - items[idx] = switch (source_kind) { - 1 => .{ .vector = .{ - .index_name = index_name, - .document_source = switch (kind) { - 1 => .top_level_embedding, - 2 => .chunk_embeddings, - 3 => .chunk_embeddings_or_top_level, - else => return error.InvalidManifest, - }, - } }, - 2 => .{ .sparse = .{ - .index_name = index_name, - .document_source = switch (kind) { - 1 => .sparse_embedding, - else => return error.InvalidManifest, - }, - } }, - 3 => .{ .text = .{ - .index_name = index_name, - } }, - else => return error.InvalidManifest, - }; - initialized_sources += 1; - } - published_search_sources = blk: { - const owned = items; - var out = search_sources.PublishedSearchSources{ .items = owned }; - for (owned) |item| switch (item) { - .text => |value| { - if (out.text == null) out.text = value; - }, - .vector => |value| out.vector = value, - .sparse => |value| out.sparse = value, - }; - break :blk out; - }; - } - } else { - var legacy_items = std.ArrayListUnmanaged(search_sources.SearchSourceDescriptor).empty; + if (published_search_source_count > 0) { + if (published_search_source_count > (data.len - pos) / 6) return error.InvalidManifest; + const items = try alloc.alloc(search_sources.SearchSourceDescriptor, published_search_source_count); + errdefer alloc.free(items); + var initialized_sources: usize = 0; errdefer { - for (legacy_items.items) |*item| search_sources.deinitSearchSourceDescriptor(alloc, item); - legacy_items.deinit(alloc); + for (items[0..initialized_sources]) |*item| search_sources.deinitSearchSourceDescriptor(alloc, item); } - if (has_vector_source) { - const value = blk: { - if (pos + 1 + 4 > data.len) return error.InvalidManifest; - const kind = data[pos]; - pos += 1; - const index_name_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + index_name_len > data.len) return error.InvalidManifest; - const index_name = try alloc.dupe(u8, data[pos .. pos + index_name_len]); - pos += index_name_len; - break :blk search_sources.VectorSourceDescriptor{ + for (0..published_search_source_count) |idx| { + if (pos + 1 + 1 + 4 > data.len) return error.InvalidManifest; + const source_kind = data[pos]; + pos += 1; + const kind = data[pos]; + pos += 1; + const index_name_len = std.mem.readInt(u32, data[pos..][0..4], .little); + pos += 4; + if (pos + index_name_len > data.len) return error.InvalidManifest; + const index_name = try alloc.dupe(u8, data[pos .. pos + index_name_len]); + errdefer alloc.free(index_name); + pos += index_name_len; + items[idx] = switch (source_kind) { + 1 => .{ .vector = .{ .index_name = index_name, .document_source = switch (kind) { 1 => .top_level_embedding, @@ -633,32 +606,23 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest 3 => .chunk_embeddings_or_top_level, else => return error.InvalidManifest, }, - }; - }; - try legacy_items.append(alloc, .{ .vector = value }); - } - if (has_sparse_source) { - const value = blk: { - if (pos + 1 + 4 > data.len) return error.InvalidManifest; - const kind = data[pos]; - pos += 1; - const index_name_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + index_name_len > data.len) return error.InvalidManifest; - const index_name = try alloc.dupe(u8, data[pos .. pos + index_name_len]); - pos += index_name_len; - break :blk search_sources.SparseSourceDescriptor{ + } }, + 2 => .{ .sparse = .{ .index_name = index_name, .document_source = switch (kind) { 1 => .sparse_embedding, else => return error.InvalidManifest, }, - }; + } }, + 3 => .{ .text = .{ + .index_name = index_name, + } }, + else => return error.InvalidManifest, }; - try legacy_items.append(alloc, .{ .sparse = value }); + initialized_sources += 1; } - if (legacy_items.items.len > 0) { - const owned = try legacy_items.toOwnedSlice(alloc); + published_search_sources = blk: { + const owned = items; var out = search_sources.PublishedSearchSources{ .items = owned }; for (owned) |item| switch (item) { .text => |value| { @@ -667,68 +631,21 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest .vector => |value| out.vector = value, .sparse => |value| out.sparse = value, }; - published_search_sources = out; - } + break :blk out; + }; } var derived_outputs: search_sources.MaterializedDerivedOutputs = .{}; errdefer search_sources.deinitMaterializedDerivedOutputs(alloc, &derived_outputs); - if (version >= 5) { - if (derived_output_count > 0) { - const items = try alloc.alloc(search_sources.DerivedOutputDescriptor, derived_output_count); - errdefer alloc.free(items); - var initialized_outputs: usize = 0; - errdefer { - for (items[0..initialized_outputs]) |*item| search_sources.deinitDerivedOutputDescriptor(alloc, item); - } - for (0..derived_output_count) |idx| { - if (pos + 1 + 4 > data.len) return error.InvalidManifest; - const kind = data[pos]; - pos += 1; - const name_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + name_len > data.len) return error.InvalidManifest; - const name = try alloc.dupe(u8, data[pos .. pos + name_len]); - pos += name_len; - items[idx] = .{ - .name = name, - .kind = switch (kind) { - 1 => .chunk_preview, - 2 => .chunk_embeddings, - 3 => .rerank_terms, - else => return error.InvalidManifest, - }, - }; - initialized_outputs += 1; - } - derived_outputs = .{ .items = items }; - } - } else { - var legacy_outputs = std.ArrayListUnmanaged(search_sources.DerivedOutputDescriptor).empty; + if (derived_output_count > 0) { + if (derived_output_count > (data.len - pos) / 5) return error.InvalidManifest; + const items = try alloc.alloc(search_sources.DerivedOutputDescriptor, derived_output_count); + errdefer alloc.free(items); + var initialized_outputs: usize = 0; errdefer { - for (legacy_outputs.items) |*item| search_sources.deinitDerivedOutputDescriptor(alloc, item); - legacy_outputs.deinit(alloc); - } - if (has_chunk_preview_output) { - if (pos + 1 + 4 > data.len) return error.InvalidManifest; - const kind = data[pos]; - pos += 1; - const name_len = std.mem.readInt(u32, data[pos..][0..4], .little); - pos += 4; - if (pos + name_len > data.len) return error.InvalidManifest; - const name = try alloc.dupe(u8, data[pos .. pos + name_len]); - pos += name_len; - try legacy_outputs.append(alloc, .{ - .name = name, - .kind = switch (kind) { - 1 => .chunk_preview, - 2 => .chunk_embeddings, - 3 => .rerank_terms, - else => return error.InvalidManifest, - }, - }); + for (items[0..initialized_outputs]) |*item| search_sources.deinitDerivedOutputDescriptor(alloc, item); } - if (has_rerank_terms_output) { + for (0..derived_output_count) |idx| { if (pos + 1 + 4 > data.len) return error.InvalidManifest; const kind = data[pos]; pos += 1; @@ -736,8 +653,9 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest pos += 4; if (pos + name_len > data.len) return error.InvalidManifest; const name = try alloc.dupe(u8, data[pos .. pos + name_len]); + errdefer alloc.free(name); pos += name_len; - try legacy_outputs.append(alloc, .{ + items[idx] = .{ .name = name, .kind = switch (kind) { 1 => .chunk_preview, @@ -745,15 +663,14 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest 3 => .rerank_terms, else => return error.InvalidManifest, }, - }); - } - if (legacy_outputs.items.len > 0) { - derived_outputs = .{ .items = try legacy_outputs.toOwnedSlice(alloc) }; - } else { - legacy_outputs.deinit(alloc); + }; + initialized_outputs += 1; } + derived_outputs = .{ .items = items }; } + const min_artifact_header_len = 1 + 4 + 4 + 8 + 4 + 2 + 8 + 8 + 8 + 8; + if (artifact_count > (data.len - pos) / min_artifact_header_len) return error.InvalidManifest; const artifacts = try alloc.alloc(manifest_types.ArtifactRef, artifact_count); errdefer alloc.free(artifacts); @@ -767,21 +684,96 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest } for (0..artifact_count) |idx| { - const min_artifact_header_len: usize = if (version >= 9) 1 + 4 + 4 + 8 + 4 else 1 + 4 + 8 + 4; if (pos + min_artifact_header_len > data.len) return error.InvalidManifest; - const kind: manifest_types.ArtifactKind = @enumFromInt(data[pos]); + const kind = std.enums.fromInt(manifest_types.ArtifactKind, data[pos]) orelse return error.InvalidManifest; pos += 1; - const name_len = if (version >= 9) blk: { + const name_len = blk: { const value = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; break :blk value; - } else 0; + }; const artifact_id_len = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; const byte_len = std.mem.readInt(u64, data[pos..][0..8], .little); pos += 8; const checksum_len = std.mem.readInt(u32, data[pos..][0..4], .little); pos += 4; + const metadata_version = blk: { + const value = std.mem.readInt(u16, data[pos..][0..2], .little); + pos += 2; + break :blk value; + }; + if (kind == .graph_metric_segment and metadata_version != artifact_ref.graph_metric_segment_wire_version) { + return error.InvalidManifest; + } + const published_generation = blk: { + const value = std.mem.readInt(u64, data[pos..][0..8], .little); + pos += 8; + break :blk value; + }; + const edge_generation = blk: { + const value = std.mem.readInt(u64, data[pos..][0..8], .little); + pos += 8; + break :blk value; + }; + const computed_at_ms = blk: { + const value = std.mem.readInt(u64, data[pos..][0..8], .little); + pos += 8; + break :blk value; + }; + const materializer_fingerprint = blk: { + const value = std.mem.readInt(u64, data[pos..][0..8], .little); + pos += 8; + break :blk value; + }; + var graph_topology_control_checksum: [32]u8 = @splat(0); + if (kind == .graph_segment) { + if (data.len - pos < 32) return error.InvalidManifest; + graph_topology_control_checksum = data[pos..][0..32].*; + pos += 32; + } + var graph_metric_control_len: u32 = 0; + var graph_metric_routing_footer_len: u32 = 0; + var graph_metric_control_checksum: [32]u8 = @splat(0); + var graph_metric_routing_checksum: [32]u8 = @splat(0); + var graph_metric_point_index_checksum: [32]u8 = @splat(0); + var graph_metric_config_fingerprint: u64 = 0; + var graph_metric_source_checksum: [32]u8 = @splat(0); + var graph_metric_topology_checksum: [32]u8 = @splat(0); + var graph_metric_materialization_state: artifact_ref.GraphMetricMaterializationState = .ready; + var graph_metric_rejection_reason: artifact_ref.GraphMetricRejectionReason = .none; + // Graph metrics are admitted only on the current manifest wire above, + // so there is no partially populated legacy integrity shape here. + if (kind == .graph_metric_segment) { + const integrity_len: usize = 4 + 4 + 32 + 32 + 32 + 8 + 32 + 32 + 1 + 1; + if (pos + integrity_len > data.len) return error.InvalidManifest; + graph_metric_control_len = std.mem.readInt(u32, data[pos..][0..4], .little); + pos += 4; + graph_metric_routing_footer_len = std.mem.readInt(u32, data[pos..][0..4], .little); + pos += 4; + @memcpy(&graph_metric_control_checksum, data[pos..][0..32]); + pos += 32; + @memcpy(&graph_metric_routing_checksum, data[pos..][0..32]); + pos += 32; + @memcpy(&graph_metric_point_index_checksum, data[pos..][0..32]); + pos += 32; + graph_metric_config_fingerprint = std.mem.readInt(u64, data[pos..][0..8], .little); + pos += 8; + @memcpy(&graph_metric_source_checksum, data[pos..][0..32]); + pos += 32; + @memcpy(&graph_metric_topology_checksum, data[pos..][0..32]); + pos += 32; + graph_metric_materialization_state = std.enums.fromInt(artifact_ref.GraphMetricMaterializationState, data[pos]) orelse return error.InvalidManifest; + pos += 1; + graph_metric_rejection_reason = std.enums.fromInt(artifact_ref.GraphMetricRejectionReason, data[pos]) orelse return error.InvalidManifest; + pos += 1; + if (graph_metric_control_len == 0 or graph_metric_routing_footer_len == 0 or + (graph_metric_materialization_state == .ready and graph_metric_rejection_reason != .none) or + (graph_metric_materialization_state == .rejected and graph_metric_rejection_reason == .none)) + { + return error.InvalidManifest; + } + } if (pos + name_len + artifact_id_len + checksum_len > data.len) return error.InvalidManifest; const name = if (name_len > 0) try alloc.dupe(u8, data[pos .. pos + name_len]) else &.{}; @@ -799,19 +791,33 @@ pub fn decodeAlloc(alloc: Allocator, data: []const u8) !manifest_types.Manifest .artifact_id = artifact_id, .byte_len = byte_len, .checksum = checksum, + .metadata_version = metadata_version, + .published_generation = published_generation, + .edge_generation = edge_generation, + .computed_at_ms = computed_at_ms, + .materializer_fingerprint = materializer_fingerprint, + .graph_metric_control_len = graph_metric_control_len, + .graph_topology_control_checksum = graph_topology_control_checksum, + .graph_metric_routing_footer_len = graph_metric_routing_footer_len, + .graph_metric_control_checksum = graph_metric_control_checksum, + .graph_metric_routing_checksum = graph_metric_routing_checksum, + .graph_metric_point_index_checksum = graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = graph_metric_config_fingerprint, + .graph_metric_source_checksum = graph_metric_source_checksum, + .graph_metric_topology_checksum = graph_metric_topology_checksum, + .graph_metric_materialization_state = graph_metric_materialization_state, + .graph_metric_rejection_reason = graph_metric_rejection_reason, }; initialized += 1; } var base_source: ?manifest_types.BaseSourceDescriptor = null; errdefer if (base_source) |*descriptor| manifest_base_source.freeOwnedDescriptor(alloc, descriptor); - if (version >= 12) { - if (pos + base_source_len > data.len) return error.InvalidManifest; - if (base_source_len > 0) { - base_source = try decodeBaseSourceAlloc(alloc, data[pos .. pos + base_source_len]); - } - pos += base_source_len; + if (pos + base_source_len > data.len) return error.InvalidManifest; + if (base_source_len > 0) { + base_source = try decodeBaseSourceAlloc(alloc, data[pos .. pos + base_source_len]); } + pos += base_source_len; if (pos != data.len) return error.InvalidManifest; @@ -1001,7 +1007,7 @@ test "serverless manifest codec round-trips deterministically" { .published_search_sources = try search_sources.defaultPublishedSearchSourcesAlloc(alloc), .derived_outputs = try search_sources.defaultMaterializedDerivedOutputsAlloc(alloc), }, - .artifacts = try alloc.alloc(manifest_types.ArtifactRef, 4), + .artifacts = try alloc.alloc(manifest_types.ArtifactRef, 5), }; defer manifest.deinit(alloc); @@ -1029,6 +1035,27 @@ test "serverless manifest codec round-trips deterministically" { .artifact_id = try alloc.dupe(u8, "graph-0001"), .byte_len = 512, .checksum = try alloc.dupe(u8, "sha256:graph"), + .graph_topology_control_checksum = @splat(0x77), + }; + manifest.artifacts[4] = .{ + .kind = .graph_metric_segment, + .name = try alloc.dupe(u8, "5:graph8:pagerank"), + .artifact_id = try alloc.dupe(u8, "metric-0001"), + .byte_len = 256, + .checksum = try alloc.dupe(u8, "sha256:metric"), + .metadata_version = artifact_ref.graph_metric_segment_wire_version, + .published_generation = 40, + .edge_generation = 39, + .computed_at_ms = 123, + .materializer_fingerprint = 0x1234, + .graph_metric_control_len = 73, + .graph_metric_routing_footer_len = 17, + .graph_metric_control_checksum = @splat(0x11), + .graph_metric_routing_checksum = @splat(0x22), + .graph_metric_point_index_checksum = @splat(0x44), + .graph_metric_config_fingerprint = 0x5678, + .graph_metric_source_checksum = @splat(0x33), + .graph_metric_topology_checksum = @splat(0x55), }; const encoded_a = try encodeAlloc(alloc, manifest); @@ -1053,14 +1080,72 @@ test "serverless manifest codec round-trips deterministically" { try std.testing.expectEqualStrings(search_sources.default_chunk_preview_output_name, decoded.stats.derived_outputs.findByKind(.chunk_preview).?.name); try std.testing.expectEqualStrings(search_sources.default_chunk_embeddings_output_name, decoded.stats.derived_outputs.findByKind(.chunk_embeddings).?.name); try std.testing.expectEqualStrings(search_sources.default_rerank_terms_output_name, decoded.stats.derived_outputs.findByKind(.rerank_terms).?.name); - try std.testing.expectEqual(@as(usize, 4), decoded.artifacts.len); + try std.testing.expectEqual(@as(usize, 5), decoded.artifacts.len); try std.testing.expectEqual(manifest_types.ArtifactKind.text_segment, decoded.artifacts[0].kind); try std.testing.expectEqualStrings("vec-0001", decoded.artifacts[1].artifact_id); try std.testing.expectEqual(manifest_types.ArtifactKind.sparse_segment, decoded.artifacts[2].kind); try std.testing.expectEqual(manifest_types.ArtifactKind.graph_segment, decoded.artifacts[3].kind); + try std.testing.expectEqual(manifest_types.ArtifactKind.graph_metric_segment, decoded.artifacts[4].kind); + try std.testing.expectEqual(artifact_ref.graph_metric_segment_wire_version, decoded.artifacts[4].metadata_version); + try std.testing.expectEqualSlices(u8, &manifest.artifacts[3].graph_topology_control_checksum, &decoded.artifacts[3].graph_topology_control_checksum); + try std.testing.expectEqual(@as(u64, 40), decoded.artifacts[4].published_generation); + try std.testing.expectEqual(@as(u64, 39), decoded.artifacts[4].edge_generation); + try std.testing.expectEqual(@as(u64, 123), decoded.artifacts[4].computed_at_ms); + try std.testing.expectEqual(@as(u64, 0x1234), decoded.artifacts[4].materializer_fingerprint); + try std.testing.expectEqual(@as(u32, 73), decoded.artifacts[4].graph_metric_control_len); + try std.testing.expectEqual(@as(u32, 17), decoded.artifacts[4].graph_metric_routing_footer_len); + try std.testing.expectEqualSlices(u8, &([_]u8{0x11} ** 32), &decoded.artifacts[4].graph_metric_control_checksum); + try std.testing.expectEqualSlices(u8, &([_]u8{0x44} ** 32), &decoded.artifacts[4].graph_metric_point_index_checksum); + try std.testing.expectEqualSlices(u8, &([_]u8{0x55} ** 32), &decoded.artifacts[4].graph_metric_topology_checksum); + + // The current manifest layout retains the old field positions, but graph + // metrics deliberately fail closed if a pre-release version is forged. + const graph_metric_integrity_bytes: usize = 4 + 4 + 32 + 32 + 32 + 8 + 32 + 32 + 1 + 1; + const materializer_bytes: usize = 8; + const provenance_bytes: usize = 2 + 8 + 8 + 8; + var current_artifact_bytes: usize = 0; + for (manifest.artifacts) |artifact| current_artifact_bytes += artifactEncodedSize(artifact); + const prefix_len = encoded_a.len - current_artifact_bytes; + const encoded_v14 = try alloc.alloc(u8, encoded_a.len - materializer_bytes * manifest.artifacts.len - graph_metric_integrity_bytes - 32); + defer alloc.free(encoded_v14); + @memcpy(encoded_v14[0..prefix_len], encoded_a[0..prefix_len]); + var src_pos = prefix_len; + var dst_pos = prefix_len; + const artifact_header_bytes: usize = 1 + 4 + 4 + 8 + 4; + for (manifest.artifacts) |artifact| { + const retained_header_bytes = artifact_header_bytes + provenance_bytes; + @memcpy(encoded_v14[dst_pos..][0..retained_header_bytes], encoded_a[src_pos..][0..retained_header_bytes]); + src_pos += retained_header_bytes + materializer_bytes; + if (artifact.kind == .graph_metric_segment) src_pos += graph_metric_integrity_bytes; + if (artifact.kind == .graph_segment) src_pos += 32; + dst_pos += retained_header_bytes; + const payload_len = artifact.name.len + artifact.artifact_id.len + artifact.checksum.len; + @memcpy(encoded_v14[dst_pos..][0..payload_len], encoded_a[src_pos..][0..payload_len]); + src_pos += payload_len; + dst_pos += payload_len; + } + std.mem.writeInt(u16, encoded_v14[4..6], 14, .little); + try std.testing.expectError(error.UnsupportedManifestVersion, decodeAlloc(alloc, encoded_v14)); + + const encoded_v13 = try alloc.alloc(u8, encoded_v14.len - provenance_bytes * manifest.artifacts.len); + defer alloc.free(encoded_v13); + @memcpy(encoded_v13[0..prefix_len], encoded_v14[0..prefix_len]); + src_pos = prefix_len; + dst_pos = prefix_len; + for (manifest.artifacts) |artifact| { + @memcpy(encoded_v13[dst_pos..][0..artifact_header_bytes], encoded_v14[src_pos..][0..artifact_header_bytes]); + src_pos += artifact_header_bytes + provenance_bytes; + dst_pos += artifact_header_bytes; + const payload_len = artifact.name.len + artifact.artifact_id.len + artifact.checksum.len; + @memcpy(encoded_v13[dst_pos..][0..payload_len], encoded_v14[src_pos..][0..payload_len]); + src_pos += payload_len; + dst_pos += payload_len; + } + std.mem.writeInt(u16, encoded_v13[4..6], 13, .little); + try std.testing.expectError(error.UnsupportedManifestVersion, decodeAlloc(alloc, encoded_v13)); } -test "manifest codec round-trips optional lake base source" { +test "serverless manifest codec round-trips optional lake base source" { const alloc = std.testing.allocator; var manifest = manifest_types.Manifest{ .namespace = try alloc.dupe(u8, "events"), @@ -1105,13 +1190,111 @@ test "manifest codec round-trips optional lake base source" { try std.testing.expectEqualStrings("external-files-0001", decoded.artifacts[0].artifact_id); } -test "manifest codec rejects bad magic" { +test "serverless manifest codec rejects bad magic" { const alloc = std.testing.allocator; const bad = [_]u8{ 'B', 'A', 'D', '!', 1, 0 }; try std.testing.expectError(error.InvalidManifest, decodeAlloc(alloc, &bad)); } -test "lake manifest base source decoder rejects forged string-list counts before allocation" { +test "serverless manifest readers and writers require the latest wire" { + const alloc = std.testing.allocator; + var manifest = manifest_types.Manifest{ + .namespace = "docs", + .version = 1, + .built_at_ns = 1, + .wal_start_lsn = 0, + .wal_end_lsn = 0, + .stats = .{}, + .artifacts = &.{}, + }; + const encoded = try encodeForVersionAlloc(alloc, manifest, wire_version); + defer alloc.free(encoded); + try std.testing.expectEqual(wire_version, std.mem.readInt(u16, encoded[4..6], .little)); + var decoded = try decodeAlloc(alloc, encoded); + defer decoded.deinit(alloc); + try std.testing.expectEqualStrings("docs", decoded.namespace); + + for (0..wire_version + 2) |candidate| { + if (candidate == wire_version) continue; + const unsupported: u16 = @intCast(candidate); + const forged = try alloc.dupe(u8, encoded); + defer alloc.free(forged); + std.mem.writeInt(u16, forged[4..6], unsupported, .little); + try std.testing.expectError(error.UnsupportedManifestVersion, decodeAlloc(alloc, forged)); + try std.testing.expectError(error.UnsupportedManifestWriteVersion, encodeForVersionAlloc(alloc, manifest, unsupported)); + } + for (0..header_size) |len| { + try std.testing.expectError(error.InvalidManifest, decodeAlloc(alloc, encoded[0..len])); + } + + var graph_artifacts = [_]manifest_types.ArtifactRef{.{ + .kind = .graph_metric_segment, + .artifact_id = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + .byte_len = 1, + .checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + .metadata_version = artifact_ref.graph_metric_segment_wire_version, + .graph_metric_control_len = 1, + .graph_metric_routing_footer_len = 1, + }}; + manifest.artifacts = &graph_artifacts; + try std.testing.expectError( + error.UnsupportedManifestWriteVersion, + encodeForVersionAlloc(alloc, manifest, 15), + ); + try std.testing.expectError( + error.UnsupportedManifestWriteVersion, + encodeForVersionAlloc(alloc, manifest, 16), + ); + const encoded_current = try encodeForVersionAlloc(alloc, manifest, wire_version); + defer alloc.free(encoded_current); + var decoded_current = try decodeAlloc(alloc, encoded_current); + defer decoded_current.deinit(alloc); + try std.testing.expectEqual(artifact_ref.graph_metric_segment_wire_version, decoded_current.artifacts[0].metadata_version); + for ([_]u16{ 15, 16, 17 }) |old_version| { + const forged = try alloc.dupe(u8, encoded_current); + defer alloc.free(forged); + std.mem.writeInt(u16, forged[4..6], old_version, .little); + try std.testing.expectError(error.UnsupportedManifestVersion, decodeAlloc(alloc, forged)); + try std.testing.expectError(error.UnsupportedManifestWriteVersion, encodeForVersionAlloc(alloc, manifest, old_version)); + } + try std.testing.expectError( + error.UnsupportedManifestWriteVersion, + encodeForVersionAlloc(alloc, manifest, 13), + ); +} + +test "serverless manifest rejects malformed descriptor tags without leaking names" { + const alloc = std.testing.allocator; + var sources = [_]search_sources.SearchSourceDescriptor{.{ .vector = .{ + .index_name = "vec", + .document_source = .top_level_embedding, + } }}; + var outputs = [_]search_sources.DerivedOutputDescriptor{.{ .name = "preview", .kind = .chunk_preview }}; + const manifest = manifest_types.Manifest{ + .namespace = "docs", + .version = 1, + .built_at_ns = 1, + .wal_start_lsn = 0, + .wal_end_lsn = 0, + .stats = .{ + .published_search_sources = .{ .items = &sources }, + .derived_outputs = .{ .items = &outputs }, + }, + .artifacts = &.{}, + }; + const encoded = try encodeAlloc(alloc, manifest); + defer alloc.free(encoded); + const source_offset = header_size + manifest.namespace.len; + const output_offset = source_offset + publishedSearchSourceEncodedSize(sources[0]); + for ([_]usize{ source_offset, source_offset + 1, output_offset }) |offset| { + const saved = encoded[offset]; + encoded[offset] = 255; + try std.testing.expectError(error.InvalidManifest, decodeAlloc(alloc, encoded)); + encoded[offset] = saved; + } +} + +test "serverless lake manifest base source decoder rejects forged string-list counts before allocation" { const alloc = std.testing.allocator; var encoded = [_]u8{0} ** 13; encoded[0] = @intFromEnum(manifest_types.BaseSourceKind.antfly_row_fragments); diff --git a/zig/pkg/antfly/src/serverless/manifest/compatibility.zig b/zig/pkg/antfly/src/serverless/manifest/compatibility.zig index 4301a98be5..dfbd5f485c 100644 --- a/zig/pkg/antfly/src/serverless/manifest/compatibility.zig +++ b/zig/pkg/antfly/src/serverless/manifest/compatibility.zig @@ -51,7 +51,7 @@ pub fn checkLakeBaseSource( .row_fragment_stats => report.row_fragment_stats_count += 1, .algebraic_segment => report.algebraic_segment_count += 1, .external_base_source => report.external_metadata_count += 1, - .text_segment, .vector_segment, .sparse_segment, .graph_segment => report.sidecar_count += 1, + .text_segment, .vector_segment, .sparse_segment, .graph_segment, .graph_metric_segment => report.sidecar_count += 1, .doc_values, .stored_fields, .mutation_segment, .document_segment => {}, } } @@ -77,7 +77,9 @@ fn validateArtifacts(artifacts: []const artifact_ref.ArtifactRef) !void { if (artifact.artifact_id.len == 0) return error.IncompatibleLakeManifest; if (artifact.checksum.len == 0) return error.IncompatibleLakeManifest; for (artifacts[0..idx]) |previous| { - if (std.mem.eql(u8, previous.artifact_id, artifact.artifact_id)) { + if (std.mem.eql(u8, previous.artifact_id, artifact.artifact_id) and + !artifact_ref.areGraphArtifactAliases(previous, artifact)) + { return error.DuplicateLakeManifestArtifact; } } @@ -138,6 +140,16 @@ fn hasArtifact( return false; } +test "serverless lake manifest compatibility permits consistent graph metric aliases" { + const original = artifact_ref.ArtifactRef{ .kind = .graph_metric_segment, .name = "1:a1:x", .artifact_id = "metric", .checksum = "checksum", .byte_len = 128, .metadata_version = artifact_ref.graph_metric_segment_wire_version }; + var alias = original; + alias.name = "1:b1:y"; + try validateArtifacts(&.{ original, alias }); + alias.graph_metric_control_len += 1; + try std.testing.expectError(error.DuplicateLakeManifestArtifact, validateArtifacts(&.{ original, alias })); + try std.testing.expectError(error.DuplicateLakeManifestArtifact, validateArtifacts(&.{ original, original })); +} + test "lake manifest compatibility accepts row fragments with stats" { const row_fragments = [_][]const u8{"rows-1"}; const stats = [_][]const u8{"rows-1.stats"}; diff --git a/zig/pkg/antfly/src/serverless/manifest/object_store.zig b/zig/pkg/antfly/src/serverless/manifest/object_store.zig index bf63ab07ea..773a69a1c2 100644 --- a/zig/pkg/antfly/src/serverless/manifest/object_store.zig +++ b/zig/pkg/antfly/src/serverless/manifest/object_store.zig @@ -28,6 +28,7 @@ pub const ObjectStore = struct { alloc: std.mem.Allocator, opened: object_store_support.OpenedObjectStore, clock: platform_clock.Clock, + write_version: u16 = manifest_codec.wire_version, pub fn initRemoteUri(alloc: std.mem.Allocator, uri: []const u8) !ObjectStore { return try initRemoteUriWithS3Options(alloc, uri, null); @@ -90,10 +91,17 @@ pub const ObjectStore = struct { }; } + pub fn setWriteVersion(self: *ObjectStore, write_version: u16) !void { + if (write_version != manifest_codec.wire_version) { + return error.UnsupportedManifestWriteVersion; + } + self.write_version = write_version; + } + pub fn put(self: *ObjectStore, manifest: manifest_types.Manifest) !void { const key = try manifestKeyAlloc(self.alloc, self.opened.prefix, manifest.namespace, manifest.version); defer self.alloc.free(key); - const encoded = try manifest_codec.encodeAlloc(self.alloc, manifest); + const encoded = try manifest_codec.encodeForVersionAlloc(self.alloc, manifest, self.write_version); defer self.alloc.free(encoded); if (try self.tryGetEncoded(self.alloc, key)) |existing| { diff --git a/zig/pkg/antfly/src/serverless/manifest/types.zig b/zig/pkg/antfly/src/serverless/manifest/types.zig index c323320d0d..d79c08e1d9 100644 --- a/zig/pkg/antfly/src/serverless/manifest/types.zig +++ b/zig/pkg/antfly/src/serverless/manifest/types.zig @@ -125,6 +125,22 @@ fn cloneArtifactRefAlloc(alloc: Allocator, artifact: ArtifactRef) !ArtifactRef { .artifact_id = artifact_id, .byte_len = artifact.byte_len, .checksum = checksum, + .metadata_version = artifact.metadata_version, + .published_generation = artifact.published_generation, + .edge_generation = artifact.edge_generation, + .computed_at_ms = artifact.computed_at_ms, + .materializer_fingerprint = artifact.materializer_fingerprint, + .graph_metric_control_len = artifact.graph_metric_control_len, + .graph_metric_routing_footer_len = artifact.graph_metric_routing_footer_len, + .graph_metric_control_checksum = artifact.graph_metric_control_checksum, + .graph_topology_control_checksum = artifact.graph_topology_control_checksum, + .graph_metric_routing_checksum = artifact.graph_metric_routing_checksum, + .graph_metric_point_index_checksum = artifact.graph_metric_point_index_checksum, + .graph_metric_config_fingerprint = artifact.graph_metric_config_fingerprint, + .graph_metric_source_checksum = artifact.graph_metric_source_checksum, + .graph_metric_topology_checksum = artifact.graph_metric_topology_checksum, + .graph_metric_materialization_state = artifact.graph_metric_materialization_state, + .graph_metric_rejection_reason = artifact.graph_metric_rejection_reason, }; } diff --git a/zig/pkg/antfly/src/serverless/mod.zig b/zig/pkg/antfly/src/serverless/mod.zig index d982d8a258..94feb1500e 100644 --- a/zig/pkg/antfly/src/serverless/mod.zig +++ b/zig/pkg/antfly/src/serverless/mod.zig @@ -27,6 +27,7 @@ pub const text_segment = @import("text_segment/mod.zig"); pub const sparse_segment = @import("sparse_segment/mod.zig"); pub const vector_segment = @import("vector_segment/mod.zig"); pub const graph_segment = @import("graph_segment/mod.zig"); +pub const graph_metric_segment = @import("graph_metric_segment/mod.zig"); pub const row_fragment = @import("row_fragment/mod.zig"); pub const algebraic_segment = @import("algebraic_segment/mod.zig"); pub const external_source = @import("external_source/mod.zig"); @@ -284,6 +285,12 @@ pub const LakeGraphSidecarBuildResult = build.LakeGraphSidecarBuildResult; pub const LakeGraphSidecarPublishResult = build.LakeGraphSidecarPublishResult; pub const buildLakeGraphSidecarFromRowSourceAlloc = build.buildLakeGraphSidecarFromRowSourceAlloc; pub const publishLakeGraphSidecarFromRowSourceAlloc = build.publishLakeGraphSidecarFromRowSourceAlloc; +pub const LakeGraphMetricBuildLimits = build.LakeGraphMetricBuildLimits; +pub const LakeGraphMetricBuildOptions = build.LakeGraphMetricBuildOptions; +pub const LakeGraphMetricBuildResult = build.LakeGraphMetricBuildResult; +pub const buildLakeGraphMetricFromGraphPayloadAlloc = build.buildLakeGraphMetricFromGraphPayloadAlloc; +pub const publishLakeGraphMetricFromGraphPayloadAlloc = build.publishLakeGraphMetricFromGraphPayloadAlloc; +pub const publishLakeGraphMetricFromGraphArtifactAlloc = build.publishLakeGraphMetricFromGraphArtifactAlloc; pub const LakeSparseSidecarBuildOptions = build.LakeSparseSidecarBuildOptions; pub const LakeSparseSidecarBuildResult = build.LakeSparseSidecarBuildResult; pub const LakeSparseSidecarPublishResult = build.LakeSparseSidecarPublishResult; @@ -387,6 +394,8 @@ pub const executeLakeRebuildOperationsWithOptionsAlloc = build.executeLakeRebuil pub const deleteDroppedLakeRebuildArtifactsAfterPublishAlloc = build.deleteDroppedLakeRebuildArtifactsAfterPublishAlloc; pub const reconcileLakeRebuildExecutedOperationsAlloc = build.reconcileLakeRebuildExecutedOperationsAlloc; pub const reconcileResolvedExternalLakeSidecarsAlloc = build.reconcileResolvedExternalLakeSidecarsAlloc; +pub const reconcileResolvedExternalLakeSidecarsWithCancellationAlloc = build.reconcileResolvedExternalLakeSidecarsWithCancellationAlloc; +pub const reconcileResolvedExternalLakeSidecarsWithRuntimeAlloc = build.reconcileResolvedExternalLakeSidecarsWithRuntimeAlloc; pub const LakeRangeCacheLane = query.LakeRangeCacheLane; pub const LakeRangePurpose = query.LakeRangePurpose; pub const LakeRangeObjectVersion = query.LakeRangeObjectVersion; @@ -711,6 +720,12 @@ test "serverless module compiles" { _ = LakeGraphSidecarPublishResult; _ = buildLakeGraphSidecarFromRowSourceAlloc; _ = publishLakeGraphSidecarFromRowSourceAlloc; + _ = LakeGraphMetricBuildLimits; + _ = LakeGraphMetricBuildOptions; + _ = LakeGraphMetricBuildResult; + _ = buildLakeGraphMetricFromGraphPayloadAlloc; + _ = publishLakeGraphMetricFromGraphPayloadAlloc; + _ = publishLakeGraphMetricFromGraphArtifactAlloc; _ = LakeSparseSidecarBuildOptions; _ = LakeSparseSidecarBuildResult; _ = LakeSparseSidecarPublishResult; @@ -824,6 +839,8 @@ test "serverless module compiles" { _ = deleteDroppedLakeRebuildArtifactsAfterPublishAlloc; _ = reconcileLakeRebuildExecutedOperationsAlloc; _ = reconcileResolvedExternalLakeSidecarsAlloc; + _ = reconcileResolvedExternalLakeSidecarsWithCancellationAlloc; + _ = reconcileResolvedExternalLakeSidecarsWithRuntimeAlloc; _ = LakeRangeCacheLane; _ = LakeRangePurpose; _ = LakeRangeObjectVersion; diff --git a/zig/pkg/antfly/src/serverless/query/authenticated_block_fills.zig b/zig/pkg/antfly/src/serverless/query/authenticated_block_fills.zig new file mode 100644 index 0000000000..e9a80238f2 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/query/authenticated_block_fills.zig @@ -0,0 +1,454 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../api/operation.zig").CancellationToken; + +pub fn blockKey(artifact_id: []const u8, artifact_checksum: []const u8, offset: u64, len: usize, checksum: *const [32]u8) [32]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update(artifact_id); + hash.update(artifact_checksum); + hash.update(checksum); + var extent: [16]u8 = undefined; + std.mem.writeInt(u64, extent[0..8], offset, .little); + std.mem.writeInt(u64, extent[8..16], len, .little); + hash.update(&extent); + var key: [32]u8 = undefined; + hash.final(&key); + return key; +} + +/// Shared authenticated blocks, independent of transport grouping. Batches +/// claim all missing keys atomically and never wait while owning new fills. +pub const Cache = struct { + const max_entries = 4096; + const max_bytes = 64 * 1024 * 1024; + const Entry = struct { + key: [32]u8, + data: []u8, + refs: usize = 0, + state: enum { pending, ready, failed } = .pending, + touched: u64 = 0, + }; + pub const Spec = struct { key: [32]u8, len: usize }; + pub const Item = struct { + entry: *Entry, + producer: bool, + pub fn bytes(self: Item) []const u8 { + return self.entry.data; + } + pub fn buffer(self: Item) []u8 { + std.debug.assert(self.producer and self.entry.state == .pending); + return self.entry.data; + } + }; + pub const Batch = struct { + cache: *Cache, + alloc: Allocator, + items: []Item, + pub fn publish(self: *Batch, io: ?std.Io) void { + const cache = self.cache; + cache.lock(); + for (self.items) |item| if (item.producer) { + std.debug.assert(item.entry.state == .pending); + item.entry.state = .ready; + }; + cache.wake(io); + cache.mu.unlock(); + } + pub fn deinit(self: *Batch) void { + const cache = self.cache; + cache.lock(); + for (self.items) |item| cache.releaseLocked(item.entry, item.producer); + cache.wake(null); + cache.mu.unlock(); + self.alloc.free(self.items); + self.* = undefined; + } + }; + pub const Waiter = struct { + cache: *Cache, + entry: *Entry, + pub fn deinit(self: *Waiter) void { + self.cache.lock(); + self.cache.waiters -= 1; + self.cache.releaseLocked(self.entry, false); + self.cache.wake(null); + self.cache.mu.unlock(); + self.* = undefined; + } + pub fn awaitReady(self: *Waiter, io: ?std.Io, cancellation: CancellationToken) !void { + while (true) { + try cancellation.check(); + self.cache.lock(); + const ready = self.entry.state != .pending; + const epoch = self.cache.epoch.load(.acquire); + self.cache.mu.unlock(); + if (ready) return; + try self.cache.wait(io, cancellation, epoch); + } + } + }; + pub const Lookup = union(enum) { batch: Batch, wait: Waiter, saturated: u32 }; + mu: std.atomic.Mutex = .unlocked, + entries: std.AutoHashMapUnmanaged([32]u8, *Entry) = .empty, + alloc: ?Allocator = null, + retained: usize = 0, + live_entries: usize = 0, + waiters: usize = 0, + clock: u64 = 0, + epoch: std.atomic.Value(u32) = .init(0), + + fn lock(self: *Cache) void { + @import("antfly_platform").sync.lockYielding(&self.mu); + } + fn wake(self: *Cache, io: ?std.Io) void { + _ = self.epoch.fetchAdd(1, .release); + (io orelse std.Options.debug_io).futexWake(u32, &self.epoch.raw, std.math.maxInt(u32)); + } + pub fn wait(self: *Cache, io: ?std.Io, cancellation: CancellationToken, epoch: u32) !void { + try cancellation.check(); + try (io orelse std.Options.debug_io).futexWaitTimeout(u32, &self.epoch.raw, epoch, .{ .duration = .{ .raw = .fromMilliseconds(10), .clock = .awake } }); + try cancellation.check(); + } + fn destroyLocked(self: *Cache, entry: *Entry) void { + self.retained -= entry.data.len; + self.live_entries -= 1; + self.alloc.?.free(entry.data); + self.alloc.?.destroy(entry); + } + fn releaseLocked(self: *Cache, entry: *Entry, producer: bool) void { + if (producer and entry.state == .pending) { + _ = self.entries.remove(entry.key); + entry.state = .failed; + } + std.debug.assert(entry.refs > 0); + entry.refs -= 1; + if (entry.refs == 0 and entry.state == .failed) self.destroyLocked(entry); + } + + pub fn begin(self: *Cache, owner: Allocator, result_alloc: Allocator, specs: []const Spec) !Lookup { + var bytes: usize = 0; + for (specs) |spec| { + if (spec.len == 0) return error.InvalidCacheBatch; + bytes = std.math.add(usize, bytes, spec.len) catch return error.InvalidCacheBatch; + } + if (specs.len > max_entries or bytes > 8 * 1024 * 1024) return error.InvalidCacheBatch; + const items = try result_alloc.alloc(Item, specs.len); + var transferred = false; + defer if (!transferred) result_alloc.free(items); + self.lock(); + defer self.mu.unlock(); + if (self.alloc == null) self.alloc = owner; + self.clock +%= 1; + var needed_bytes: usize = 0; + var needed_entries: usize = 0; + for (specs) |spec| { + if (self.entries.get(spec.key)) |entry| { + if (entry.data.len != spec.len or entry.touched == self.clock) return error.InvalidCacheBatch; + entry.touched = self.clock; + if (entry.state == .pending) { + entry.refs += 1; + self.waiters += 1; + return .{ .wait = .{ .cache = self, .entry = entry } }; + } + } else { + needed_bytes += spec.len; + needed_entries += 1; + } + } + if (self.retained + needed_bytes > max_bytes or self.live_entries + needed_entries > max_entries) { + // Select the whole eviction batch once; rescanning the table for + // every victim makes a broad cold miss quadratic under this lock. + var victims: [max_entries]*Entry = undefined; + var victim_count: usize = 0; + var reclaimable: usize = 0; + var it = self.entries.valueIterator(); + while (it.next()) |ptr| { + const entry = ptr.*; + if (entry.refs != 0 or entry.state != .ready or entry.touched == self.clock) continue; + victims[victim_count] = entry; + victim_count += 1; + reclaimable += entry.data.len; + } + if (self.retained - reclaimable + needed_bytes > max_bytes or self.live_entries - victim_count + needed_entries > max_entries) + return .{ .saturated = self.epoch.load(.acquire) }; + std.mem.sort(*Entry, victims[0..victim_count], {}, struct { + fn less(_: void, a: *Entry, b: *Entry) bool { + return a.touched < b.touched; + } + }.less); + for (victims[0..victim_count]) |entry| { + if (self.retained + needed_bytes <= max_bytes and self.live_entries + needed_entries <= max_entries) break; + _ = self.entries.remove(entry.key); + self.destroyLocked(entry); + } + } + const a = self.alloc.?; + try self.entries.ensureUnusedCapacity(a, @intCast(needed_entries)); + var initialized: usize = 0; + errdefer { + for (items[0..initialized]) |item| self.releaseLocked(item.entry, item.producer); + self.wake(null); + } + for (specs, items) |spec, *item| { + if (self.entries.get(spec.key)) |entry| { + // Duplicate new keys must not turn one batch into its own waiter. + if (entry.state == .pending) return error.InvalidCacheBatch; + entry.refs += 1; + item.* = .{ .entry = entry, .producer = false }; + } else { + const entry = try a.create(Entry); + errdefer a.destroy(entry); + const data = try a.alloc(u8, spec.len); + entry.* = .{ .key = spec.key, .data = data, .refs = 1, .touched = self.clock }; + self.entries.putAssumeCapacity(spec.key, entry); + self.retained += spec.len; + self.live_entries += 1; + item.* = .{ .entry = entry, .producer = true }; + } + initialized += 1; + } + transferred = true; + return .{ .batch = .{ .cache = self, .alloc = result_alloc, .items = items } }; + } + pub fn acquire(self: *Cache, owner: Allocator, result_alloc: Allocator, specs: []const Spec, io: ?std.Io, cancellation: CancellationToken) !Batch { + while (true) { + try cancellation.check(); + switch (try self.begin(owner, result_alloc, specs)) { + .batch => |batch| return batch, + .wait => |registered| { + var waiter = registered; + defer waiter.deinit(); + try waiter.awaitReady(io, cancellation); + }, + .saturated => |epoch| try self.wait(io, cancellation, epoch), + } + } + } + pub fn deinit(self: *Cache) void { + var it = self.entries.valueIterator(); + while (it.next()) |ptr| { + std.debug.assert(ptr.*.refs == 0 and ptr.*.state == .ready); + self.destroyLocked(ptr.*); + } + if (self.alloc) |a| self.entries.deinit(a); + std.debug.assert(self.retained == 0); + self.* = undefined; + } + + pub fn snapshot(self: *Cache) struct { bytes: usize, entries: usize, waiters: usize } { + self.lock(); + defer self.mu.unlock(); + return .{ .bytes = self.retained, .entries = self.live_entries, .waiters = self.waiters }; + } + + pub const Lease = struct { + cache: *Cache, + entry: *Entry, + + pub fn bytes(self: Lease) []const u8 { + return self.entry.data; + } + + pub fn deinit(self: *Lease) void { + self.cache.lock(); + self.cache.releaseLocked(self.entry, false); + self.cache.wake(null); + self.cache.mu.unlock(); + self.* = undefined; + } + }; + + pub fn leaseIfReady(self: *Cache, key: [32]u8) ?Lease { + self.lock(); + const entry = self.entries.get(key) orelse { + self.mu.unlock(); + return null; + }; + if (entry.state != .ready) { + self.mu.unlock(); + return null; + } + entry.refs += 1; + self.clock +%= 1; + entry.touched = self.clock; + self.mu.unlock(); + return .{ .cache = self, .entry = entry }; + } + + pub fn copyIfReadyAlloc(self: *Cache, result_alloc: Allocator, key: [32]u8) !?[]u8 { + var lease = self.leaseIfReady(key) orelse return null; + defer lease.deinit(); + return try result_alloc.dupe(u8, lease.bytes()); + } + + /// Optional disk-hit promotion. Never wait: the caller may itself own a + /// pending fill, and retention pressure must not prevent serving a hit. + /// The caller authenticates bytes against this canonical key first. + pub fn retainVerified(self: *Cache, owner: Allocator, key: [32]u8, bytes: []const u8) void { + switch (self.begin(owner, owner, &.{.{ .key = key, .len = bytes.len }}) catch return) { + .batch => |reserved| { + var batch = reserved; + defer batch.deinit(); + if (batch.items[0].producer) @memcpy(batch.items[0].buffer(), bytes); + batch.publish(null); + }, + .wait => |registered| { + var waiter = registered; + waiter.deinit(); + }, + .saturated => {}, + } + } +}; + +test "serverless canonical block disk promotion bypasses pending fills and allocation pressure" { + const alloc = std.testing.allocator; + var cache = Cache{}; + defer cache.deinit(); + const key = [_]u8{7} ** 32; + var pending = try cache.acquire(alloc, alloc, &.{.{ .key = key, .len = 4 }}, null, .none); + defer pending.deinit(); + cache.retainVerified(alloc, key, "data"); + try std.testing.expectEqual(@as(usize, 0), cache.snapshot().waiters); + try std.testing.expect(cache.leaseIfReady(key) == null); + @memcpy(pending.items[0].buffer(), "data"); + pending.publish(null); + var lease = cache.leaseIfReady(key).?; + defer lease.deinit(); + try std.testing.expectEqualStrings("data", lease.bytes()); + + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + var pressured = Cache{}; + defer pressured.deinit(); + pressured.retainVerified(failing.allocator(), key, "data"); + try std.testing.expectEqual(@as(usize, 0), pressured.snapshot().entries); +} + +test "serverless canonical block fills atomically share overlapping transport sets" { + const alloc = std.testing.allocator; + var cache = Cache{}; + defer cache.deinit(); + const a = Cache.Spec{ .key = @splat(1), .len = 4 }; + const b = Cache.Spec{ .key = @splat(2), .len = 4 }; + const c = Cache.Spec{ .key = @splat(3), .len = 4 }; + var first = (try cache.begin(alloc, alloc, &.{ a, b })).batch; + var waiter = (try cache.begin(alloc, alloc, &.{ b, c })).wait; + defer waiter.deinit(); + try std.testing.expectEqual(@as(usize, 2), cache.entries.count()); + @memcpy(first.items[0].buffer(), "aaaa"); + @memcpy(first.items[1].buffer(), "bbbb"); + first.publish(null); + first.deinit(); + try waiter.awaitReady(null, .none); + var second = try cache.acquire(alloc, alloc, &.{ b, c }, null, .none); + defer second.deinit(); + try std.testing.expect(!second.items[0].producer); + try std.testing.expect(second.items[1].producer); + try std.testing.expectEqualStrings("bbbb", second.items[0].bytes()); + @memcpy(second.items[1].buffer(), "cccc"); + second.publish(null); +} + +test "serverless canonical block fill cancellation leaves producer alive and failure permits takeover" { + const alloc = std.testing.allocator; + var cache = Cache{}; + defer cache.deinit(); + const spec = Cache.Spec{ .key = @splat(1), .len = 4 }; + var producer = (try cache.begin(alloc, alloc, &.{spec})).batch; + var waiter = (try cache.begin(alloc, alloc, &.{spec})).wait; + const Cancel = struct { + fn canceled(_: *const anyopaque) bool { + return true; + } + }; + try std.testing.expectError(error.Canceled, waiter.awaitReady(null, .{ .ptr = &cache, .is_cancelled_fn = Cancel.canceled })); + waiter.deinit(); + try std.testing.expectEqual(@as(usize, 1), producer.items[0].entry.refs); + waiter = (try cache.begin(alloc, alloc, &.{spec})).wait; + producer.deinit(); + try waiter.awaitReady(null, .none); + try std.testing.expectEqual(@as(usize, 1), cache.live_entries); + waiter.deinit(); + try std.testing.expectEqual(@as(usize, 0), cache.live_entries); + var replacement = try cache.acquire(alloc, alloc, &.{spec}, null, .none); + defer replacement.deinit(); + try std.testing.expect(replacement.items[0].producer); + @memcpy(replacement.items[0].buffer(), "good"); + replacement.publish(null); +} + +test "serverless canonical block fill allocations unwind every partial claim" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var cache = Cache{}; + defer cache.deinit(); + var batch = try cache.acquire(alloc, alloc, &.{ + .{ .key = @splat(1), .len = 4 }, .{ .key = @splat(2), .len = 4 }, + }, null, .none); + defer batch.deinit(); + for (batch.items) |item| @memset(item.buffer(), 0); + batch.publish(null); + const copied = (try cache.copyIfReadyAlloc(alloc, @splat(1))).?; + defer alloc.free(copied); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "serverless canonical block fills reject duplicate keys without leaking pending claims" { + const alloc = std.testing.allocator; + var cache = Cache{}; + defer cache.deinit(); + const spec = Cache.Spec{ .key = @splat(1), .len = 4 }; + try std.testing.expectError(error.InvalidCacheBatch, cache.begin(alloc, alloc, &.{ spec, spec })); + try std.testing.expectEqual(@as(usize, 0), cache.live_entries); +} + +test "serverless canonical block admission bounds failed pinned entries and preserves requested hits on eviction" { + const alloc = std.testing.allocator; + var cache = Cache{}; + defer cache.deinit(); + const specs = try alloc.alloc(Cache.Spec, Cache.max_entries); + defer alloc.free(specs); + for (specs, 0..) |*spec, i| { + spec.* = .{ .key = @splat(0), .len = 1 }; + std.mem.writeInt(u64, spec.key[0..8], i, .little); + } + var producer = (try cache.begin(alloc, alloc, specs)).batch; + var waiter = (try cache.begin(alloc, alloc, specs[0..1])).wait; + const extra = Cache.Spec{ .key = @splat(255), .len = 1 }; + try std.testing.expect((try cache.begin(alloc, alloc, &.{extra})) == .saturated); + cache.retainVerified(alloc, extra.key, "x"); + try std.testing.expectEqual(Cache.max_entries, cache.snapshot().entries); + try std.testing.expect(cache.leaseIfReady(extra.key) == null); + producer.deinit(); + // The failed entry has left the map, but its waiter still pins memory. + try std.testing.expectEqual(@as(usize, 1), cache.live_entries); + try std.testing.expect((try cache.begin(alloc, alloc, specs)) == .saturated); + waiter.deinit(); + producer = (try cache.begin(alloc, alloc, specs)).batch; + for (producer.items) |item| @memset(item.buffer(), 42); + producer.publish(null); + producer.deinit(); + var mixed = (try cache.begin(alloc, alloc, &.{ specs[0], extra })).batch; + defer mixed.deinit(); + try std.testing.expect(!mixed.items[0].producer and mixed.items[1].producer); + try std.testing.expectEqual(@as(u8, 42), mixed.items[0].bytes()[0]); + @memset(mixed.items[1].buffer(), 1); + mixed.publish(null); + try std.testing.expectEqual(@as(usize, Cache.max_entries), cache.live_entries); +} diff --git a/zig/pkg/antfly/src/serverless/query/authenticated_block_persistence.zig b/zig/pkg/antfly/src/serverless/query/authenticated_block_persistence.zig new file mode 100644 index 0000000000..68330da5a0 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/query/authenticated_block_persistence.zig @@ -0,0 +1,140 @@ +// Copyright 2026 Antfly, Inc. +// SPDX-License-Identifier: Elastic-2.0 + +//! Optional cache-owned disk retention. No request allocator, cancellation +//! token, or executor outlives its request. One std.Io worker drains a bounded +//! queue; pressure drops retention, never authenticated query results. +const std = @import("std"); +const cache_mod = @import("cache.zig"); +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; + +pub const max_jobs = 32; +pub const max_bytes = 16 * 1024 * 1024; + +pub const Worker = struct { + owner: *cache_mod.QueryCache, + io_impl: std.Io.Threaded, + group: std.Io.Group = .init, + mu: std.Io.Mutex = .init, + changed: std.Io.Condition = .init, + stop: std.atomic.Value(bool) = .init(false), + jobs: [max_jobs]Job = undefined, + head: usize = 0, + count: usize = 0, + outstanding: usize = 0, + bytes: usize = 0, + failures: usize = 0, + bypasses: usize = 0, + + const Job = struct { + storage: []u8, + artifact_id: []const u8, + checksum: []const u8, + byte_len: u64, + blocks: [cache_mod.max_authenticated_publication_blocks]cache_mod.AuthenticatedBlockPublication, + count: usize, + }; + + pub fn create(owner: *cache_mod.QueryCache) !*Worker { + const self = try owner.alloc.create(Worker); + errdefer owner.alloc.destroy(self); + self.* = .{ .owner = owner, .io_impl = std.Io.Threaded.init(owner.alloc, .{ .async_limit = .nothing, .concurrent_limit = .limited(1) }) }; + errdefer self.io_impl.deinit(); + // Unlike async, concurrent may not run the infinite worker inline. + try self.group.concurrent(self.io_impl.io(), run, .{self}); + return self; + } + + pub fn deinit(self: *Worker) void { + const io = self.io_impl.io(); + self.stop.store(true, .release); + self.mu.lockUncancelable(io); + self.changed.broadcast(io); + self.mu.unlock(io); + self.group.await(io) catch {}; + self.io_impl.deinit(); + self.owner.alloc.destroy(self); + } + + /// Only already authenticated immutable bytes may enter this queue. The + /// disk publisher independently checks digests before writing its records. + pub fn enqueue(self: *Worker, artifact_id: []const u8, byte_len: u64, checksum: []const u8, blocks: []const cache_mod.AuthenticatedBlockPublication) !bool { + if (blocks.len == 0) return true; + if (blocks.len > cache_mod.max_authenticated_publication_blocks) return error.InvalidCacheBatch; + var len = try std.math.add(usize, artifact_id.len, checksum.len); + for (blocks) |block| { + len = try std.math.add(usize, len, block.block_id.len); + len = try std.math.add(usize, len, block.contents.len); + } + const io = self.io_impl.io(); + self.mu.lockUncancelable(io); + defer self.mu.unlock(io); + if (self.stop.load(.acquire) or self.outstanding == max_jobs or len > max_bytes - self.bytes) { + self.bypasses += 1; + return false; + } + const storage = try self.owner.alloc.alloc(u8, len); + var remaining = storage; + var job = Job{ + .storage = storage, + .artifact_id = copy(&remaining, artifact_id), + .checksum = copy(&remaining, checksum), + .byte_len = byte_len, + .blocks = undefined, + .count = blocks.len, + }; + for (blocks, job.blocks[0..blocks.len]) |block, *owned| { + owned.* = block; + owned.block_id = copy(&remaining, block.block_id); + owned.contents = copy(&remaining, block.contents); + } + self.jobs[(self.head + self.count) % max_jobs] = job; + self.count += 1; + self.outstanding += 1; + self.bytes += len; + self.changed.signal(io); + return true; + } + + fn copy(remaining: *[]u8, bytes: []const u8) []const u8 { + const result = remaining.*[0..bytes.len]; + @memcpy(result, bytes); + remaining.* = remaining.*[bytes.len..]; + return result; + } + + fn run(self: *Worker) void { + const io = self.io_impl.io(); + while (true) { + self.mu.lockUncancelable(io); + while (self.count == 0 and !self.stop.load(.acquire)) self.changed.waitUncancelable(io, &self.mu); + if (self.count == 0) { + self.mu.unlock(io); + return; + } + const job = self.jobs[self.head]; + self.head = (self.head + 1) % max_jobs; + self.count -= 1; + self.mu.unlock(io); + var failed = false; + if (!self.stop.load(.acquire)) self.owner.publishAuthenticatedBlocks(job.artifact_id, job.byte_len, job.checksum, job.blocks[0..job.count], CancellationToken.fromAtomic(&self.stop)) catch { + failed = true; + }; + self.owner.alloc.free(job.storage); + self.mu.lockUncancelable(io); + self.bytes -= job.storage.len; + self.outstanding -= 1; + self.failures += @intFromBool(failed); + self.changed.broadcast(io); + self.mu.unlock(io); + } + } + + /// Maintenance/test barrier, never part of a query read. + pub fn drain(self: *Worker) void { + const io = self.io_impl.io(); + self.mu.lockUncancelable(io); + defer self.mu.unlock(io); + while (self.outstanding != 0) self.changed.waitUncancelable(io, &self.mu); + } +}; diff --git a/zig/pkg/antfly/src/serverless/query/cache.zig b/zig/pkg/antfly/src/serverless/query/cache.zig index 4f2fca93f9..3d793342dd 100644 --- a/zig/pkg/antfly/src/serverless/query/cache.zig +++ b/zig/pkg/antfly/src/serverless/query/cache.zig @@ -19,8 +19,13 @@ const Allocator = std.mem.Allocator; const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const fs_paths = @import("../../common/fs_paths.zig"); const artifacts_mod = @import("../artifacts/mod.zig"); +const graph_metric_routing_cache = @import("graph_metric_routing_cache.zig"); +const block_persistence = @import("authenticated_block_persistence.zig"); -const cache_record_magic = "AFQCR001"; +// v2 invalidates range/block entries written before provider identities were +// pinned across verification and fetch. Full entries remain cheap to rebuild +// and sharing one record format keeps startup reconciliation fail-closed. +const cache_record_magic = "AFQCR002"; const cache_record_digest_len = std.crypto.hash.sha2.Sha256.digest_length; const cache_record_header_len = cache_record_magic.len + @sizeOf(u64) + cache_record_digest_len; const abandoned_cache_write_age_ns: i96 = 24 * std.time.ns_per_hour; @@ -38,9 +43,37 @@ const abandoned_cache_write_suffix = ".abandoned"; pub const QueryCacheConfig = struct { max_bytes: u64 = 0, max_payload_bytes: u64 = 0, + /// Independent process-memory budget; disk-cache capacity is unchanged. + max_graph_metric_routing_bytes: usize = 16 * 1024 * 1024, +}; + +pub const AuthenticatedSubrange = struct { + relative_offset: usize, + len: usize, + checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, +}; + +// Bound simultaneously held publication leases, not only payload memory. +// A query may publish several transport responses concurrently. +pub const max_authenticated_publication_blocks = 32; +pub const AuthenticatedBlockPublication = struct { + block_id: []const u8, + offset: u64, + contents: []const u8, + checksum: [std.crypto.hash.sha2.Sha256.digest_length]u8, }; pub const QueryCacheStats = struct { + decoded_graph_metric_routing_hits: u64 = 0, + decoded_graph_metric_routing_misses: u64 = 0, + decoded_graph_metric_routing_bytes: u64 = 0, + shared_graph_metric_block_bytes: u64 = 0, + shared_graph_metric_block_entries: u64 = 0, + shared_graph_metric_block_waiters: u64 = 0, + graph_metric_persistence_bytes: u64 = 0, + graph_metric_persistence_jobs: u64 = 0, + graph_metric_persistence_failures: u64 = 0, + graph_metric_persistence_bypasses: u64 = 0, hits: u64 = 0, misses: u64 = 0, writes: u64 = 0, @@ -179,6 +212,26 @@ const CachePublicationLease = struct { } }; +pub const AuthenticatedBlockLease = union(enum) { + cached: @import("authenticated_block_fills.zig").Cache.Lease, + owned: struct { alloc: Allocator, data: []u8 }, + + pub fn bytes(self: AuthenticatedBlockLease) []const u8 { + return switch (self) { + .cached => |lease| lease.bytes(), + .owned => |value| value.data, + }; + } + + pub fn deinit(self: *AuthenticatedBlockLease) void { + switch (self.*) { + .cached => |*lease| lease.deinit(), + .owned => |value| value.alloc.free(value.data), + } + self.* = undefined; + } +}; + pub const QueryCache = struct { alloc: Allocator, root_dir: []u8, @@ -191,8 +244,15 @@ pub const QueryCache = struct { cfg: QueryCacheConfig, stats_mu: std.atomic.Mutex = .unlocked, maintenance_mu: std.atomic.Mutex = .unlocked, + // Batching must not multiply descriptor usage by concurrent queries. + // Saturation bypasses optional disk retention without delaying results. + publication_slots: std.atomic.Value(usize) = .init(0), usage: CacheUsage = .{}, stats: QueryCacheStats = .{}, + graph_metric_routing: graph_metric_routing_cache.Cache = .{}, + graph_metric_blocks: @import("authenticated_block_fills.zig").Cache = .{}, + persistence_mu: std.atomic.Mutex = .unlocked, + persistence: ?*block_persistence.Worker = null, pub fn init(alloc: Allocator, root_dir: []const u8) !QueryCache { return try initWithConfig(alloc, root_dir, .{}); @@ -251,6 +311,9 @@ pub const QueryCache = struct { } pub fn deinit(self: *QueryCache) void { + if (self.persistence) |worker| worker.deinit(); + self.graph_metric_routing.deinit(); + self.graph_metric_blocks.deinit(); var io_impl = threadedIo(); defer io_impl.deinit(); const coordination_locked = blk: { @@ -276,7 +339,49 @@ pub const QueryCache = struct { pub fn statsSnapshot(self: *QueryCache) QueryCacheStats { lockAtomic(&self.stats_mu); defer self.stats_mu.unlock(); - return self.stats; + var stats = self.stats; + const routing = self.graph_metric_routing.snapshot(); + stats.decoded_graph_metric_routing_hits = routing.hits; + stats.decoded_graph_metric_routing_misses = routing.misses; + stats.decoded_graph_metric_routing_bytes = routing.bytes; + const blocks = self.graph_metric_blocks.snapshot(); + stats.shared_graph_metric_block_bytes = blocks.bytes; + stats.shared_graph_metric_block_entries = blocks.entries; + stats.shared_graph_metric_block_waiters = blocks.waiters; + lockAtomic(&self.persistence_mu); + defer self.persistence_mu.unlock(); + if (self.persistence) |worker| { + const io = worker.io_impl.io(); + worker.mu.lockUncancelable(io); + defer worker.mu.unlock(io); + stats.graph_metric_persistence_bytes = worker.bytes; + stats.graph_metric_persistence_jobs = worker.outstanding; + stats.graph_metric_persistence_failures += worker.failures; + stats.graph_metric_persistence_bypasses += worker.bypasses; + } + return stats; + } + + pub fn retainAuthenticatedBlocks(self: *QueryCache, artifact_id: []const u8, byte_len: u64, checksum: []const u8, blocks: []const AuthenticatedBlockPublication) void { + const accepted = enqueue: { + lockAtomic(&self.persistence_mu); + defer self.persistence_mu.unlock(); + if (self.persistence == null) self.persistence = block_persistence.Worker.create(self) catch break :enqueue false; + _ = self.persistence.?.enqueue(artifact_id, byte_len, checksum, blocks) catch break :enqueue false; + break :enqueue true; + }; + if (!accepted) { + lockAtomic(&self.stats_mu); + self.stats.graph_metric_persistence_bypasses += 1; + self.stats_mu.unlock(); + } + } + + pub fn drainGraphMetricPersistence(self: *QueryCache) void { + lockAtomic(&self.persistence_mu); + const worker = self.persistence; + self.persistence_mu.unlock(); + if (worker) |value| value.drain(); } pub fn getOrFetchAlloc(self: *QueryCache, artifacts: *artifacts_mod.ArtifactStore, artifact_id: []const u8) ![]u8 { @@ -463,8 +568,125 @@ pub const QueryCache = struct { offset: u64, len: usize, cancellation: CancellationToken, + ) ![]u8 { + return try self.getRangeOrFetchImplAlloc( + result_alloc, + artifacts, + artifact_id, + null, + offset, + len, + cancellation, + ); + } + + pub fn getVerifiedRangeOrFetchAllocWithCancellationUsingAllocator( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + return try self.getRangeOrFetchImplAlloc( + result_alloc, + artifacts, + artifact_id, + .{ .byte_len = expected_byte_len, .checksum = expected_checksum }, + offset, + len, + cancellation, + ); + } + + /// Reads a bounded range whose independently authenticated subranges are + /// rooted in trusted manifest metadata. Cache hits are re-authenticated, + /// and misses are authenticated before publication so a transient bad + /// provider response cannot become a durable cache poison. + pub fn getAuthenticatedRangeOrFetchAllocWithCancellationUsingAllocator( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + subranges: []const AuthenticatedSubrange, + cancellation: CancellationToken, ) ![]u8 { try cancellation.check(); + try validateExpectedRange(artifact_id, .{ + .byte_len = expected_byte_len, + .checksum = expected_checksum, + }, offset, len); + try validateAuthenticatedSubrangeLayout(len, subranges); + + const range_path = try rangeCachePathAlloc(self.alloc, self.root_dir, artifact_id, offset, len); + defer self.alloc.free(range_path); + const cached = readVerifiedCacheRecordAllocWithCancellation(result_alloc, range_path, len, cancellation) catch |err| switch (err) { + error.FileNotFound => null, + error.CacheEntryCorrupt => blk: { + try removeCorruptCacheEntry(self, range_path, cancellation); + break :blk null; + }, + else => return err, + }; + if (cached) |value| { + if (authenticateSubranges(value, subranges, cancellation)) |_| { + touchFileNow(range_path) catch {}; + recordRangeHit(self); + return value; + } else |err| { + result_alloc.free(value); + switch (err) { + error.ArtifactIntegrityMismatch => try removeCorruptCacheEntry(self, range_path, cancellation), + else => return err, + } + } + } + + const contents = try artifacts.getRangeAllocWithCancellationUsingAllocator( + result_alloc, + artifact_id, + offset, + len, + cancellation, + ); + errdefer result_alloc.free(contents); + if (contents.len != len) { + recordIntegrityFailure(self); + return error.ArtifactIntegrityMismatch; + } + authenticateSubranges(contents, subranges, cancellation) catch |err| { + if (err == error.ArtifactIntegrityMismatch) recordIntegrityFailure(self); + return err; + }; + const published = try publishCacheEntry(self, range_path, contents, .range, cancellation); + recordRangeMiss(self, published); + return contents; + } + + const ExpectedArtifact = struct { + byte_len: u64, + checksum: []const u8, + }; + + fn getRangeOrFetchImplAlloc( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + expected: ?ExpectedArtifact, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + try cancellation.check(); + if (expected) |value| try validateExpectedRange(artifact_id, value, offset, len); const range_path = try rangeCachePathAlloc(self.alloc, self.root_dir, artifact_id, offset, len); defer self.alloc.free(range_path); const cached = readVerifiedCacheRecordAllocWithCancellation(result_alloc, range_path, len, cancellation) catch |err| switch (err) { @@ -482,7 +704,18 @@ pub const QueryCache = struct { return value; } - const contents = try artifacts.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); + const contents = if (expected) |value| + try artifacts.getVerifiedRangeAllocWithCancellationUsingAllocator( + result_alloc, + artifact_id, + value.byte_len, + value.checksum, + offset, + len, + cancellation, + ) + else + try artifacts.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); errdefer result_alloc.free(contents); if (contents.len != len) return error.ArtifactIntegrityMismatch; const published = try publishCacheEntry(self, range_path, contents, .range, cancellation); @@ -522,8 +755,237 @@ pub const QueryCache = struct { offset: u64, len: usize, cancellation: CancellationToken, + ) ![]u8 { + return try self.getBlockOrFetchRangeImplAlloc( + result_alloc, + artifacts, + artifact_id, + block_id, + null, + offset, + len, + cancellation, + ); + } + + pub fn getVerifiedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + block_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + return try self.getBlockOrFetchRangeImplAlloc( + result_alloc, + artifacts, + artifact_id, + block_id, + .{ .byte_len = expected_byte_len, .checksum = expected_checksum }, + offset, + len, + cancellation, + ); + } + + /// Fetches one immutable logical block and authenticates it with the digest + /// published in the artifact's routing metadata. Unlike a generic range + /// cache entry, the block id is stable across candidate sets, so sparse + /// graph-metric reads do not fragment the cache or require large fixed + /// windows merely to obtain a reusable identity. + pub fn getAuthenticatedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + block_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + block_checksum: *const [std.crypto.hash.sha2.Sha256.digest_length]u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) ![]u8 { + if (try self.readAuthenticatedBlockIfPresentAlloc(result_alloc, artifact_id, block_id, expected_byte_len, expected_checksum, block_checksum, offset, len, cancellation)) |cached| return cached; + const contents = try artifacts.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); + errdefer result_alloc.free(contents); + try self.publishAuthenticatedBlock(artifact_id, block_id, expected_byte_len, expected_checksum, block_checksum, offset, len, contents, cancellation); + return contents; + } + + pub fn readAuthenticatedBlockIfPresentAlloc( + self: *QueryCache, + result_alloc: Allocator, + artifact_id: []const u8, + block_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + block_checksum: *const [std.crypto.hash.sha2.Sha256.digest_length]u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) !?[]u8 { + var lease = (try self.readAuthenticatedBlockIfPresentLease(result_alloc, artifact_id, block_id, expected_byte_len, expected_checksum, block_checksum, offset, len, cancellation)) orelse return null; + if (lease == .owned) return lease.owned.data; + defer lease.deinit(); + return try result_alloc.dupe(u8, lease.bytes()); + } + + pub fn readAuthenticatedBlockIfPresentLease( + self: *QueryCache, + result_alloc: Allocator, + artifact_id: []const u8, + block_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + block_checksum: *const [std.crypto.hash.sha2.Sha256.digest_length]u8, + offset: u64, + len: usize, + cancellation: CancellationToken, + ) !?AuthenticatedBlockLease { + try cancellation.check(); + try validateExpectedRange(artifact_id, .{ + .byte_len = expected_byte_len, + .checksum = expected_checksum, + }, offset, len); + if (len == 0) return error.InvalidRange; + + const block_class = classifyBlockId(block_id); + const payload_block_class = classifyPayloadBlockId(block_id); + const key = @import("authenticated_block_fills.zig").blockKey(artifact_id, expected_checksum, offset, len, block_checksum); + if (self.graph_metric_blocks.leaseIfReady(key)) |value| { + var lease = value; + errdefer lease.deinit(); + try cancellation.check(); + recordBlockHit(self, block_class, payload_block_class); + return .{ .cached = lease }; + } + const block_path = try blockCachePathAlloc(self.alloc, self.root_dir, artifact_id, block_id, offset, len, block_class); + defer self.alloc.free(block_path); + const cached = readVerifiedCacheRecordAllocWithCancellation(result_alloc, block_path, len, cancellation) catch |err| switch (err) { + error.FileNotFound => null, + error.CacheEntryCorrupt => blk: { + try removeCorruptCacheEntry(self, block_path, cancellation); + break :blk null; + }, + else => return err, + }; + if (cached) |value| { + var actual: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(value, &actual, .{}); + if (std.mem.eql(u8, &actual, block_checksum)) { + self.graph_metric_blocks.retainVerified(self.alloc, key, value); + touchFileNow(block_path) catch {}; + recordBlockHit(self, block_class, payload_block_class); + return .{ .owned = .{ .alloc = result_alloc, .data = value } }; + } + result_alloc.free(value); + try removeCorruptCacheEntry(self, block_path, cancellation); + } + + return null; + } + + /// Publish a canonical logical block after a coalesced transport read. + /// Authentication remains mandatory; transport shape is never cache identity. + pub fn publishAuthenticatedBlock( + self: *QueryCache, + artifact_id: []const u8, + block_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + block_checksum: *const [std.crypto.hash.sha2.Sha256.digest_length]u8, + offset: u64, + len: usize, + contents: []const u8, + cancellation: CancellationToken, + ) !void { + try cancellation.check(); + try validateExpectedRange(artifact_id, .{ .byte_len = expected_byte_len, .checksum = expected_checksum }, offset, len); + if (len == 0 or contents.len != len) { + recordIntegrityFailure(self); + return error.ArtifactIntegrityMismatch; + } + var actual: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(contents, &actual, .{}); + if (!std.mem.eql(u8, &actual, block_checksum)) { + recordIntegrityFailure(self); + return error.ArtifactIntegrityMismatch; + } + const block_class = classifyBlockId(block_id); + const payload_block_class = classifyPayloadBlockId(block_id); + const path = try blockCachePathAlloc(self.alloc, self.root_dir, artifact_id, block_id, offset, len, block_class); + defer self.alloc.free(path); + const published = try publishCacheEntry(self, path, contents, switch (block_class) { + .routing => .routing_block, + .payload => .payload_block, + }, cancellation); + recordBlockMiss(self, block_class, payload_block_class, published); + } + + /// Authenticate the complete batch before reserving capacity once. Logical + /// blocks keep independent identities and durable publication leases. + pub fn publishAuthenticatedBlocks( + self: *QueryCache, + artifact_id: []const u8, + expected_byte_len: u64, + expected_checksum: []const u8, + blocks: []const AuthenticatedBlockPublication, + cancellation: CancellationToken, + ) !void { + try cancellation.check(); + if (blocks.len > max_authenticated_publication_blocks) return error.InvalidCacheBatch; + if (blocks.len == 0) return; + var entries: [max_authenticated_publication_blocks]BatchCacheEntry = @splat(.{}); + defer for (entries[0..blocks.len]) |entry| if (entry.path) |path| self.alloc.free(path); + const lane: CacheWriteLane = switch (classifyBlockId(blocks[0].block_id)) { + .routing => .routing_block, + .payload => .payload_block, + }; + var payload_bytes: usize = 0; + for (blocks, entries[0..blocks.len], 0..) |block, *entry, i| { + try validateExpectedRange(artifact_id, .{ .byte_len = expected_byte_len, .checksum = expected_checksum }, block.offset, block.contents.len); + if (block.contents.len == 0) return error.InvalidCacheBatch; + payload_bytes = std.math.add(usize, payload_bytes, block.contents.len) catch return error.InvalidCacheBatch; + if (payload_bytes > 8 * 1024 * 1024) return error.InvalidCacheBatch; + const block_class = classifyBlockId(block.block_id); + if ((lane == .routing_block) != (block_class == .routing)) return error.InvalidCacheBatch; + var actual: [cache_record_digest_len]u8 = undefined; + try sha256DigestWithCancellation(block.contents, &actual, cancellation); + if (!std.mem.eql(u8, &actual, &block.checksum)) { + recordIntegrityFailure(self); + return error.ArtifactIntegrityMismatch; + } + entry.path = try blockCachePathAlloc(self.alloc, self.root_dir, artifact_id, block.block_id, block.offset, block.contents.len, block_class); + for (entries[0..i]) |prior| { + if (std.mem.eql(u8, prior.path.?, entry.path.?)) return error.InvalidCacheBatch; + } + entry.contents = block.contents; + entry.incoming_bytes = try storedCacheEntryBytes(block.contents.len, lane); + } + defer for (blocks, entries[0..blocks.len]) |block, entry| { + if (entry.finished) recordBlockMiss(self, classifyBlockId(block.block_id), classifyPayloadBlockId(block.block_id), entry.published); + }; + try publishCacheEntries(self, entries[0..blocks.len], lane, cancellation); + } + + fn getBlockOrFetchRangeImplAlloc( + self: *QueryCache, + result_alloc: Allocator, + artifacts: *artifacts_mod.ArtifactStore, + artifact_id: []const u8, + block_id: []const u8, + expected: ?ExpectedArtifact, + offset: u64, + len: usize, + cancellation: CancellationToken, ) ![]u8 { try cancellation.check(); + if (expected) |value| try validateExpectedRange(artifact_id, value, offset, len); const block_class = classifyBlockId(block_id); const payload_block_class = classifyPayloadBlockId(block_id); const block_path = try blockCachePathAlloc(self.alloc, self.root_dir, artifact_id, block_id, offset, len, block_class); @@ -543,7 +1005,18 @@ pub const QueryCache = struct { return value; } - const contents = try artifacts.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); + const contents = if (expected) |value| + try artifacts.getVerifiedRangeAllocWithCancellationUsingAllocator( + result_alloc, + artifact_id, + value.byte_len, + value.checksum, + offset, + len, + cancellation, + ) + else + try artifacts.getRangeAllocWithCancellationUsingAllocator(result_alloc, artifact_id, offset, len, cancellation); errdefer result_alloc.free(contents); if (contents.len != len) return error.ArtifactIntegrityMismatch; const published = try publishCacheEntry(self, block_path, contents, switch (block_class) { @@ -555,6 +1028,42 @@ pub const QueryCache = struct { } }; +fn validateExpectedRange(artifact_id: []const u8, expected: QueryCache.ExpectedArtifact, offset: u64, len: usize) !void { + artifacts_mod.validateSha256ArtifactIdentity(artifact_id, expected.checksum) catch return error.ArtifactIntegrityMismatch; + const end = std.math.add(u64, offset, std.math.cast(u64, len) orelse return error.InvalidRange) catch return error.InvalidRange; + if (end > expected.byte_len) return error.InvalidRange; +} + +fn validateAuthenticatedSubrangeLayout(total_len: usize, subranges: []const AuthenticatedSubrange) !void { + if (total_len == 0 or subranges.len == 0) return error.InvalidRange; + var covered: usize = 0; + for (subranges) |subrange| { + if (subrange.len == 0 or subrange.relative_offset != covered) return error.InvalidRange; + covered = std.math.add(usize, covered, subrange.len) catch return error.InvalidRange; + if (covered > total_len) return error.InvalidRange; + } + if (covered != total_len) return error.InvalidRange; +} + +fn authenticateSubranges( + contents: []const u8, + subranges: []const AuthenticatedSubrange, + cancellation: CancellationToken, +) !void { + try validateAuthenticatedSubrangeLayout(contents.len, subranges); + for (subranges, 0..) |subrange, subrange_index| { + if (subrange_index % 64 == 0) try cancellation.check(); + var actual: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + try sha256DigestWithCancellation( + contents[subrange.relative_offset..][0..subrange.len], + &actual, + cancellation, + ); + if (!std.mem.eql(u8, &actual, &subrange.checksum)) return error.ArtifactIntegrityMismatch; + } + try cancellation.check(); +} + fn threadedIo() std.Io.Threaded { return std.Io.Threaded.init(std.heap.page_allocator, .{}); } @@ -939,9 +1448,38 @@ fn publishCacheEntry( recordBypass(self); return false; } - var publication_lease: ?CachePublicationLease = null; - defer if (publication_lease) |*lease| lease.release(self.alloc); - var tmp_path: []u8 = undefined; + var entries = [_]BatchCacheEntry{.{ .path = path, .contents = contents, .incoming_bytes = incoming_bytes }}; + try publishCacheEntries(self, &entries, lane, cancellation); + return entries[0].published; +} + +const BatchCacheEntry = struct { + path: ?[]const u8 = null, + contents: []const u8 = &.{}, + incoming_bytes: u64 = 0, + lease: ?CachePublicationLease = null, + tmp_path: ?[]u8 = null, + reservation_active: bool = false, + finished: bool = false, + published: bool = false, + owns_publication_slot: bool = false, +}; + +fn tryAcquirePublicationSlot(self: *QueryCache) bool { + var current = self.publication_slots.load(.monotonic); + while (current < max_authenticated_publication_blocks) { + current = self.publication_slots.cmpxchgWeak(current, current + 1, .monotonic, .monotonic) orelse return true; + } + return false; +} + +fn publishCacheEntries(self: *QueryCache, entries: []BatchCacheEntry, lane: CacheWriteLane, cancellation: CancellationToken) !void { + defer for (entries) |*entry| { + if (entry.reservation_active) discardCacheReservationBestEffort(self, entry.tmp_path.?, entry.incoming_bytes, lane); + if (entry.lease) |*lease| lease.release(self.alloc); + if (entry.owns_publication_slot) _ = self.publication_slots.fetchSub(1, .monotonic); + if (entry.tmp_path) |path| self.alloc.free(path); + }; { try lockAtomicWithCancellation(&self.maintenance_mu, cancellation); defer self.maintenance_mu.unlock(); @@ -950,40 +1488,65 @@ fn publishCacheEntry( const io = coordination_io.io(); try lockFileExclusiveWithCancellation(self.coordination_file, io, cancellation); defer self.coordination_file.unlock(io); - const now_ns = platform_time.monotonicNs(); - const periodic_reconcile_due = now_ns >= self.next_abandoned_write_sweep_ns; - try synchronizeCacheUsageUnderCoordinationLock(self, io, cancellation, periodic_reconcile_due); - if (periodic_reconcile_due) { - self.next_abandoned_write_sweep_ns = now_ns +| abandoned_cache_write_sweep_interval_ns; - } - if (fileExists(path)) return false; + const periodic = now_ns >= self.next_abandoned_write_sweep_ns; + try synchronizeCacheUsageUnderCoordinationLock(self, io, cancellation, periodic); + if (periodic) self.next_abandoned_write_sweep_ns = now_ns +| abandoned_cache_write_sweep_interval_ns; try reapAbandonedCacheWritesUnderCoordinationLock(self, io, cancellation); - // The caller already owns the fetched bytes, so waiting for another - // publisher would only add tail latency. Treat its durable reservation - // as a per-key publication lease and let this caller return its bytes - // without consuming budget or evicting unrelated entries. - publication_lease = (try tryAcquireCachePublicationLease(self.alloc, io, path)) orelse return false; - - // Publish the mutation token before eviction or reservation creation. - // A process crash at any later instruction forces every overlapping - // writer to reconcile the filesystem before trusting local usage. + var incoming: u64 = 0; + for (entries) |*entry| { + try cancellation.check(); + if (fileExists(entry.path.?)) { + entry.finished = true; + continue; + } + const combined = std.math.add(u64, incoming, entry.incoming_bytes) catch return error.CacheEntryTooLarge; + // Small caches retain a fitting subset instead of bypassing the + // entire response or thrashing already published batch members. + if (!entryFitsEmptyCache(self.cfg, combined, lane)) { + recordBypass(self); + entry.finished = true; + continue; + } + if (!tryAcquirePublicationSlot(self)) { + recordBypass(self); + entry.finished = true; + continue; + } + entry.owns_publication_slot = true; + entry.lease = try tryAcquireCachePublicationLease(self.alloc, io, entry.path.?); + if (entry.lease == null) { + entry.finished = true; + continue; + } + incoming = combined; + } + if (incoming == 0) return; + // Advance before the first eviction/reservation mutation so a crash + // forces overlapping writers to reconcile their usage snapshots. self.observed_coordination_generation = try advanceCacheCoordinationGeneration(self.coordination_file, io); - if (!try ensureCapacityForWrite(self, incoming_bytes, lane, cancellation)) { - recordBypass(self); - publication_lease.?.releaseAndDeleteUnderCoordinationLock(self.alloc, io); - publication_lease = null; - return false; + if (!try ensureCapacityForWrite(self, incoming, lane, cancellation)) { + for (entries) |*entry| if (entry.lease) |*lease| { + recordBypass(self); + lease.releaseAndDeleteUnderCoordinationLock(self.alloc, io); + entry.lease = null; + entry.finished = true; + }; + return; + } + defer recordUsage(self); + for (entries) |*entry| { + if (entry.lease == null) continue; + try cancellation.check(); + entry.tmp_path = try reserveTempFile(self.alloc, entry.path.?, self.instance_id, entry.incoming_bytes); + addUsage(&self.usage, entry.incoming_bytes, lane); + entry.reservation_active = true; } - tmp_path = try reserveTempFile(self.alloc, path, self.instance_id, incoming_bytes); - addUsage(&self.usage, incoming_bytes, lane); - recordUsage(self); } - defer self.alloc.free(tmp_path); - var reservation_active = true; - defer if (reservation_active) discardCacheReservationBestEffort(self, tmp_path, incoming_bytes, lane); - - try writeReservedTempFileWithCancellation(tmp_path, contents, lane, cancellation); + // File contents are written outside both global locks. + for (entries) |entry| { + if (entry.reservation_active) try writeReservedTempFileWithCancellation(entry.tmp_path.?, entry.contents, lane, cancellation); + } try lockAtomicWithCancellation(&self.maintenance_mu, cancellation); defer self.maintenance_mu.unlock(); var coordination_io = threadedIo(); @@ -993,25 +1556,25 @@ fn publishCacheEntry( defer self.coordination_file.unlock(io); try synchronizeCacheUsageUnderCoordinationLock(self, io, cancellation, false); self.observed_coordination_generation = try advanceCacheCoordinationGeneration(self.coordination_file, io); - if (fileExists(path)) { - deleteFilePath(io, tmp_path) catch |err| switch (err) { - error.FileNotFound => {}, - else => return err, - }; - subtractUsage(&self.usage, incoming_bytes, lane); - recordUsage(self); - reservation_active = false; - publication_lease.?.releaseAndDeleteUnderCoordinationLock(self.alloc, io); - publication_lease = null; - return false; + defer recordUsage(self); + for (entries) |*entry| { + if (!entry.reservation_active) continue; + try cancellation.check(); + if (fileExists(entry.path.?)) { + deleteFilePath(io, entry.tmp_path.?) catch |err| switch (err) { + error.FileNotFound => {}, + else => return err, + }; + subtractUsage(&self.usage, entry.incoming_bytes, lane); + } else { + try renameFilePath(io, entry.tmp_path.?, entry.path.?); + entry.published = true; + } + entry.reservation_active = false; + entry.finished = true; + entry.lease.?.releaseAndDeleteUnderCoordinationLock(self.alloc, io); + entry.lease = null; } - try cancellation.check(); - try renameFilePath(io, tmp_path, path); - recordUsage(self); - reservation_active = false; - publication_lease.?.releaseAndDeleteUnderCoordinationLock(self.alloc, io); - publication_lease = null; - return true; } fn tryAcquireCachePublicationLease(alloc: Allocator, io: std.Io, path: []const u8) !?CachePublicationLease { @@ -1737,6 +2300,12 @@ fn recordBypass(self: *QueryCache) void { self.stats.bypasses += 1; } +fn recordIntegrityFailure(self: *QueryCache) void { + lockAtomic(&self.stats_mu); + defer self.stats_mu.unlock(); + self.stats.integrity_failures +|= 1; +} + fn removeCorruptCacheEntry(self: *QueryCache, path: []const u8, cancellation: CancellationToken) !void { try lockAtomicWithCancellation(&self.maintenance_mu, cancellation); defer self.maintenance_mu.unlock(); @@ -2044,6 +2613,164 @@ test "serverless query cache cancels bounded positional reads without recording try std.testing.expectEqual(@as(u64, 0), stats.range_hits); } +test "serverless query cache authenticates ranges before publication and self heals poisoned hits" { + const alloc = std.testing.allocator; + var cache_root_buf: [256]u8 = undefined; + const cache_root = tmpPath(&cache_root_buf, "cache-authenticated-ranges"); + defer cleanupTmp(cache_root); + + const payload = "abcdefgh"; + var payload_digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &payload_digest, .{}); + const checksum = std.fmt.bytesToHex(payload_digest, .lower); + var artifact_id: [artifacts_mod.store.sha256_artifact_id_prefix.len + checksum.len]u8 = undefined; + @memcpy(artifact_id[0..artifacts_mod.store.sha256_artifact_id_prefix.len], artifacts_mod.store.sha256_artifact_id_prefix); + @memcpy(artifact_id[artifacts_mod.store.sha256_artifact_id_prefix.len..], &checksum); + + const State = struct { + payload: []const u8, + corrupt_next: bool = false, + range_calls: usize = 0, + + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedPut; + } + fn getAlloc(_: *anyopaque, _: Allocator, _: []const u8) ![]u8 { + return error.UnexpectedFullRead; + } + fn getRangeAlloc(ptr: *anyopaque, result_alloc: Allocator, _: []const u8, offset: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.range_calls += 1; + const start = std.math.cast(usize, offset) orelse return error.InvalidRange; + if (start > self.payload.len or len > self.payload.len - start) return error.InvalidRange; + const result = try result_alloc.dupe(u8, self.payload[start..][0..len]); + if (self.corrupt_next and result.len > 0) { + result[0] ^= 0x01; + self.corrupt_next = false; + } + return result; + } + fn stat(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedStat; + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.UnexpectedDelete; + } + + const vtable = artifacts_mod.ArtifactStore.VTable{ + .deinit = deinit, + .put = put, + .get_alloc = getAlloc, + .get_range_alloc = getRangeAlloc, + .stat = stat, + .delete = delete, + }; + }; + var state = State{ .payload = payload }; + var artifacts = artifacts_mod.ArtifactStore{ .allocator = alloc, .ptr = &state, .vtable = &State.vtable }; + defer artifacts.deinit(); + var cache = try QueryCache.init(alloc, std.mem.span(cache_root)); + defer cache.deinit(); + + var first_digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload[0..4], &first_digest, .{}); + const first_subranges = [_]AuthenticatedSubrange{.{ .relative_offset = 0, .len = 4, .checksum = first_digest }}; + state.corrupt_next = true; + try std.testing.expectError(error.ArtifactIntegrityMismatch, cache.getAuthenticatedRangeOrFetchAllocWithCancellationUsingAllocator( + alloc, + &artifacts, + &artifact_id, + payload.len, + &checksum, + 0, + 4, + &first_subranges, + .none, + )); + const first_path = try rangeCachePathAlloc(alloc, std.mem.span(cache_root), &artifact_id, 0, 4); + defer alloc.free(first_path); + try std.testing.expect(!fileExists(first_path)); + try std.testing.expectEqual(@as(u64, 1), cache.statsSnapshot().integrity_failures); + + const first = try cache.getAuthenticatedRangeOrFetchAllocWithCancellationUsingAllocator( + alloc, + &artifacts, + &artifact_id, + payload.len, + &checksum, + 0, + 4, + &first_subranges, + .none, + ); + defer alloc.free(first); + try std.testing.expectEqualStrings("abcd", first); + try std.testing.expect(fileExists(first_path)); + + state.corrupt_next = true; + const poisoned = try cache.getRangeOrFetchAlloc(&artifacts, &artifact_id, 4, 4); + alloc.free(poisoned); + var second_digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload[4..8], &second_digest, .{}); + const second_subranges = [_]AuthenticatedSubrange{.{ .relative_offset = 0, .len = 4, .checksum = second_digest }}; + const healed = try cache.getAuthenticatedRangeOrFetchAllocWithCancellationUsingAllocator( + alloc, + &artifacts, + &artifact_id, + payload.len, + &checksum, + 4, + 4, + &second_subranges, + .none, + ); + defer alloc.free(healed); + try std.testing.expectEqualStrings("efgh", healed); + try std.testing.expectEqual(@as(usize, 4), state.range_calls); + try std.testing.expectEqual(@as(u64, 2), cache.statsSnapshot().integrity_failures); + + // Logical block identities are independently authenticated. A generic + // cache fill cannot poison one, and a healed block is reusable without + // another object-store range request. + state.corrupt_next = true; + const poisoned_block = try cache.getBlockOrFetchRangeAlloc(&artifacts, &artifact_id, "graph-metric-score-0-exact", 0, 4); + alloc.free(poisoned_block); + const healed_block = try cache.getAuthenticatedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + alloc, + &artifacts, + &artifact_id, + "graph-metric-score-0-exact", + payload.len, + &checksum, + &first_digest, + 0, + 4, + .none, + ); + defer alloc.free(healed_block); + try std.testing.expectEqualStrings("abcd", healed_block); + try std.testing.expectEqual(@as(usize, 6), state.range_calls); + try std.testing.expectEqual(@as(u64, 3), cache.statsSnapshot().integrity_failures); + + state.corrupt_next = true; + const cached_block = try cache.getAuthenticatedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + alloc, + &artifacts, + &artifact_id, + "graph-metric-score-0-exact", + payload.len, + &checksum, + &first_digest, + 0, + 4, + .none, + ); + defer alloc.free(cached_block); + try std.testing.expectEqualStrings("abcd", cached_block); + try std.testing.expectEqual(@as(usize, 6), state.range_calls); +} + test "serverless query cache rejects unsafe artifact ids before filesystem access" { const alloc = std.testing.allocator; var artifact_root_buf: [256]u8 = undefined; @@ -2095,6 +2822,202 @@ test "serverless query cache supports relative cache directories" { try std.testing.expectEqual(@as(u64, payload.len), cache.statsSnapshot().current_bytes); } +test "serverless query cache persistence bounds outstanding jobs and owns request bytes" { + const alloc = std.testing.allocator; + var root_buf: [256]u8 = undefined; + const root = tmpPath(&root_buf, "cache-authenticated-async"); + defer cleanupTmp(root); + var cache = try QueryCache.init(alloc, std.mem.span(root)); + defer cache.deinit(); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("data", &digest, .{}); + var bytes = "data".*; + const block = AuthenticatedBlockPublication{ .block_id = "graph-metric-score-0-exact", .offset = 0, .contents = &bytes, .checksum = digest }; + // Simulate slow disk coordination. Enqueueing all 32 jobs must complete + // while publication is blocked, and the next job must bypass retention. + lockAtomic(&cache.maintenance_mu); + { + defer cache.maintenance_mu.unlock(); + for (0..block_persistence.max_jobs + 1) |_| cache.retainAuthenticatedBlocks(artifact_id, 4, checksum, &.{block}); + const stats = cache.statsSnapshot(); + try std.testing.expectEqual(@as(u64, block_persistence.max_jobs), stats.graph_metric_persistence_jobs); + try std.testing.expectEqual(@as(u64, 1), stats.graph_metric_persistence_bypasses); + try std.testing.expect(stats.graph_metric_persistence_bytes <= block_persistence.max_bytes); + @memset(&bytes, 'x'); + } + cache.drainGraphMetricPersistence(); + const stats = cache.statsSnapshot(); + try std.testing.expectEqual(@as(u64, 0), stats.graph_metric_persistence_jobs); + try std.testing.expectEqual(@as(u64, 0), stats.graph_metric_persistence_bytes); + try std.testing.expectEqual(@as(u64, 0), stats.graph_metric_persistence_failures); + const stored = (try cache.readAuthenticatedBlockIfPresentAlloc(alloc, artifact_id, block.block_id, 4, checksum, &digest, 0, 4, .none)).?; + defer alloc.free(stored); + try std.testing.expectEqualStrings("data", stored); + // A restart-style disk hit must warm the canonical memory tier. Warm + // borrowers neither touch the filesystem nor allocate a payload copy. + try std.testing.expectEqual(@as(usize, 1), cache.graph_metric_blocks.snapshot().entries); + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + var warm = (try cache.readAuthenticatedBlockIfPresentLease(failing.allocator(), artifact_id, block.block_id, 4, checksum, &digest, 0, 4, .none)).?; + defer warm.deinit(); + try std.testing.expect(warm == .cached); + try std.testing.expectEqualStrings("data", warm.bytes()); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); +} + +test "serverless query cache batches authenticated publication with one eviction pass" { + const alloc = std.testing.allocator; + var root_buf: [256]u8 = undefined; + const root = tmpPath(&root_buf, "cache-authenticated-batch"); + defer cleanupTmp(root); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + const capacity = 8 * (4 + cache_record_header_len); + const cfg = QueryCacheConfig{ .max_bytes = capacity, .max_payload_bytes = capacity }; + var cache = try QueryCache.initWithConfig(alloc, std.mem.span(root), cfg); + defer cache.deinit(); + var overlap = try QueryCache.initWithConfig(alloc, std.mem.span(root), cfg); + defer overlap.deinit(); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("data", &digest, .{}); + var blocks: [8]AuthenticatedBlockPublication = undefined; + var ids: [8][64]u8 = undefined; + for (&blocks, &ids, 0..) |*block, *id, i| { + var old_buf: [64]u8 = undefined; + const old_id = try std.fmt.bufPrint(&old_buf, "old-{d}-exact", .{i}); + try cache.publishAuthenticatedBlock(artifact_id, old_id, 32, checksum, &digest, i * 4, 4, "data", .none); + block.* = .{ .block_id = try std.fmt.bufPrint(id, "new-{d}-exact", .{i}), .offset = i * 4, .contents = "data", .checksum = digest }; + } + const before = cache.statsSnapshot(); + cache.publication_slots.store(max_authenticated_publication_blocks, .monotonic); + try cache.publishAuthenticatedBlocks(artifact_id, 32, checksum, &blocks, .none); + try std.testing.expectEqual(before.block_writes, cache.statsSnapshot().block_writes); + try std.testing.expectEqual(before.evictions, cache.statsSnapshot().evictions); + cache.publication_slots.store(0, .monotonic); + try cache.publishAuthenticatedBlocks(artifact_id, 32, checksum, &blocks, .none); + const after = cache.statsSnapshot(); + try std.testing.expectEqual(@as(u64, 1), after.evictions - before.evictions); + try std.testing.expectEqual(@as(u64, 8), after.block_writes - before.block_writes); + try std.testing.expectEqual(@as(u64, capacity), after.current_bytes); + try std.testing.expectEqual(@as(usize, 0), cache.publication_slots.load(.monotonic)); + // A handle opened before publication must reconcile without evicting or + // rewriting existing canonical keys. + try overlap.publishAuthenticatedBlocks(artifact_id, 32, checksum, &blocks, .none); + try std.testing.expectEqual(@as(u64, 0), overlap.statsSnapshot().block_writes); + try std.testing.expectEqual(@as(u64, 0), overlap.statsSnapshot().evictions); + try std.testing.expectEqual(@as(u64, capacity), overlap.statsSnapshot().current_bytes); + for (blocks) |block| { + const bytes = (try overlap.readAuthenticatedBlockIfPresentAlloc(alloc, artifact_id, block.block_id, 32, checksum, &block.checksum, block.offset, block.contents.len, .none)).?; + defer alloc.free(bytes); + try std.testing.expectEqualStrings(block.contents, bytes); + } +} + +test "serverless query cache authenticates whole batches and retains fitting subsets" { + const alloc = std.testing.allocator; + var root_buf: [256]u8 = undefined; + const root = tmpPath(&root_buf, "cache-authenticated-batch-validation"); + defer cleanupTmp(root); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + var cache = try QueryCache.initWithConfig(alloc, std.mem.span(root), .{ .max_bytes = 4 + cache_record_header_len }); + defer cache.deinit(); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("data", &digest, .{}); + var blocks = [_]AuthenticatedBlockPublication{ + .{ .block_id = "first-exact", .offset = 0, .contents = "data", .checksum = digest }, + .{ .block_id = "second-exact", .offset = 4, .contents = "data", .checksum = digest }, + }; + blocks[1].checksum[0] ^= 1; + try std.testing.expectError(error.ArtifactIntegrityMismatch, cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .none)); + try std.testing.expectEqual(@as(u64, 0), cache.statsSnapshot().current_bytes); + try std.testing.expectEqual(@as(u64, 0), cache.statsSnapshot().block_writes); + blocks[1].checksum = digest; + const Cancel = struct { + fn canceled(_: *const anyopaque) bool { + return true; + } + }; + try std.testing.expectError(error.Canceled, cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .{ .ptr = &blocks, .is_cancelled_fn = Cancel.canceled })); + try std.testing.expectError(error.InvalidCacheBatch, cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &.{ blocks[0], blocks[0] }, .none)); + try cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .none); + try std.testing.expectEqual(@as(u64, 1), cache.statsSnapshot().block_writes); + try std.testing.expectEqual(@as(u64, 1), cache.statsSnapshot().bypasses); + try std.testing.expectEqual(@as(u64, 4 + cache_record_header_len), cache.statsSnapshot().current_bytes); + const retained = (try cache.readAuthenticatedBlockIfPresentAlloc(alloc, artifact_id, blocks[0].block_id, 8, checksum, &digest, 0, 4, .none)).?; + defer alloc.free(retained); + try std.testing.expectEqualStrings("data", retained); +} + +test "serverless query cache canceled batch releases partial reservations and publication leases" { + const alloc = std.testing.allocator; + var root_buf: [256]u8 = undefined; + const root = tmpPath(&root_buf, "cache-batch-cancel-reservation"); + defer cleanupTmp(root); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + var cache = try QueryCache.init(alloc, std.mem.span(root)); + defer cache.deinit(); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("data", &digest, .{}); + const blocks = [_]AuthenticatedBlockPublication{ + .{ .block_id = "first-exact", .offset = 0, .contents = "data", .checksum = digest }, + .{ .block_id = "second-exact", .offset = 4, .contents = "data", .checksum = digest }, + }; + const Cancel = struct { + fn afterFirstReservation(ptr: *const anyopaque) bool { + const current: *const QueryCache = @ptrCast(@alignCast(ptr)); + // Only this test thread mutates the cache. + return current.usage.payload_block_count > 0; + } + }; + try std.testing.expectError(error.Canceled, cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .{ .ptr = &cache, .is_cancelled_fn = Cancel.afterFirstReservation })); + try std.testing.expectEqual(@as(u64, 0), cache.statsSnapshot().current_bytes); + try std.testing.expectEqual(@as(u64, 0), cache.statsSnapshot().block_writes); + var reopened = try QueryCache.init(alloc, std.mem.span(root)); + try std.testing.expectEqual(@as(usize, 0), cache.publication_slots.load(.monotonic)); + defer reopened.deinit(); + try std.testing.expectEqual(@as(u64, 0), reopened.statsSnapshot().current_bytes); + try reopened.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .none); + try std.testing.expectEqual(@as(u64, 2), reopened.statsSnapshot().block_writes); + try std.testing.expectEqual(@as(u64, 2 * (4 + cache_record_header_len)), reopened.statsSnapshot().current_bytes); +} + +test "serverless query cache batch publication releases allocations and reservations on failure" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var root_buf: [256]u8 = undefined; + const root = tmpPath(&root_buf, "cache-batch-allocation-failure"); + defer cleanupTmp(root); + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + // Initialization is not the operation under test; replace only + // the allocator used by batch paths, leases, and reservations. + var cache = try QueryCache.init(std.testing.allocator, std.mem.span(root)); + defer cache.deinit(); + const owner_alloc = cache.alloc; + cache.alloc = alloc; + defer cache.alloc = owner_alloc; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("data", &digest, .{}); + const blocks = [_]AuthenticatedBlockPublication{ + .{ .block_id = "first-exact", .offset = 0, .contents = "data", .checksum = digest }, + .{ .block_id = "second-exact", .offset = 4, .contents = "data", .checksum = digest }, + }; + cache.publishAuthenticatedBlocks(artifact_id, 8, checksum, &blocks, .none) catch |err| { + // Reopening reaps abandoned writes even if allocation failed + // partway through reserving or committing the batch. + var reopened = try QueryCache.init(std.testing.allocator, std.mem.span(root)); + defer reopened.deinit(); + try std.testing.expect(reopened.statsSnapshot().current_bytes <= 2 * (4 + cache_record_header_len)); + return err; + }; + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + test "serverless query cache bypasses oversized entries without evicting useful data" { const alloc = std.testing.allocator; var artifact_root_buf: [256]u8 = undefined; diff --git a/zig/pkg/antfly/src/serverless/query/graph_metric_reader.zig b/zig/pkg/antfly/src/serverless/query/graph_metric_reader.zig new file mode 100644 index 0000000000..c499f76545 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/query/graph_metric_reader.zig @@ -0,0 +1,3619 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Query-time access to immutable graph metric vectors. Every read verifies +//! that the metric was derived from the graph artifact pinned by this session, +//! preventing silent cross-generation score mixing. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const metric_segment = @import("../graph_metric_segment/mod.zig"); +const graph_mod = @import("../../graph/graph.zig"); +const graph_metric_config = @import("../build/graph_metric_config.zig"); +const lake_graph_metric = @import("../build/lake_graph_metric.zig"); +const graph_metric_policy = @import("../build/graph_metric_policy.zig"); +const runtime_mod = @import("runtime.zig"); +const operation = @import("../../api/operation.zig"); +const artifacts_mod = @import("../artifacts/mod.zig"); +const manifest_mod = @import("../manifest/mod.zig"); +const routing_cache = @import("graph_metric_routing_cache.zig"); + +pub const Limits = struct { + // Keep this aligned with the public query contract. The current wire stores this entire + // bounded prefix in independently authenticated ranked blocks. + max_top_k: usize = 10_000, + max_point_scores: usize = 100_000, + max_result_bytes: usize = 64 * 1024 * 1024, +}; + +const TopResultBudget = struct { + bytes: usize, + limit: usize, + session: *runtime_mod.QuerySession, + + fn init(session: *runtime_mod.QuerySession, count: usize, limit: usize) !TopResultBudget { + const bytes = std.math.mul(usize, count, @sizeOf(Score)) catch return error.GraphMetricQueryBudgetExceeded; + if (bytes > limit) return error.GraphMetricQueryBudgetExceeded; + try session.chargeGraphMetricRetained(bytes); + return .{ .bytes = bytes, .limit = limit, .session = session }; + } + + fn addNode(self: *TopResultBudget, len: usize) !void { + const next = std.math.add(usize, self.bytes, len) catch return error.GraphMetricQueryBudgetExceeded; + if (next > self.limit) return error.GraphMetricQueryBudgetExceeded; + try self.session.chargeGraphMetricRetained(len); + self.bytes = next; + } +}; + +fn recordRejectionDiagnostic( + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + materializer_fingerprint: u64, +) void { + session.recordGraphMetricRejection( + graph_index_name, + metric_name, + materializer_fingerprint, + ); +} + +pub const Score = struct { + node_id: []u8, + value: f64, + + pub fn deinit(self: *Score, alloc: Allocator) void { + alloc.free(self.node_id); + self.* = undefined; + } +}; + +pub const PublicScore = @import("../../storage/db/types.zig").GraphMetricScore; + +pub const Result = struct { + scores: []Score, + owns_scores: bool = true, + config_fingerprint: u64, + converged: bool, + iterations_completed: u32, + delta: f64, + edge_filter: graph_mod.GraphMetricEdgeFilter, + metadata_version: u16, + published_generation: u64, + edge_generation: u64, + computed_at_ms: u64, + + pub fn deinit(self: *Result, alloc: Allocator) void { + if (self.owns_scores) { + for (self.scores) |*score| score.deinit(alloc); + alloc.free(self.scores); + } + self.edge_filter.deinit(alloc); + self.* = undefined; + } + + /// Reframe the score descriptors, transferring node ownership without + /// copying their potentially large payloads. Failure leaves this intact. + pub fn takePublicScoresAlloc(self: *Result, alloc: Allocator, session: *runtime_mod.QuerySession) ![]PublicScore { + if (!self.owns_scores) return error.GraphMetricScoresAlreadyTaken; + try session.chargeGraphMetricRetained(std.math.mul(usize, self.scores.len, @sizeOf(PublicScore)) catch return error.GraphMetricQueryBudgetExceeded); + const result = try alloc.alloc(PublicScore, self.scores.len); + for (self.scores, result) |score, *out| out.* = .{ .node = score.node_id, .score = score.value }; + alloc.free(self.scores); + self.scores = &.{}; + self.owns_scores = false; + return result; + } +}; + +pub const PointScoresResult = struct { + scores: []?f64, + owns_scores: bool = true, + memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}, + config_fingerprint: u64, + converged: bool, + iterations_completed: u32, + delta: f64, + edge_filter: graph_mod.GraphMetricEdgeFilter, + metadata_version: u16, + published_generation: u64, + edge_generation: u64, + computed_at_ms: u64, + + pub fn deinit(self: *PointScoresResult, alloc: Allocator) void { + if (self.owns_scores) alloc.free(self.scores); + self.memory.deinit(); + self.edge_filter.deinit(alloc); + self.* = undefined; + } + + /// Transfers the column without encoding allocator ownership in a + /// sentinel slice. This remains correct for zero-row queries, where a + /// successful zero-length allocation still belongs to the caller. + pub fn takeScores(self: *PointScoresResult) ![]?f64 { + if (!self.owns_scores) return error.GraphMetricScoresAlreadyTaken; + self.owns_scores = false; + self.memory.detach(); + return self.scores; + } + + pub fn takeColumn(self: *PointScoresResult) !ScoreColumn { + if (!self.owns_scores) return error.GraphMetricScoresAlreadyTaken; + self.owns_scores = false; + const column = ScoreColumn{ .scores = self.scores, .memory = self.memory }; + self.memory = .{}; + return column; + } +}; + +pub const ScoreColumn = struct { + scores: []?f64, + memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.scores); + self.memory.deinit(); + self.* = .{ .scores = &.{} }; + } +}; + +const PointScoresMetadata = struct { + config_fingerprint: u64, + converged: bool, + iterations_completed: u32, + delta: f64, + metadata_version: u16, + published_generation: u64, + edge_generation: u64, + computed_at_ms: u64, +}; + +pub const PointScoreColumnsResult = struct { + columns: []PointScoresResult, + + pub fn deinit(self: *@This(), alloc: Allocator) void { + for (self.columns) |*column| column.deinit(alloc); + alloc.free(self.columns); + self.* = undefined; + } +}; + +const RankedScoreBoundaryValidator = struct { + previous_node_id: [metric_segment.codec.max_score_node_id_bytes]u8 = undefined, + previous_node_id_len: usize = 0, + previous_value: f64 = 0, + has_previous: bool = false, + + fn observeBlock(self: *@This(), decoded: metric_segment.codec.DecodedScoreBlock) !void { + if (decoded.len == 0) return error.InvalidGraphMetricSegment; + const first = decoded.scores[0]; + if (self.has_previous) { + const node_order = first.orderNode(decoded.node_prefix, self.previous_node_id[0..self.previous_node_id_len]); + if (self.previous_value < first.value or + (self.previous_value == first.value and node_order != .gt)) + { + return error.InvalidGraphMetricSegment; + } + } + + const last = decoded.scores[decoded.len - 1]; + if (last.nodeIdLen(decoded.node_prefix) > self.previous_node_id.len) return error.InvalidGraphMetricSegment; + _ = try last.copyNode(decoded.node_prefix, self.previous_node_id[0..]); + self.previous_node_id_len = last.nodeIdLen(decoded.node_prefix); + self.previous_value = last.value; + self.has_previous = true; + } +}; +const ScoreFetchRange = struct { first_block: usize, last_block: usize, offset: u64, len: usize }; + +// Covers an authenticated fill/lease plus its contiguous output (or origin +// temporary), and the block descriptors used while both are live. +fn transportMemoryBytes(bytes: usize, blocks: usize) !usize { + const payload = std.math.mul(usize, bytes, 2) catch return error.GraphMetricQueryBudgetExceeded; + const metadata = std.math.mul(usize, blocks, 256) catch return error.GraphMetricQueryBudgetExceeded; + return std.math.add(usize, payload, metadata) catch return error.GraphMetricQueryBudgetExceeded; +} + +const OwnedMetricRange = struct { + bytes: []u8, + memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}, + + fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.bytes); + self.memory.deinit(); + self.* = undefined; + } +}; +// Default used by isolated planner tests. Production derives this from the +// whole query's remaining budget after preparing every column's routing. +const max_score_range_requests: usize = 60; +const max_top_score_range_requests: usize = 64; +const max_parallel_metric_range_requests: usize = 8; +const max_point_score_columns: usize = 16; +// A column may itself hold a 16 MiB routing footer and a 32 MiB range batch. +// Two columns retain useful overlap without allowing nested fan-out to turn a +// valid request into an avoidable serverless memory spike. +const max_parallel_point_score_columns: usize = 2; +/// Bound aggregate live payload memory as well as request count. Coalescing +/// deliberately makes ranges variable-sized, so a count-only fanout can turn +/// eight harmless-looking reads into a large transient allocation spike. +const max_parallel_metric_range_bytes: usize = 32 * 1024 * 1024; +const coalesced_score_window_bytes: u64 = 8 * 1024 * 1024; +const ranked_fetch_window_bytes: usize = 8 * 1024 * 1024; + +fn metricRangeBatchEnd(ranges: []const ScoreFetchRange, start: usize) usize { + var end = start; + var bytes: usize = 0; + while (end < ranges.len and end - start < max_parallel_metric_range_requests) : (end += 1) { + const next_bytes = std.math.add(usize, bytes, ranges[end].len) catch break; + if (end > start and next_bytes > max_parallel_metric_range_bytes) break; + bytes = next_bytes; + } + return end; +} + +fn admittedMetricRangeBatchEnd(session: *runtime_mod.QuerySession, ranges: []const ScoreFetchRange, start: usize) !usize { + const available = if (session.graph_metric_transport_credit != 0) session.graph_metric_transport_credit else session.graphMetricMemoryAvailable(); + const cap = metricRangeBatchEnd(ranges, start); + var used: usize = 0; + var end = start; + while (end < cap) : (end += 1) { + const range = ranges[end]; + const bytes = try transportMemoryBytes(range.len, range.last_block -| range.first_block + 1); + if (bytes > available -| used) break; + used += bytes; + } + if (end == start) return error.GraphMetricQueryBudgetExceeded; + return end; +} + +fn minimumPlanTransportMemory(plan: PointScorePlan) !usize { + var minimum: usize = 0; + for (plan.ranges) |range| minimum = @max(minimum, try transportMemoryBytes(range.len, range.last_block -| range.first_block + 1)); + return minimum; +} + +fn preferredPlanTransportMemory(plan: PointScorePlan) !usize { + var preferred: usize = 0; + var start: usize = 0; + while (start < plan.ranges.len) { + const end = metricRangeBatchEnd(plan.ranges, start); + var bytes: usize = 0; + for (plan.ranges[start..end]) |range| bytes = std.math.add(usize, bytes, try transportMemoryBytes(range.len, range.last_block -| range.first_block + 1)) catch return error.GraphMetricQueryBudgetExceeded; + preferred = @max(preferred, bytes); + start = end; + } + return preferred; +} + +pub fn scoreAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, graph_index_name: []const u8, metric_name: []const u8, node_id: []const u8) !?Score { + const node_ids = [_][]const u8{node_id}; + var loaded = try scoresAlloc(alloc, session, graph_index_name, metric_name, &node_ids); + defer loaded.deinit(alloc); + const value = loaded.scores[0] orelse return null; + return .{ .node_id = try alloc.dupe(u8, node_id), .value = value }; +} + +fn pointScoresResultAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + scores: []?f64, + metadata: PointScoresMetadata, +) !PointScoresResult { + const specs = try session.graphMetricSpecs(); + const config = findConfig(specs, graph_index_name, metric_name) orelse return error.MetricNotConfigured; + var edge_filter = try config.edge_filter.cloneAlloc(alloc); + errdefer edge_filter.deinit(alloc); + return .{ + .scores = scores, + .config_fingerprint = metadata.config_fingerprint, + .converged = metadata.converged, + .iterations_completed = metadata.iterations_completed, + .delta = metadata.delta, + .edge_filter = edge_filter, + .metadata_version = metadata.metadata_version, + .published_generation = metadata.published_generation, + .edge_generation = metadata.edge_generation, + .computed_at_ms = metadata.computed_at_ms, + }; +} + +const ColumnPass = enum { prepare, execute }; + +fn preparationMemoryEnvelope(ref: manifest_mod.ArtifactRef, candidates: usize) u64 { + // Footer payloads, decoded locators, canonical fill/output overlap, and + // candidate/page descriptors. Deliberately pessimistic only for fanout; + // sparse requests are admitted by their actual reservations. + const routing = @as(u64, ref.graph_metric_routing_footer_len) * 6; + const controls = @as(u64, ref.graph_metric_control_len) * 3; + const rows = std.math.mul(u64, candidates, 512) catch return std.math.maxInt(u64); + return std.math.add(u64, routing + controls + 64 * 1024, rows) catch std.math.maxInt(u64); +} + +fn scoreColumnWorker( + child: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + node_ids: []const []const u8, + candidate_order: []const u32, + scores: []?f64, + plan: *?PointScorePlan, + pass: ColumnPass, + failure: *?anyerror, + cancel_siblings: *std.atomic.Value(bool), +) void { + if (pass == .prepare) { + plan.* = preparePointScoresAlloc(std.heap.smp_allocator, child, graph_index_name, metric_name, node_ids, candidate_order, scores) catch |err| { + failure.* = err; + cancel_siblings.store(true, .release); + return; + }; + } else { + executePointScores(child, &plan.*.?, node_ids, scores) catch |err| { + failure.* = err; + cancel_siblings.store(true, .release); + }; + } +} + +const CombinedColumnCancellation = struct { + parent: operation.CancellationToken, + sibling_failure: *const std.atomic.Value(bool), + + fn token(self: *const @This()) operation.CancellationToken { + return .{ + .ptr = self, + .is_cancelled_fn = isCancelled, + }; + } + + fn isCancelled(ptr: *const anyopaque) bool { + const self: *const @This() = @ptrCast(@alignCast(ptr)); + return self.parent.isCancelled() or self.sibling_failure.load(.acquire); + } +}; + +/// Resolves an entire graph-query metric shape with bounded cross-column +/// parallelism. Each worker owns its transient decode state while all workers +/// share the pinned manifest, cancellation token, cache, and atomic read +/// budget. Returned columns preserve `metric_names` order. +pub fn scoreColumnsAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_names: []const []const u8, + node_ids: []const []const u8, +) !PointScoreColumnsResult { + const result = try scoreColumnsScopedAlloc(alloc, session, graph_index_name, metric_names, node_ids); + for (result.columns) |*column| column.memory.detach(); + return result; +} + +/// Request-scoped columns release admission on destruction or replacement. +/// These columns must be destroyed before the owning query session; callers +/// exporting results beyond that lifetime use scoreColumnsAlloc instead. +pub fn scoreColumnsScopedAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_names: []const []const u8, + node_ids: []const []const u8, +) !PointScoreColumnsResult { + if (metric_names.len > max_point_score_columns or node_ids.len > (Limits{}).max_point_scores) return error.GraphMetricQueryBudgetExceeded; + if (metric_names.len == 0) return .{ .columns = try alloc.alloc(PointScoresResult, 0) }; + var output_memory = try admitPointOutputs(session, node_ids.len, metric_names.len); + defer output_memory.deinit(); + try validatePointNodeIds(session, node_ids); + _ = try session.graphMetricSpecs(); + var scratch = try session.reserveGraphMetricMemory(0); + defer scratch.deinit(); + var scoped = session.forkGraphMetricRead(session.alloc); + defer scoped.deinit(); + scoped.diagnostics = session.diagnostics; + scoped.graph_metric_retained_scope = &scratch; + return scoreColumnsWithScopeAlloc(alloc, &scoped, graph_index_name, metric_names, node_ids, &output_memory); +} + +fn scoreColumnsWithScopeAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_names: []const []const u8, + node_ids: []const []const u8, + output_memory: *runtime_mod.GraphMetricReadBudget.Reservation, +) !PointScoreColumnsResult { + if (metric_names.len > max_point_score_columns or node_ids.len > (Limits{}).max_point_scores) return error.GraphMetricQueryBudgetExceeded; + if (metric_names.len == 0) return .{ .columns = try alloc.alloc(PointScoresResult, 0) }; + var owned_order = try candidateOrderValidatedScopedAlloc(alloc, session, node_ids); + defer owned_order.deinit(alloc); + const candidate_order = owned_order.rows; + const specs = try session.graphMetricSpecs(); + // Plan immutable computations, then fan out logical names/provenance. + // Aliases must not multiply transport admission, routing or block decode. + var physical_names: [max_point_score_columns][]const u8 = undefined; + var physical_refs: [max_point_score_columns]manifest_mod.ArtifactRef = undefined; + var physical_configs: [max_point_score_columns]graph_mod.GraphMetricConfig = undefined; + var logical_refs: [max_point_score_columns]manifest_mod.ArtifactRef = undefined; + var mapping: [max_point_score_columns]usize = undefined; + var owners: [max_point_score_columns]usize = undefined; + var physical_count: usize = 0; + for (metric_names, 0..) |name, i| { + try session.checkCancellation(); + const config = findConfig(specs, graph_index_name, name) orelse return error.MetricNotConfigured; + const encoded_name = try metric_segment.artifactNameAlloc(alloc, graph_index_name, name); + defer alloc.free(encoded_name); + const index = session.findNamedArtifactIndex(.graph_metric_segment, encoded_name) orelse return error.MetricNotReady; + const ref = session.artifactRef(index) orelse return error.InvalidGraphMetricSegment; + logical_refs[i] = ref; + const existing = for (physical_refs[0..physical_count], physical_configs[0..physical_count], 0..) |prior, prior_config, p| { + if (lake_graph_metric.sameComputation(config, prior_config) and + (std.mem.eql(u8, ref.name, prior.name) or @import("../manifest/artifact_ref.zig").areGraphArtifactAliases(ref, prior))) break p; + } else null; + mapping[i] = existing orelse physical_count; + if (existing == null) { + physical_names[physical_count] = name; + physical_refs[physical_count] = ref; + physical_configs[physical_count] = config; + owners[physical_count] = i; + physical_count += 1; + } + } + try session.chargeGraphMetricDecode(0, (metric_names.len - physical_count) * node_ids.len); + const columns = try alloc.alloc(PointScoresResult, metric_names.len); + var initialized_columns: usize = 0; + errdefer { + for (columns[0..initialized_columns]) |*column| column.deinit(alloc); + alloc.free(columns); + } + var plans: [max_point_score_columns]?PointScorePlan = @splat(null); + defer for (plans[0..physical_count]) |*plan| if (plan.*) |*value| value.deinit(); + var buffers: [max_point_score_columns]?[]?f64 = @splat(null); + defer for (buffers[0..physical_count]) |buffer| if (buffer) |value| alloc.free(value); + for (buffers[0..physical_count]) |*buffer| buffer.* = try alloc.alloc(?f64, node_ids.len); + + for ([_]ColumnPass{ .prepare, .execute }) |pass| { + if (pass == .execute) try admitPointPlans(alloc, session, plans[0..physical_count]); + var start: usize = 0; + while (start < physical_count) { + var end = @min(start + max_parallel_point_score_columns, physical_count); + if (pass == .prepare) { + // This bound selects fanout, not query eligibility. Unknown + // sparse/cache shapes fall back to one column, where exact + // live reservations decide admission. Range I/O within a + // column remains parallel. + const available = session.graphMetricMemoryAvailable(); + var peak: u64 = 0; + for (physical_refs[start..end]) |ref| { + peak = std.math.add(u64, peak, preparationMemoryEnvelope(ref, node_ids.len)) catch std.math.maxInt(u64); + } + if (peak > available) end = start + 1; + } + var transport: runtime_mod.GraphMetricReadBudget.Reservation = .{}; + defer transport.deinit(); + var transport_credit: usize = 0; + if (pass == .execute) { + const available = session.graphMetricMemoryAvailable(); + // Reserve the whole execution group before launching children. + // Reduce fanout before rejecting a request which fits serially. + while (true) { + var minimum: usize = 0; + var preferred: usize = 0; + for (plans[start..end]) |plan| { + minimum = @max(minimum, try minimumPlanTransportMemory(plan.?)); + preferred = @max(preferred, try preferredPlanTransportMemory(plan.?)); + } + const share = available / (end - start); + if (minimum <= share) { + transport_credit = @min(share, preferred); + transport = try session.reserveGraphMetricMemory(transport_credit * (end - start)); + break; + } + if (end == start + 1) return error.GraphMetricQueryBudgetExceeded; + end -= 1; + } + } + const count = end - start; + var children: [max_parallel_point_score_columns]runtime_mod.QuerySession = undefined; + var diagnostics: [max_parallel_point_score_columns]operation.RequestDiagnostics = @splat(.{}); + var failures: [max_parallel_point_score_columns]?anyerror = @splat(null); + var sibling_failure = std.atomic.Value(bool).init(false); + var cancellations: [max_parallel_point_score_columns]CombinedColumnCancellation = undefined; + var group: std.Io.Group = .init; + // Every fallible allocation happened before launching these workers. + for (physical_names[start..end], 0..) |metric_name, i| { + children[i] = session.forkGraphMetricRead(std.heap.smp_allocator); + if (pass == .execute) children[i].graph_metric_transport_credit = transport_credit; + cancellations[i] = .{ .parent = session.cancellation, .sibling_failure = &sibling_failure }; + children[i].cancellation = cancellations[i].token(); + if (session.diagnostics != null) children[i].setDiagnostics(&diagnostics[i]); + const args = .{ &children[i], graph_index_name, metric_name, node_ids, candidate_order, buffers[start + i].?, &plans[start + i], pass, &failures[i], &sibling_failure }; + if (session.io) |io| group.async(io, scoreColumnWorker, args) else @call(.auto, scoreColumnWorker, args); + } + const joined = if (session.io) |io| group.await(io) else {}; + for (children[0..count]) |*child| child.deinit(); + try joined; + for (diagnostics[0..count]) |diagnostic| { + const rejection = diagnostic.graph_metric_rejection orelse continue; + session.recordGraphMetricRejection(rejection.graphIndexName(), rejection.metricName(), rejection.materializer_fingerprint); + break; + } + for (failures[0..count]) |failure| if (failure) |err| { + if (err != error.Canceled) return err; + }; + for (failures[0..count]) |failure| if (failure) |err| return err; + start = end; + } + } + for (metric_names, 0..) |name, i| { + const p = mapping[i]; + var metadata = plans[p].?.metadata; + const ref = logical_refs[i]; + metadata.published_generation = if (ref.published_generation != 0) ref.published_generation else session.manifest.version; + metadata.edge_generation = if (ref.edge_generation != 0) ref.edge_generation else session.manifest.version; + metadata.computed_at_ms = if (ref.computed_at_ms != 0) ref.computed_at_ms else @divTrunc(session.manifest.built_at_ns, std.time.ns_per_ms); + const owns_physical = owners[p] == i; + // Logical result copies were admitted before any worker or I/O. + const scores = if (owns_physical) buffers[p].? else try alloc.dupe(?f64, columns[owners[p]].scores); + errdefer if (!owns_physical) alloc.free(scores); + columns[i] = try pointScoresResultAlloc(alloc, session, graph_index_name, name, scores, metadata); + columns[i].memory = output_memory.split(node_ids.len * @sizeOf(?f64) + @sizeOf(PointScoresResult)); + initialized_columns += 1; + if (owns_physical) buffers[p] = null; + } + return .{ .columns = columns }; +} + +/// Resolves exact node scores using authenticated routing and cached blocks, +/// with at most one range fetch per missed block. Positions match `node_ids`. +pub fn scoresAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + node_ids: []const []const u8, +) !PointScoresResult { + if (node_ids.len > (Limits{}).max_point_scores) return error.GraphMetricQueryBudgetExceeded; + var output_memory = try admitPointOutputs(session, node_ids.len, 1); + defer output_memory.deinit(); + try validatePointNodeIds(session, node_ids); + _ = try session.graphMetricSpecs(); + var scratch = try session.reserveGraphMetricMemory(0); + defer scratch.deinit(); + var scoped = session.forkGraphMetricRead(session.alloc); + defer scoped.deinit(); + scoped.diagnostics = session.diagnostics; + scoped.graph_metric_retained_scope = &scratch; + return scoresWithScopeAlloc(alloc, &scoped, graph_index_name, metric_name, node_ids, &output_memory); +} + +fn scoresWithScopeAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + node_ids: []const []const u8, + output_memory: *runtime_mod.GraphMetricReadBudget.Reservation, +) !PointScoresResult { + var owned_order = try candidateOrderValidatedScopedAlloc(alloc, session, node_ids); + defer owned_order.deinit(alloc); + const candidate_order = owned_order.rows; + const values = try alloc.alloc(?f64, node_ids.len); + errdefer alloc.free(values); + var plans = [_]?PointScorePlan{try preparePointScoresAlloc(alloc, session, graph_index_name, metric_name, node_ids, candidate_order, values)}; + defer plans[0].?.deinit(); + try admitPointPlans(alloc, session, &plans); + try executePointScores(session, &plans[0].?, node_ids, values); + const metadata = plans[0].?.metadata; + const result = try pointScoresResultAlloc(alloc, session, graph_index_name, metric_name, values, metadata); + output_memory.detach(); + return result; +} + +fn admitPointOutputs(session: *runtime_mod.QuerySession, count: usize, columns: usize) !runtime_mod.GraphMetricReadBudget.Reservation { + try session.checkCancellation(); + if (count > (Limits{}).max_point_scores or columns > max_point_score_columns) return error.GraphMetricQueryBudgetExceeded; + const items = std.math.mul(usize, count, columns) catch return error.GraphMetricQueryBudgetExceeded; + const payload = std.math.mul(usize, items, @sizeOf(?f64)) catch return error.GraphMetricQueryBudgetExceeded; + const descriptors = std.math.mul(usize, columns, @sizeOf(PointScoresResult)) catch return error.GraphMetricQueryBudgetExceeded; + return session.reserveGraphMetricMemory(std.math.add(usize, payload, descriptors) catch return error.GraphMetricQueryBudgetExceeded); +} + +/// One admitted permutation shared by every metric's routing and score plan. +/// Original row indexes preserve duplicate IDs and caller-visible ordering. +pub fn candidateOrderAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, node_ids: []const []const u8) ![]u32 { + var result = try candidateOrderScopedAlloc(alloc, session, node_ids); + result.memory.detach(); + return result.rows; +} + +const CandidateOrder = struct { + rows: []u32, + memory: runtime_mod.GraphMetricReadBudget.Reservation, + + fn deinit(self: *@This(), alloc: Allocator) void { + alloc.free(self.rows); + self.memory.deinit(); + } +}; + +fn candidateOrderScopedAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, node_ids: []const []const u8) !CandidateOrder { + try validatePointNodeIds(session, node_ids); + return candidateOrderValidatedScopedAlloc(alloc, session, node_ids); +} + +fn validatePointNodeIds(session: *runtime_mod.QuerySession, node_ids: []const []const u8) !void { + try session.checkCancellation(); + if (node_ids.len > (Limits{}).max_point_scores) return error.GraphMetricQueryBudgetExceeded; + for (node_ids, 0..) |id, i| { + if (i % 4096 == 0) try session.checkCancellation(); + if (id.len == 0 or id.len > metric_segment.codec.max_score_node_id_bytes) return error.InvalidGraphMetricNodeId; + } +} + +fn candidateOrderValidatedScopedAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, node_ids: []const []const u8) !CandidateOrder { + const sort_work = std.math.mul(usize, node_ids.len, 2 + std.math.log2_int(usize, @max(node_ids.len, 1))) catch return error.GraphMetricQueryBudgetExceeded; + try session.chargeGraphMetricDecode(0, sort_work); + var memory = try session.reserveGraphMetricMemory(node_ids.len * (@sizeOf(u32) + @sizeOf(u64))); + defer memory.deinit(); + const order = try alloc.alloc(u32, node_ids.len); + errdefer alloc.free(order); + for (order, 0..) |*row, i| row.* = @intCast(i); + // Document IDs commonly share a collection/path prefix. Inspect it once, + // not at every comparison in the candidate sort. + var prefix_len: usize = if (node_ids.len == 0) 0 else node_ids[0].len; + for (node_ids, 0..) |id, i| { + if (i % 4096 == 0) try session.checkCancellation(); + prefix_len = @min(prefix_len, id.len); + var equal: usize = 0; + while (equal < prefix_len and id[equal] == node_ids[0][equal]) : (equal += 1) {} + prefix_len = equal; + if (prefix_len == 0) break; + } + // A transient fixed-width key keeps comparisons on contiguous integers; + // ties fall back to the full suffix, preserving binary IDs and prefixes. + const heads = try alloc.alloc(u64, node_ids.len); + defer alloc.free(heads); + for (node_ids, heads, 0..) |id, *head, i| { + if (i % 4096 == 0) try session.checkCancellation(); + var bytes: [8]u8 = @splat(0); + const n = @min(bytes.len, id.len - prefix_len); + @memcpy(bytes[0..n], id[prefix_len..][0..n]); + head.* = std.mem.readInt(u64, &bytes, .big); + } + const Order = struct { + ids: []const []const u8, + heads: []const u64, + prefix: usize, + fn less(self: @This(), a: u32, b: u32) bool { + if (self.heads[a] != self.heads[b]) return self.heads[a] < self.heads[b]; + return switch (std.mem.order(u8, self.ids[a][self.prefix..], self.ids[b][self.prefix..])) { + .lt => true, + .gt => false, + .eq => a < b, + }; + } + }; + std.mem.sort(u32, order, Order{ .ids = node_ids, .heads = heads, .prefix = prefix_len }, Order.less); + try session.checkCancellation(); + return .{ .rows = order, .memory = memory.split(node_ids.len * @sizeOf(u32)) }; +} + +const TouchedBlock = struct { + block_index: usize, + /// Span in the shared candidate permutation, not a per-column row map. + first_pending: usize, + pending_count: usize, +}; + +const CandidateBlocks = struct { + node_ids: []const []const u8, + order: []const u32, + routing: metric_segment.codec.RoutingIndex, + position: usize = 0, + + fn next(self: *CandidateBlocks) ?TouchedBlock { + while (self.position < self.order.len) { + const start = self.position; + const index = self.routing.findIndex(self.node_ids[self.order[start]]) orelse { + if (self.routing.entries.len == 0) { + self.position = self.order.len; + return null; + } + self.seekBoundary(self.routing.entries[0].first_node_id); + continue; + }; + self.position += 1; + // Jump across a dense span without rescanning every candidate + // for every column (or every level of a paged routing index). + if (index + 1 < self.routing.entries.len) { + self.seekBoundary(self.routing.entries[index + 1].first_node_id); + } else self.position = self.order.len; + return .{ .block_index = index, .first_pending = start, .pending_count = self.position - start }; + } + return null; + } + + fn seekBoundary(self: *CandidateBlocks, boundary: []const u8) void { + var end = self.order.len; + while (self.position < end) { + const mid = self.position + (end - self.position) / 2; + if (std.mem.order(u8, self.node_ids[self.order[mid]], boundary) == .lt) self.position = mid + 1 else end = mid; + } + } +}; + +const PointScorePlan = struct { + alloc: Allocator, + range_alloc: Allocator, + metadata: PointScoresMetadata, + metric_index: usize, + score_count: usize, + score_data_offset: u64, + point_routing: ?PointRouting = null, + candidate_order: []const u32 = &.{}, + touched_blocks: []TouchedBlock = &.{}, + ranges: []ScoreFetchRange = &.{}, + + fn deinit(self: *@This()) void { + if (self.point_routing) |*routing| routing.deinit(); + self.alloc.free(self.touched_blocks); + self.range_alloc.free(self.ranges); + } +}; + +/// Row-mapping microbenchmark; keeps all legacy per-column maps alive to +/// model the two-pass planner. Routing/control and span materialization are +/// deliberately excluded from both paths. +pub fn benchmarkCandidatePlanningAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, node_ids: []const []const u8, routing: metric_segment.codec.RoutingIndex, columns: usize, reference: bool) !u64 { + if (columns > max_point_score_columns) return error.GraphMetricQueryBudgetExceeded; + var sum: u64 = 0; + if (reference) { + const Pending = struct { + block: usize, + row: usize, + fn less(_: void, a: @This(), b: @This()) bool { + return a.block < b.block or (a.block == b.block and a.row < b.row); + } + }; + var maps: [max_point_score_columns]?[]Pending = @splat(null); + defer for (maps) |map| if (map) |rows| alloc.free(rows); + for (maps[0..columns]) |*map| { + const rows = try alloc.alloc(Pending, node_ids.len); + map.* = rows; + for (node_ids, rows, 0..) |id, *row, i| row.* = .{ .block = routing.findIndex(id) orelse return error.InvalidBenchmarkResult, .row = i }; + std.mem.sort(Pending, rows, {}, Pending.less); + for (rows) |row| sum +%= row.row * 31 + row.block; + } + } else { + const order = try candidateOrderAlloc(alloc, session, node_ids); + defer alloc.free(order); + for (0..columns) |_| { + var blocks = CandidateBlocks{ .node_ids = node_ids, .order = order, .routing = routing }; + while (blocks.next()) |block| for (order[block.first_pending..][0..block.pending_count]) |row| { + sum +%= @as(u64, row) * 31 + block.block_index; + }; + } + } + return sum; +} + +fn preparePointScoresAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + node_ids: []const []const u8, + candidate_order: []const u32, + values: []?f64, +) !PointScorePlan { + try session.checkCancellation(); + if (values.len != node_ids.len or candidate_order.len != node_ids.len) return error.InvalidGraphMetricSegment; + if (node_ids.len > (Limits{}).max_point_scores) return error.GraphMetricQueryBudgetExceeded; + try session.chargeGraphMetricDecode(0, node_ids.len); + + const specs = try session.graphMetricSpecs(); + const config = findConfig(specs, graph_index_name, metric_name) orelse return error.MetricNotConfigured; + const graph_index = session.findNamedArtifactIndex(.graph_segment, graph_index_name) orelse return error.GraphSegmentNotFound; + const graph_artifact = session.artifactRef(graph_index).?; + const artifact_name = try metric_segment.artifactNameAlloc(alloc, graph_index_name, metric_name); + defer alloc.free(artifact_name); + const metric_index = session.findNamedArtifactIndex(.graph_metric_segment, artifact_name) orelse return error.MetricNotReady; + const metric_artifact = session.artifactRef(metric_index) orelse return error.InvalidGraphMetricSegment; + + const control_len = metric_artifact.graph_metric_control_len; + var control_range = try fetchControlAlloc(session, metric_index, metric_artifact, control_len); + defer control_range.deinit(session.alloc); + const control = try metric_segment.decodeControl(control_range.bytes, config.edge_filter); + try validateControl(control.header, graph_artifact, metric_artifact, config); + if (control.header.materialization_state == .rejected) { + recordRejectionDiagnostic(session, graph_index_name, metric_name, control.header.materializer_fingerprint); + return error.GraphMetricMaterializationRejected; + } + const metadata = PointScoresMetadata{ + .config_fingerprint = control.header.config_fingerprint, + .converged = control.header.converged, + .iterations_completed = control.header.iterations_completed, + .delta = control.header.delta, + .metadata_version = control.header.version, + .published_generation = if (metric_artifact.published_generation != 0) metric_artifact.published_generation else session.manifest.version, + .edge_generation = if (metric_artifact.edge_generation != 0) metric_artifact.edge_generation else session.manifest.version, + .computed_at_ms = if (metric_artifact.computed_at_ms != 0) metric_artifact.computed_at_ms else @divTrunc(session.manifest.built_at_ns, std.time.ns_per_ms), + }; + if (node_ids.len == 0) return .{ .alloc = alloc, .range_alloc = alloc, .metadata = metadata, .metric_index = metric_index, .score_count = control.score_count, .score_data_offset = control.score_data_offset }; + + const footer_len = try routingFooterLen(metric_artifact, control.header.version); + const footer_offset = metric_artifact.byte_len - footer_len; + const expected_blocks = @as(usize, control.score_count) / metric_segment.score_block_entries + + @intFromBool(@as(usize, control.score_count) % metric_segment.score_block_entries != 0); + var point_routing = try loadPointRouting(alloc, session, metric_index, metric_artifact, control, footer_offset, expected_blocks, node_ids, candidate_order); + errdefer point_routing.deinit(); + const routing = point_routing.routing; + + @memset(values, null); + // Count exact spans before allocating. All columns borrow one permutation; + // only unresolved block spans survive preparation. + var blocks = CandidateBlocks{ .node_ids = node_ids, .order = candidate_order, .routing = routing }; + var block_count: usize = 0; + while (blocks.next() != null) : (block_count += 1) try session.checkCancellation(); + // Shrinking the owned slice may need a replacement allocation. + try session.chargeGraphMetricRetained(block_count * @sizeOf(TouchedBlock) * 2); + var touched_blocks = std.ArrayListUnmanaged(TouchedBlock).empty; + errdefer touched_blocks.deinit(alloc); + try touched_blocks.ensureTotalCapacityPrecise(alloc, block_count); + blocks.position = 0; + while (blocks.next()) |touched| { + try session.checkCancellation(); + if (session.cache != null) { + const entry = routing.entries[touched.block_index]; + var id_buf: [64]u8 = undefined; + const id = try metricBlockId(&id_buf, .score, entry.block_index); + if (try session.readCachedAuthenticatedBlockLease(std.heap.smp_allocator, metric_index, id, entry.offset, entry.len, &entry.checksum)) |hit| { + var lease = hit; + defer lease.deinit(); + try decodePointScoreBlock(session, entry, control.score_count, lease.bytes(), candidate_order[touched.first_pending..][0..touched.pending_count], node_ids, values); + continue; + } + } + touched_blocks.appendAssumeCapacity(touched); + } + const touched = try touched_blocks.toOwnedSlice(alloc); + return .{ .alloc = alloc, .range_alloc = alloc, .metadata = metadata, .metric_index = metric_index, .score_count = control.score_count, .score_data_offset = control.score_data_offset, .point_routing = point_routing, .candidate_order = candidate_order, .touched_blocks = touched }; +} + +fn decodePointScoreBlock(session: *runtime_mod.QuerySession, entry: metric_segment.codec.RoutingEntry, score_count: usize, payload: []const u8, candidate_rows: []const u32, node_ids: []const []const u8, values: []?f64) !void { + const first_score = std.math.mul(usize, entry.block_index, metric_segment.score_block_entries) catch return error.InvalidGraphMetricSegment; + if (first_score >= score_count) return error.InvalidGraphMetricSegment; + const expected = @min(metric_segment.score_block_entries, score_count - first_score); + try session.chargeGraphMetricDecode(1, expected); + const decoded = try metric_segment.decodeScoreBlockWithCancellation(payload, session.cancellation); + if (decoded.len != expected or !decoded.scores[0].eqlNode(decoded.node_prefix, entry.first_node_id)) return error.InvalidGraphMetricSegment; + try decoded.populateSorted(node_ids, candidate_rows, values, session.cancellation); +} + +fn executePointScores(session: *runtime_mod.QuerySession, plan: *const PointScorePlan, node_ids: []const []const u8, values: []?f64) !void { + const point_routing = plan.point_routing orelse return; + const routing = point_routing.routing; + const alloc = plan.alloc; + const metric_index = plan.metric_index; + const fetch_ranges = plan.ranges; + var fetch_start: usize = 0; + while (fetch_start < fetch_ranges.len) { + const fetch_end = try admittedMetricRangeBatchEnd(session, fetch_ranges, fetch_start); + const range_batch = fetch_ranges[fetch_start..fetch_end]; + var fetched_ranges = try fetchMetricRangeBatchAlloc( + alloc, + session, + metric_index, + plan.metadata.metadata_version, + routing.entries, + range_batch, + .reserved_score, + ); + defer fetched_ranges.deinit(alloc); + + for (range_batch, fetched_ranges.payloads) |range, payload| { + try session.checkCancellation(); + var touched_index = std.sort.lowerBound(TouchedBlock, plan.touched_blocks, range.first_block, struct { + fn order(block_index: usize, touched: TouchedBlock) std.math.Order { + return std.math.order(block_index, touched.block_index); + } + }.order); + while (touched_index < plan.touched_blocks.len and plan.touched_blocks[touched_index].block_index <= range.last_block) : (touched_index += 1) { + const touched = plan.touched_blocks[touched_index]; + const block_index = touched.block_index; + const entry = routing.entries[block_index]; + if (entry.offset < range.offset) return error.InvalidGraphMetricSegment; + const relative_offset = std.math.cast(usize, entry.offset - range.offset) orelse return error.InvalidGraphMetricSegment; + const relative_end = std.math.add(usize, relative_offset, entry.len) catch return error.InvalidGraphMetricSegment; + if (relative_end > payload.len) return error.InvalidGraphMetricSegment; + try decodePointScoreBlock(session, entry, plan.score_count, payload[relative_offset..relative_end], plan.candidate_order[touched.first_pending..][0..touched.pending_count], node_ids, values); + } + } + fetch_start = fetch_end; + } +} + +fn admitPointPlans(alloc: Allocator, session: *runtime_mod.QuerySession, plans: []?PointScorePlan) !void { + const remaining = session.graphMetricRangeBudget(); + const allowance = std.math.cast(usize, remaining.requests) orelse std.math.maxInt(usize); + var requests: usize = 0; + for (plans) |*maybe_plan| { + const plan = &maybe_plan.*.?; + plan.range_alloc = alloc; + const routing = plan.point_routing orelse continue; + try session.checkCancellation(); + plan.ranges = try planSparseScoreFetchRangesWithBudgetAlloc(alloc, routing.routing.entries, plan.touched_blocks, plan.score_data_offset, std.math.maxInt(usize), .{ .session = session, .cancellation = session.cancellation }); + requests = std.math.add(usize, requests, plan.ranges.len) catch return error.GraphMetricQueryBudgetExceeded; + } + if (requests > allowance) { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + try session.chargeGraphMetricRetained(plans.len * @sizeOf(RangePlanningColumn)); + const inputs = try temp.alloc(RangePlanningColumn, plans.len); + for (plans, inputs) |*maybe_plan, *input| { + const plan = &maybe_plan.*.?; + if (plan.point_routing) |*routing| try routing.expandForCoalescing(session, plan.touched_blocks); + const entries = if (plan.point_routing) |routing| routing.routing.entries else &.{}; + try session.checkCancellation(); + try session.chargeGraphMetricRetained(std.math.mul(usize, entries.len, @sizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded); + const counts = try temp.alloc(usize, entries.len); + @memset(counts, 0); + for (plan.touched_blocks) |block| counts[block.block_index] = block.pending_count; + input.* = .{ .entries = entries, .counts = counts, .score_data_offset = plan.score_data_offset }; + } + const ranges = try planScoreColumnsWithinBudgetAlloc(alloc, inputs, allowance, remaining.bytes, .{ .cancellation = session.cancellation, .session = session }); + defer alloc.free(ranges); + for (plans, ranges) |*maybe_plan, planned| { + const plan = &maybe_plan.*.?; + alloc.free(plan.ranges); + plan.ranges = planned; + } + } + requests = 0; + var bytes: usize = 0; + for (plans) |maybe_plan| for (maybe_plan.?.ranges) |range| { + requests = std.math.add(usize, requests, 1) catch return error.GraphMetricQueryBudgetExceeded; + bytes = std.math.add(usize, bytes, range.len) catch return error.GraphMetricQueryBudgetExceeded; + }; + // Reserve the complete score plan atomically before any score I/O starts. + try session.reserveGraphMetricRanges(requests, bytes); +} + +fn appendExactMissRun(alloc: Allocator, ranges: *std.ArrayListUnmanaged(ScoreFetchRange), block_index: usize, entry: metric_segment.codec.RoutingEntry) !void { + if (entry.len == 0 or entry.len > coalesced_score_window_bytes) return error.GraphMetricQueryBudgetExceeded; + _ = std.math.add(u64, entry.offset, entry.len) catch return error.InvalidGraphMetricSegment; + if (ranges.items.len != 0) { + const last = &ranges.items[ranges.items.len - 1]; + if (last.last_block + 1 == block_index and last.offset + last.len == entry.offset and entry.len <= coalesced_score_window_bytes - last.len) { + last.last_block = block_index; + last.len += entry.len; + return; + } + } + try ranges.append(alloc, .{ .first_block = block_index, .last_block = block_index, .offset = entry.offset, .len = entry.len }); +} + +fn planSparseScoreFetchRangesAlloc(alloc: Allocator, entries: []const metric_segment.codec.RoutingEntry, touched_blocks: anytype, score_data_offset: u64, request_limit: usize) ![]ScoreFetchRange { + return planSparseScoreFetchRangesWithBudgetAlloc(alloc, entries, touched_blocks, score_data_offset, request_limit, .{}); +} + +fn planSparseScoreFetchRangesWithBudgetAlloc(alloc: Allocator, entries: []const metric_segment.codec.RoutingEntry, touched_blocks: anytype, score_data_offset: u64, request_limit: usize, budget: ScorePlanningBudget) ![]ScoreFetchRange { + // Precise upper-bound capacity avoids geometric growth before admission. + // Include a possible owned-slice shrink while the original remains live. + try budget.charge(touched_blocks.len, std.math.mul(usize, touched_blocks.len, 2 * @sizeOf(ScoreFetchRange)) catch return error.GraphMetricQueryBudgetExceeded); + var runs = std.ArrayListUnmanaged(ScoreFetchRange).empty; + defer runs.deinit(alloc); + try runs.ensureTotalCapacityPrecise(alloc, touched_blocks.len); + for (touched_blocks) |touched| { + if (touched.block_index >= entries.len or entries[touched.block_index].offset < score_data_offset) return error.InvalidGraphMetricSegment; + try appendExactMissRun(alloc, &runs, touched.block_index, entries[touched.block_index]); + } + if (runs.items.len <= request_limit) return runs.toOwnedSlice(alloc); + try budget.charge(entries.len, std.math.mul(usize, entries.len, @sizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded); + const counts = try alloc.alloc(usize, entries.len); + defer alloc.free(counts); + @memset(counts, 0); + for (touched_blocks) |touched| counts[touched.block_index] = touched.pending_count; + return planScoreFetchRangesAlloc(alloc, entries, counts, score_data_offset, request_limit); +} + +const RangePlanningColumn = struct { + entries: []const metric_segment.codec.RoutingEntry, + counts: []const usize, + score_data_offset: u64, +}; + +const ScorePlanningBudget = struct { + cancellation: operation.CancellationToken = .{}, + session: ?*runtime_mod.QuerySession = null, + + fn charge(self: @This(), work: usize, bytes: usize) !void { + try self.cancellation.check(); + if (self.session) |session| { + try session.chargeGraphMetricDecode(0, work); + try session.chargeGraphMetricRetained(bytes); + } + } +}; + +fn planScoreFetchRangesAlloc(alloc: Allocator, entries: []const metric_segment.codec.RoutingEntry, counts: []const usize, score_data_offset: u64, request_limit: usize) ![]ScoreFetchRange { + const columns = try planScoreColumnsRangesAlloc(alloc, &.{.{ .entries = entries, .counts = counts, .score_data_offset = score_data_offset }}, request_limit); + defer alloc.free(columns); + return columns[0]; +} + +fn planScoreColumnsRangesAlloc(alloc: Allocator, columns: []const RangePlanningColumn, request_limit: usize) ![][]ScoreFetchRange { + return planScoreColumnsWithinBudgetAlloc(alloc, columns, request_limit, std.math.maxInt(u64), .{}); +} + +const PlanningBlock = struct { + column: usize, + block_index: usize, + offset: u64, + end: u64, + region_start: usize, +}; +const PlannedColumnRange = struct { column: usize, range: ScoreFetchRange }; + +fn copyColumnRangesAlloc(alloc: Allocator, column_count: usize, planned: []const PlannedColumnRange, reverse: bool) ![][]ScoreFetchRange { + const result = try alloc.alloc([]ScoreFetchRange, column_count); + var initialized: usize = 0; + errdefer { + for (result[0..initialized]) |ranges| alloc.free(ranges); + alloc.free(result); + } + for (result, 0..) |*ranges, column| { + var count: usize = 0; + for (planned) |item| if (item.column == column) { + count += 1; + }; + ranges.* = try alloc.alloc(ScoreFetchRange, count); + initialized += 1; + var i: usize = 0; + for (planned) |item| if (item.column == column) { + ranges.*[if (reverse) count - 1 - i else i] = item.range; + i += 1; + }; + } + return result; +} + +/// Exact range partitioning in O(missed blocks * request limit). A monotone +/// queue evaluates all authenticated contiguous start positions, including +/// partial merges and ranges crossing former fixed-window boundaries. +fn planScoreColumnsWithinBudgetAlloc(alloc: Allocator, columns: []const RangePlanningColumn, request_limit: usize, byte_limit: u64, budget: ScorePlanningBudget) ![][]ScoreFetchRange { + var count: usize = 0; + var exact_bytes: u64 = 0; + for (columns) |column| { + if (column.entries.len != column.counts.len) return error.InvalidGraphMetricSegment; + // Both validation passes visit routing entries, including cache hits. + try budget.charge(std.math.mul(usize, column.entries.len, 2) catch return error.GraphMetricQueryBudgetExceeded, 0); + for (column.entries, column.counts, 0..) |entry, hits, i| { + if (i % 1024 == 0) try budget.cancellation.check(); + if (hits == 0) continue; + count = std.math.add(usize, count, 1) catch return error.GraphMetricQueryBudgetExceeded; + exact_bytes = std.math.add(u64, exact_bytes, entry.len) catch return error.GraphMetricQueryBudgetExceeded; + } + } + if (exact_bytes > byte_limit) return error.GraphMetricQueryBudgetExceeded; + const scratch_bytes = std.math.mul(usize, count, @sizeOf(PlanningBlock) + @sizeOf(PlannedColumnRange)) catch return error.GraphMetricQueryBudgetExceeded; + try budget.charge(0, scratch_bytes); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const temp = arena.allocator(); + const blocks = try temp.alloc(PlanningBlock, count); + const baseline = try temp.alloc(PlannedColumnRange, count); + var position: usize = 0; + var run_count: usize = 0; + for (columns, 0..) |column, column_index| { + var region_start = position; + var previous_end: ?u64 = null; + for (column.entries, column.counts, 0..) |entry, hits, block_index| { + if (block_index % 1024 == 0) try budget.cancellation.check(); + const end = std.math.add(u64, entry.offset, entry.len) catch return error.InvalidGraphMetricSegment; + if (entry.offset < column.score_data_offset or (previous_end != null and entry.offset < previous_end.?)) return error.InvalidGraphMetricSegment; + if (previous_end == null or entry.offset != previous_end.?) region_start = position; + previous_end = end; + if (hits == 0) continue; + if (entry.len == 0 or entry.len > coalesced_score_window_bytes) return error.GraphMetricQueryBudgetExceeded; + blocks[position] = .{ .column = column_index, .block_index = block_index, .offset = entry.offset, .end = end, .region_start = region_start }; + position += 1; + if (run_count != 0) { + const last = &baseline[run_count - 1]; + if (last.column == column_index and last.range.last_block + 1 == block_index and last.range.offset + last.range.len == entry.offset and entry.len <= coalesced_score_window_bytes - last.range.len) { + last.range.last_block = block_index; + last.range.len += entry.len; + continue; + } + } + baseline[run_count] = .{ .column = column_index, .range = .{ .first_block = block_index, .last_block = block_index, .offset = entry.offset, .len = entry.len } }; + run_count += 1; + } + } + if (run_count <= request_limit) return copyColumnRangesAlloc(alloc, columns.len, baseline[0..run_count], false); + const limit = request_limit; + // Greedy longest ranges establish a tight minimum request count before + // allocating the partition table. Never bridge unauthenticated gaps. + var minimum_reads: usize = 0; + var i: usize = 0; + while (i < blocks.len) { + minimum_reads += 1; + const first = i; + i += 1; + while (i < blocks.len and blocks[i].region_start <= first and blocks[i].end - blocks[first].offset <= coalesced_score_window_bytes) : (i += 1) {} + } + if (minimum_reads > limit) return error.GraphMetricQueryBudgetExceeded; + const width = count + 1; + const cells = std.math.mul(usize, limit, width) catch return error.GraphMetricQueryBudgetExceeded; + if (cells > (runtime_mod.GraphMetricReadLimits{}).max_work_items or count > std.math.maxInt(u32)) return error.GraphMetricQueryBudgetExceeded; + const table_bytes = std.math.mul(usize, cells, @sizeOf(u32)) catch return error.GraphMetricQueryBudgetExceeded; + const vector_bytes = std.math.mul(usize, width, 2 * @sizeOf(u64) + @sizeOf(usize)) catch return error.GraphMetricQueryBudgetExceeded; + try budget.charge(cells, std.math.add(usize, table_bytes, vector_bytes) catch return error.GraphMetricQueryBudgetExceeded); + const predecessors = try temp.alloc(u32, cells); + var costs = try temp.alloc(u64, width); + var next = try temp.alloc(u64, width); + const queue = try temp.alloc(usize, width); + const infinity = std.math.maxInt(u64); + @memset(costs, infinity); + costs[0] = 0; + var best_cost: u64 = infinity; + var best_reads: usize = 0; + for (0..limit) |round| { + try budget.cancellation.check(); + @memset(next, infinity); + var head: usize = 0; + var tail: usize = 0; + for (blocks, 0..) |block, j| { + if (j % 1024 == 0) try budget.cancellation.check(); + if (block.region_start == j) { + head = 0; + tail = 0; + } + while (head < tail and (queue[head] < block.region_start or block.end - blocks[queue[head]].offset > coalesced_score_window_bytes)) : (head += 1) {} + if (costs[j] != infinity) { + const key = @as(i128, costs[j]) - @as(i128, block.offset); + while (head < tail) { + const prior = queue[tail - 1]; + if (@as(i128, costs[prior]) - @as(i128, blocks[prior].offset) < key) break; + tail -= 1; + } + queue[tail] = j; + tail += 1; + } + if (head == tail) continue; + const first = queue[head]; + const cost = std.math.add(u64, costs[first], block.end - blocks[first].offset) catch continue; + if (cost > byte_limit) continue; + next[j + 1] = cost; + predecessors[round * width + j + 1] = @intCast(first); + } + if (next[count] < best_cost) { + best_cost = next[count]; + best_reads = round + 1; + } + std.mem.swap([]u64, &costs, &next); + if (best_cost == exact_bytes) break; + } + if (best_reads == 0) return error.GraphMetricQueryBudgetExceeded; + var remaining = count; + var reads = best_reads; + var output_count: usize = 0; + while (reads != 0) { + reads -= 1; + const first = predecessors[reads * width + remaining]; + const a = blocks[first]; + const b = blocks[remaining - 1]; + baseline[output_count] = .{ .column = a.column, .range = .{ .first_block = a.block_index, .last_block = b.block_index, .offset = a.offset, .len = @intCast(b.end - a.offset) } }; + output_count += 1; + remaining = first; + } + std.debug.assert(remaining == 0); + return copyColumnRangesAlloc(alloc, columns.len, baseline[0..output_count], true); +} + +fn fetchControlAlloc( + session: *runtime_mod.QuerySession, + metric_index: usize, + artifact: manifest_mod.ArtifactRef, + expected_len: usize, +) !OwnedMetricRange { + if (artifact.metadata_version != metric_segment.wire_version or + artifact.graph_metric_control_len != expected_len) return error.InvalidGraphMetricSegment; + const entries = [_]metric_segment.codec.RoutingEntry{.{ + .first_node_id = "", + .block_index = 2, + .offset = 0, + .len = expected_len, + .checksum = artifact.graph_metric_control_checksum, + }}; + return fetchMetadataRangeAlloc(session, metric_index, &entries); +} + +fn routingFooterLen( + artifact: manifest_mod.ArtifactRef, + segment_version: u16, +) !usize { + if (segment_version != metric_segment.wire_version or artifact.metadata_version != segment_version or + artifact.graph_metric_routing_footer_len == 0 or + artifact.graph_metric_routing_footer_len > metric_segment.codec.max_routing_bytes or + artifact.graph_metric_routing_footer_len > artifact.byte_len) + { + return error.InvalidGraphMetricSegment; + } + return artifact.graph_metric_routing_footer_len; +} + +fn fetchRoutingFooterAlloc( + session: *runtime_mod.QuerySession, + metric_index: usize, + artifact: manifest_mod.ArtifactRef, + segment_version: u16, + footer_offset: u64, + footer_len: usize, + root_len: usize, +) !OwnedMetricRange { + if (segment_version != metric_segment.wire_version) return error.InvalidGraphMetricSegment; + if (root_len > footer_len) return error.InvalidGraphMetricSegment; + const root = metric_segment.codec.RoutingEntry{ + .first_node_id = "", + .block_index = 1, + .offset = footer_offset + footer_len - root_len, + .len = root_len, + .checksum = artifact.graph_metric_routing_checksum, + }; + const both = [_]metric_segment.codec.RoutingEntry{ .{ + .first_node_id = "", + .offset = footer_offset, + .len = footer_len - root_len, + .checksum = artifact.graph_metric_point_index_checksum, + }, root }; + return fetchMetadataRangeAlloc(session, metric_index, if (footer_len == root_len) &.{root} else &both); +} + +fn fetchMetadataRangeAlloc(session: *runtime_mod.QuerySession, metric_index: usize, entries: []const metric_segment.codec.RoutingEntry) !OwnedMetricRange { + if (entries.len == 0) return error.InvalidGraphMetricSegment; + const last = entries[entries.len - 1]; + const end = std.math.add(u64, last.offset, last.len) catch return error.InvalidGraphMetricSegment; + const extent = std.math.sub(u64, end, entries[0].offset) catch return error.InvalidGraphMetricSegment; + const len = std.math.cast(usize, extent) orelse return error.InvalidGraphMetricSegment; + // Large valid metadata can exceed the canonical memory pool's per-fill + // limit. Authenticate it directly; optional disk retention must not become + // a synchronous fallback on the serving path. + if (len > coalesced_score_window_bytes) { + var memory = try session.reserveGraphMetricMemory(try transportMemoryBytes(len, entries.len)); + errdefer memory.deinit(); + try session.chargeGraphMetricRange(len); + const bytes = try fetchCanonicalRunAlloc(session.alloc, session, metric_index, entries); + var temporary = memory.split(memory.bytes - len); + temporary.deinit(); + return .{ .bytes = bytes, .memory = memory }; + } + var fetched = try fetchMetricRangeAlloc(session.alloc, session, metric_index, metric_segment.wire_version, entries, .{ + .offset = entries[0].offset, + .len = len, + .first_block = 0, + .last_block = entries.len - 1, + }, .metadata); + var temporary = fetched.memory.split(fetched.memory.bytes - len); + temporary.deinit(); + return fetched; +} + +const DirectoryRead = struct { + footer_offset: u64, + checksum: [32]u8, + block_count: usize, +}; + +const OwnedRoutingLease = struct { + lease: routing_cache.Lease, + entry: *routing_cache.Entry, + memory: runtime_mod.GraphMetricReadBudget.Reservation, + + fn admit(session: *runtime_mod.QuerySession, lease_value: routing_cache.Lease) !@This() { + var lease = lease_value; + errdefer lease.deinit(); + const memory = try session.reserveGraphMetricMemory(lease.entry.bytes()); + return .{ .lease = lease, .entry = lease.entry, .memory = memory }; + } + + fn deinit(self: *@This()) void { + self.lease.deinit(); + self.memory.deinit(); + } +}; + +const PointRouting = struct { + alloc: Allocator, + payload_alloc: Allocator, + payloads: std.ArrayListUnmanaged([]u8) = .empty, + routing: metric_segment.codec.RoutingIndex, + lease: ?OwnedRoutingLease = null, + page_leases: []?OwnedRoutingLease = &.{}, + + /// Only request-budget pressure needs intervening block locators. Ordinary + /// sparse reads retain selected locators; the exact coalescer can lazily + /// expand already leased pages without another fetch or decode. + fn expandForCoalescing(self: *@This(), session: *runtime_mod.QuerySession, touched: []TouchedBlock) !void { + if (self.page_leases.len == 0) return; + var count: usize = 0; + for (self.page_leases) |lease| count += lease.?.entry.routing.entries.len; + if (count == self.routing.entries.len) return; + try session.chargeGraphMetricRetained(count * @sizeOf(metric_segment.codec.RoutingEntry)); + try session.chargeGraphMetricDecode(0, count); + const expanded = try self.alloc.alloc(metric_segment.codec.RoutingEntry, count); + errdefer self.alloc.free(expanded); + var offset: usize = 0; + for (self.page_leases) |lease| { + const entries = lease.?.entry.routing.entries; + @memcpy(expanded[offset..][0..entries.len], entries); + offset += entries.len; + } + var index: usize = 0; + for (touched) |*block| { + const global = self.routing.entries[block.block_index].block_index; + while (index < expanded.len and expanded[index].block_index < global) : (index += 1) {} + if (index == expanded.len or expanded[index].block_index != global) return error.InvalidGraphMetricSegment; + block.block_index = index; + } + self.alloc.free(self.routing.entries); + self.routing.entries = expanded; + } + + fn deinit(self: *@This()) void { + if (self.lease) |*lease| lease.deinit() else self.routing.deinit(self.alloc); + for (self.payloads.items) |payload| self.payload_alloc.free(payload); + self.payloads.deinit(self.alloc); + for (self.page_leases) |*lease| if (lease.*) |*held| held.deinit(); + self.alloc.free(self.page_leases); + } +}; + +fn loadPointRouting(alloc: Allocator, session: *runtime_mod.QuerySession, metric_index: usize, artifact: manifest_mod.ArtifactRef, control: metric_segment.codec.Control, footer_offset: u64, block_count: usize, node_ids: []const []const u8, candidate_order: []const u32) !PointRouting { + const codec = metric_segment.codec; + const root_len = codec.routingRootLen(control.score_count); + if (root_len > artifact.graph_metric_routing_footer_len) return error.InvalidGraphMetricSegment; + // A one-page index is bounded even with maximum-length identifiers. + // Small byte extents also keep the one-round-trip decoded cache fast path. + if (block_count <= codec.routing_page_entries or artifact.graph_metric_routing_footer_len <= 64 * 1024) { + var lease = try acquireRouting(session, metric_index, artifact, control.header.version, footer_offset, artifact.graph_metric_routing_footer_len, block_count, root_len, false, null); + errdefer lease.deinit(); + const routing = lease.entry.routing; + if (routing.entries.len != block_count or routing.footer_offset != footer_offset or routing.primary_data_offset != control.score_data_offset) return error.InvalidGraphMetricSegment; + return .{ .alloc = alloc, .payload_alloc = session.alloc, .routing = routing, .lease = lease }; + } + var root_lease = try acquireRouting(session, metric_index, artifact, control.header.version, artifact.byte_len - root_len, root_len, 40, root_len, true, null); + defer root_lease.deinit(); + const root = root_lease.entry.routing; + if (root.footer_offset != footer_offset or root.primary_data_offset != control.score_data_offset) return error.InvalidGraphMetricSegment; + const directory_offset = artifact.byte_len - root_len - root.directory_len; + const page_count = std.math.divCeil(usize, block_count, codec.routing_page_entries) catch return error.InvalidGraphMetricSegment; + var directory_lease = try acquireRouting(session, metric_index, artifact, control.header.version, directory_offset, root.directory_len, page_count, root_len, false, .{ + .footer_offset = footer_offset, + .checksum = root.directory_checksum, + .block_count = block_count, + }); + defer directory_lease.deinit(); + const directory = directory_lease.entry.routing; + var selected = std.ArrayListUnmanaged(usize).empty; + defer selected.deinit(alloc); + const max_selected = @min(node_ids.len, directory.entries.len); + try session.chargeGraphMetricRetained(max_selected * @sizeOf(usize)); + try selected.ensureTotalCapacityPrecise(alloc, max_selected); + var pages = CandidateBlocks{ .node_ids = node_ids, .order = candidate_order, .routing = directory }; + while (pages.next()) |page| { + try session.checkCancellation(); + selected.appendAssumeCapacity(page.block_index); + } + var entries = std.ArrayListUnmanaged(codec.RoutingEntry).empty; + errdefer entries.deinit(alloc); + var entry_count: usize = 0; + for (selected.items) |i| entry_count += @min(codec.routing_page_entries, block_count - directory.entries[i].block_index); + // A sparse plan retains one locator per candidate block, not every entry + // of its routing page. Node IDs borrow immutable decoded-page leases. + entry_count = @min(entry_count, node_ids.len); + try session.chargeGraphMetricRetained(2 * entry_count * @sizeOf(codec.RoutingEntry) + selected.items.len * (@sizeOf([]u8) + @sizeOf(usize))); + try entries.ensureTotalCapacityPrecise(alloc, entry_count); + const payload_alloc = std.heap.smp_allocator; + var payloads = std.ArrayListUnmanaged([]u8).empty; + defer { + for (payloads.items) |payload| payload_alloc.free(payload); + payloads.deinit(alloc); + } + try payloads.ensureTotalCapacityPrecise(alloc, selected.items.len); + // Cache units are authenticated pages, independent of transport grouping. + // Keep scratch state proportional to selected pages, not the full index. + try session.chargeGraphMetricRetained(std.math.mul(usize, selected.items.len, @sizeOf(?[]const u8)) catch return error.GraphMetricQueryBudgetExceeded); + const page_views = try alloc.alloc(?[]const u8, selected.items.len); + defer alloc.free(page_views); + @memset(page_views, null); + try session.chargeGraphMetricRetained(selected.items.len * @sizeOf(?OwnedRoutingLease)); + const page_leases = try alloc.alloc(?OwnedRoutingLease, selected.items.len); + @memset(page_leases, null); + errdefer { + for (page_leases) |*lease| if (lease.*) |*held| held.deinit(); + alloc.free(page_leases); + } + var misses = std.ArrayListUnmanaged(usize).empty; + defer misses.deinit(alloc); + try misses.ensureTotalCapacityPrecise(alloc, @min(selected.items.len, 64)); + var completed: usize = 0; + var first_missing: usize = 0; + while (completed < selected.items.len) { + try session.checkCancellation(); + var payload_memory = try session.reserveGraphMetricMemory(0); + defer payload_memory.deinit(); + defer { + for (payloads.items) |payload| payload_alloc.free(payload); + payloads.clearRetainingCapacity(); + } + while (first_missing < selected.items.len and page_leases[first_missing] != null) first_missing += 1; + // Never wait while owning unpublished fills: overlapping multi-page + // requests can otherwise deadlock each other or saturate the fill table. + const PendingPage = struct { + view_index: usize, + fill: ?usize = null, + waiter: ?routing_cache.Cache.Waiter = null, + memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}, + }; + var pending: [64]PendingPage = undefined; + var pending_count: usize = 0; + var saturated: ?u32 = null; + defer for (pending[0..pending_count]) |*item| { + if (item.fill) |index| session.cache.?.graph_metric_routing.finish(index, session.io); + if (item.waiter) |*waiter| waiter.deinit(); + item.memory.deinit(); + }; + misses.clearRetainingCapacity(); + for (selected.items[first_missing..], first_missing..) |i, view_index| { + if (page_leases[view_index] != null) continue; + if (pending_count == pending.len) break; + const page = directory.entries[i]; + var item = PendingPage{ .view_index = view_index }; + if (session.cache) |cache| switch (cache.graph_metric_routing.begin(pointPageCacheKey(artifact, page, block_count, root))) { + .hit => |cached| { + page_leases[view_index] = try OwnedRoutingLease.admit(session, cached); + completed += 1; + try session.chargeGraphMetricDecode(0, 1); + continue; + }, + .fill => |index| item.fill = index, + .wait => |waiter| item.waiter = waiter, + .saturated => |epoch| { + saturated = epoch; + break; + }, + }; + pending[pending_count] = item; + pending_count += 1; + if (item.waiter != null) continue; + const count = @min(codec.routing_page_entries, block_count - page.block_index); + try session.chargeGraphMetricDecode(1, count); + pending[pending_count - 1].memory = try session.reserveGraphMetricMemory(page.len + count * @sizeOf(codec.RoutingEntry) + @sizeOf(routing_cache.Entry)); + var id_buf: [64]u8 = undefined; + const id = try metricBlockId(&id_buf, .routing, page.block_index); + const prior_payload_bytes = payload_memory.bytes; + try payload_memory.grow(page.len); + if (try session.readCachedAuthenticatedBlockAlloc(payload_alloc, metric_index, id, page.offset, page.len, &page.checksum)) |bytes| { + payloads.append(alloc, bytes) catch |err| { + payload_alloc.free(bytes); + return err; + }; + page_views[view_index] = bytes; + } else { + payload_memory.shrinkTo(prior_payload_bytes); + try misses.append(alloc, i); + } + } + try session.chargeGraphMetricRetained(misses.items.len * 2 * @sizeOf(ScoreFetchRange)); + const ranges = try planRoutingPageRangesAlloc(alloc, directory.entries, misses.items); + defer alloc.free(ranges); + var start: usize = 0; + while (start < ranges.len) { + const end = try admittedMetricRangeBatchEnd(session, ranges, start); + var fetched = try fetchMetricRangeBatchAlloc(alloc, session, metric_index, control.header.version, directory.entries, ranges[start..end], .routing); + defer fetched.deinit(alloc); + for (ranges[start..end], fetched.payloads) |range, *payload| { + try payloads.append(alloc, payload.*); + const bytes = payload.*; + payload.* = @constCast((&[_]u8{})[0..]); + var owned_memory = fetched.memory.split(bytes.len); + payload_memory.absorb(&owned_memory); + const first_view = std.sort.lowerBound(usize, selected.items, range.first_block, struct { + fn order(needle: usize, item: usize) std.math.Order { + return std.math.order(needle, item); + } + }.order); + for (range.first_block..range.last_block + 1) |i| { + const page = directory.entries[i]; + const offset: usize = @intCast(page.offset - range.offset); + const view_index = first_view + i - range.first_block; + std.debug.assert(selected.items[view_index] == i); + page_views[view_index] = bytes[offset..][0..page.len]; + } + } + start = end; + } + + for (pending[0..pending_count]) |*item| { + if (item.waiter != null) continue; + const view_index = item.view_index; + const i = selected.items[view_index]; + const page = directory.entries[i]; + const bytes = page_views[view_index] orelse return error.InvalidGraphMetricSegment; + const owner = if (session.cache) |cache| cache.alloc else alloc; + const owned = try owner.dupe(u8, bytes); + errdefer owner.free(owned); + const decoded = try codec.decodePointPageAlloc(owner, owned, page, block_count, root.primary_data_offset, root.primary_data_end, session.cancellation); + errdefer owner.free(decoded); + if (i + 1 < directory.entries.len and std.mem.order(u8, decoded[decoded.len - 1].first_node_id, directory.entries[i + 1].first_node_id) != .lt) return error.InvalidGraphMetricSegment; + const entry = try owner.create(routing_cache.Entry); + entry.* = .{ + .key = pointPageCacheKey(artifact, page, block_count, root), + .class = .page, + .alloc = owner, + .footer = owned, + .routing = .{ .entries = decoded, .ranked_entries = &.{}, .top_score_count = 0, .footer_offset = footer_offset }, + }; + const lease = if (session.cache) |cache| + cache.graph_metric_routing.publish(item.fill.?, entry, cache.cfg.max_graph_metric_routing_bytes) + else + routing_cache.Lease{ .entry = entry }; + item.memory.shrinkTo(lease.entry.bytes()); + page_leases[view_index] = .{ .lease = lease, .entry = lease.entry, .memory = item.memory }; + item.memory = .{}; + + completed += 1; + if (item.fill) |index| { + session.cache.?.graph_metric_routing.finish(index, session.io); + item.fill = null; + } + } + for (pending[0..pending_count]) |*item| { + if (item.waiter) |*waiter| { + if (try waiter.awaitResult(session.io, session.cancellation)) |shared| { + page_leases[item.view_index] = try OwnedRoutingLease.admit(session, shared); + completed += 1; + try session.chargeGraphMetricDecode(0, 1); + } + waiter.deinit(); + item.waiter = null; + } + } + // With no ownership or registrations, wait for table capacity. Failed + // leaders are retried by the next pass under the same cancellation. + if (pending_count == 0) if (saturated) |epoch| + try session.cache.?.graph_metric_routing.awaitFill(session.io, session.cancellation, epoch); + } + pages.position = 0; + for (selected.items, 0..) |i, view_index| { + const decoded = page_leases[view_index].?.entry.routing.entries; + if (i + 1 < directory.entries.len and std.mem.order(u8, decoded[decoded.len - 1].first_node_id, directory.entries[i + 1].first_node_id) != .lt) return error.InvalidGraphMetricSegment; + const span = pages.next() orelse return error.InvalidGraphMetricSegment; + std.debug.assert(span.block_index == i); + var candidates = CandidateBlocks{ + .node_ids = node_ids, + .order = candidate_order[span.first_pending..][0..span.pending_count], + .routing = page_leases[view_index].?.entry.routing, + }; + while (candidates.next()) |candidate| { + try session.checkCancellation(); + entries.appendAssumeCapacity(decoded[candidate.block_index]); + } + } + const owned_entries = try entries.toOwnedSlice(alloc); + errdefer alloc.free(owned_entries); + return .{ + .alloc = alloc, + .payload_alloc = payload_alloc, + .page_leases = page_leases, + .routing = .{ + .entries = owned_entries, + .ranked_entries = try alloc.alloc(codec.RankedRoutingEntry, 0), + .top_score_count = 0, + .footer_offset = footer_offset, + .primary_data_offset = root.primary_data_offset, + .primary_data_end = root.primary_data_end, + }, + }; +} + +fn pointPageCacheKey(artifact: manifest_mod.ArtifactRef, page: metric_segment.codec.RoutingEntry, block_count: usize, root: metric_segment.codec.RoutingIndex) [32]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update("graph-metric-decoded-point-page-v1"); + hash.update(artifact.artifact_id); + hash.update(&.{0}); + hash.update(artifact.checksum); + hash.update(&artifact.graph_metric_point_index_checksum); + hash.update(&page.checksum); + for ([_]u64{ artifact.byte_len, page.offset, page.len, page.block_index, block_count, root.primary_data_offset, root.primary_data_end }) |value| { + var bytes: [8]u8 = undefined; + std.mem.writeInt(u64, &bytes, value, .little); + hash.update(&bytes); + } + hash.update(page.first_node_id); + var key: [32]u8 = undefined; + hash.final(&key); + return key; +} + +/// Contiguous selected routing pages share one authenticated range. Independent +/// runs use the same bounded byte/fanout executor as primary score reads. +fn planRoutingPageRangesAlloc(alloc: Allocator, entries: []const metric_segment.codec.RoutingEntry, selected: []const usize) ![]ScoreFetchRange { + var ranges = std.ArrayListUnmanaged(ScoreFetchRange).empty; + errdefer ranges.deinit(alloc); + try ranges.ensureTotalCapacityPrecise(alloc, selected.len); + var previous: ?usize = null; + for (selected) |index| { + if (previous == index) continue; + if (index >= entries.len or (previous != null and index < previous.?)) return error.InvalidGraphMetricSegment; + previous = index; + const entry = entries[index]; + if (entry.len == 0 or entry.len > coalesced_score_window_bytes) return error.InvalidGraphMetricSegment; + _ = std.math.add(u64, entry.offset, entry.len) catch return error.InvalidGraphMetricSegment; + if (ranges.items.len > 0) { + const last = &ranges.items[ranges.items.len - 1]; + if (last.last_block + 1 == index and last.offset + last.len == entry.offset and entry.len <= coalesced_score_window_bytes - last.len) { + last.last_block = index; + last.len += entry.len; + continue; + } + } + try ranges.append(alloc, .{ .first_block = index, .last_block = index, .offset = entry.offset, .len = entry.len }); + } + return ranges.toOwnedSlice(alloc); +} + +fn acquireRouting( + session: *runtime_mod.QuerySession, + metric_index: usize, + artifact: manifest_mod.ArtifactRef, + version: u16, + offset: u64, + len: usize, + decode_work: usize, + root_len: usize, + ranked_only: bool, + directory: ?DirectoryRead, +) !OwnedRoutingLease { + try session.cancellation.check(); + // Include all authentication and interpretation inputs, not merely a + // logical metric name (which can point at a new immutable publication). + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + hash.update(artifact.artifact_id); + hash.update(&.{0}); + hash.update(artifact.checksum); + hash.update(&artifact.graph_metric_routing_checksum); + hash.update(&artifact.graph_metric_point_index_checksum); + hash.update(&.{ @intFromBool(ranked_only), @intFromBool(directory != null) }); + if (directory) |info| hash.update(&info.checksum); + var dimensions: [42]u8 = undefined; + std.mem.writeInt(u64, dimensions[0..8], artifact.byte_len, .little); + std.mem.writeInt(u64, dimensions[8..16], offset, .little); + std.mem.writeInt(u64, dimensions[16..24], len, .little); + std.mem.writeInt(u16, dimensions[24..26], version, .little); + std.mem.writeInt(u64, dimensions[26..34], if (directory) |info| info.block_count else 0, .little); + std.mem.writeInt(u64, dimensions[34..42], if (directory) |info| info.footer_offset else 0, .little); + hash.update(&dimensions); + var key: [32]u8 = undefined; + hash.final(&key); + var fill: ?usize = null; + defer if (fill) |index| session.cache.?.graph_metric_routing.finish(index, session.io); + if (session.cache) |cache| { + while (true) { + try session.cancellation.check(); + switch (cache.graph_metric_routing.begin(key)) { + .hit => |cached| { + var lease = try OwnedRoutingLease.admit(session, cached); + errdefer lease.deinit(); + try session.chargeGraphMetricDecode(0, 1); + return lease; + }, + .fill => |index| { + fill = index; + break; + }, + .wait => |registered| { + var waiter = registered; + defer waiter.deinit(); + if (try waiter.awaitResult(session.io, session.cancellation)) |shared| { + var lease = try OwnedRoutingLease.admit(session, shared); + errdefer lease.deinit(); + try session.chargeGraphMetricDecode(0, 1); + return lease; + } + }, + .saturated => |epoch| { + try cache.graph_metric_routing.awaitFill(session.io, session.cancellation, epoch); + }, + } + } + } + // Reserve transient decode memory before fetching/duplicating the footer. + // The bounded top tier contributes at most 40 ranked routing entries. + const routing_bytes = std.math.mul(usize, decode_work, @sizeOf(metric_segment.codec.RoutingEntry)) catch return error.GraphMetricQueryBudgetExceeded; + const overhead = @sizeOf(routing_cache.Entry) + + (metric_segment.codec.max_persisted_top_entries / metric_segment.codec.ranked_score_block_entries + 1) * @sizeOf(metric_segment.codec.RankedRoutingEntry); + var decode_memory = try session.reserveGraphMetricMemory(std.math.add(usize, routing_bytes, overhead) catch return error.GraphMetricQueryBudgetExceeded); + defer decode_memory.deinit(); + try session.chargeGraphMetricDecode(1, decode_work); + var footer_range = if (directory) |info| blk: { + break :blk try fetchMetadataRangeAlloc(session, metric_index, &.{.{ .first_node_id = "", .block_index = 3, .offset = offset, .len = len, .checksum = info.checksum }}); + } else try fetchRoutingFooterAlloc(session, metric_index, artifact, version, offset, len, root_len); + defer footer_range.memory.deinit(); + const footer = footer_range.bytes; + var owns_footer = true; + defer if (owns_footer) session.alloc.free(footer); + const owner = if (session.cache) |cache| cache.alloc else session.alloc; + const same_allocator = owner.ptr == session.alloc.ptr and owner.vtable == session.alloc.vtable; + var copy_memory = try session.reserveGraphMetricMemory(if (same_allocator) 0 else len); + defer copy_memory.deinit(); + const owned = if (owner.ptr == session.alloc.ptr and owner.vtable == session.alloc.vtable) blk: { + owns_footer = false; + break :blk footer; + } else try owner.dupe(u8, footer); + errdefer owner.free(owned); + // The control's count is the admitted decode/memory shape. Check the + // authenticated footer count before its decoder allocates routing entries. + if (!ranked_only and directory == null and + (owned.len < 8 or std.mem.readInt(u32, owned[4..8], .little) != decode_work)) return error.InvalidGraphMetricSegment; + var routing = if (directory) |info| blk: { + const entries = try metric_segment.codec.decodePointDirectoryAlloc(owner, owned, offset, info.footer_offset, info.block_count, session.cancellation); + errdefer owner.free(entries); + break :blk metric_segment.codec.RoutingIndex{ + .entries = entries, + .ranked_entries = try owner.alloc(metric_segment.codec.RankedRoutingEntry, 0), + .top_score_count = 0, + .footer_offset = info.footer_offset, + .point_index_checksum = artifact.graph_metric_point_index_checksum, + }; + } else if (ranked_only) + try metric_segment.codec.decodeRoutingRootAlloc(owner, owned, artifact.byte_len, version, session.cancellation) + else + try metric_segment.decodeRoutingIndexForVersionWithCancellationAlloc(owner, owned, artifact.byte_len, version, session.cancellation); + errdefer routing.deinit(owner); + if (!std.mem.eql(u8, &routing.point_index_checksum, &artifact.graph_metric_point_index_checksum)) return error.InvalidGraphMetricSegment; + const entry = try owner.create(routing_cache.Entry); + errdefer owner.destroy(entry); + entry.* = .{ .key = key, .alloc = owner, .footer = owned, .routing = routing }; + try session.cancellation.check(); + // Retain exactly the immutable lease footprint. Admission never drops + // between the construction owner and the returned query owner. + decode_memory.absorb(&footer_range.memory); + decode_memory.shrinkTo(entry.bytes()); + const memory = decode_memory; + decode_memory = .{}; + const lease = if (session.cache) |cache| + cache.graph_metric_routing.publish(fill.?, entry, cache.cfg.max_graph_metric_routing_bytes) + else + routing_cache.Lease{ .entry = entry }; + return .{ .lease = lease, .entry = lease.entry, .memory = memory }; +} + +const MetricRangeKind = enum { score, reserved_score, routing, ranked, metadata }; + +fn metricBlockId(buf: []u8, kind: MetricRangeKind, block_index: usize) ![]const u8 { + return std.fmt.bufPrint(buf, "graph-metric-{s}-{d}-exact", .{ switch (kind) { + .routing => "routing", + .metadata => "metadata", + .ranked => "ranked", + .score, .reserved_score => "score", + }, block_index }); +} + +fn fetchMetricRangeAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + metric_index: usize, + segment_version: u16, + entries: []const metric_segment.codec.RoutingEntry, + range: ScoreFetchRange, + kind: MetricRangeKind, +) !OwnedMetricRange { + const memory_bytes = try transportMemoryBytes(range.len, range.last_block -| range.first_block + 1); + var memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}; + if (session.graph_metric_transport_credit == 0) { + memory = try session.reserveGraphMetricMemory(memory_bytes); + } else if (memory_bytes > session.graph_metric_transport_credit) return error.GraphMetricQueryBudgetExceeded; + errdefer memory.deinit(); + return .{ .bytes = try fetchMetricRangeBytesAlloc(alloc, session, metric_index, segment_version, entries, range, kind), .memory = memory }; +} + +fn fetchMetricRangeBytesAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + metric_index: usize, + segment_version: u16, + entries: []const metric_segment.codec.RoutingEntry, + range: ScoreFetchRange, + kind: MetricRangeKind, +) ![]u8 { + if (kind != .reserved_score) try session.chargeGraphMetricRange(range.len); + if (segment_version != metric_segment.wire_version) return error.InvalidGraphMetricSegment; + if (range.len == 0 or range.len > coalesced_score_window_bytes) return error.InvalidGraphMetricSegment; + if (range.first_block > range.last_block or range.last_block >= entries.len) return error.InvalidGraphMetricSegment; + const selected = entries[range.first_block .. range.last_block + 1]; + var covered: u64 = range.offset; + for (selected) |entry| { + if (entry.offset != covered or entry.len == 0) return error.InvalidGraphMetricSegment; + covered = std.math.add(u64, covered, entry.len) catch return error.InvalidGraphMetricSegment; + } + if (covered - range.offset != range.len) return error.InvalidGraphMetricSegment; + const cache = session.cache orelse return fetchCanonicalRunAlloc(alloc, session, metric_index, selected); + const fills = @import("authenticated_block_fills.zig"); + const specs = try alloc.alloc(fills.Cache.Spec, selected.len); + defer alloc.free(specs); + const artifact = session.artifactRef(metric_index) orelse return error.ArtifactNotFound; + for (selected, specs) |entry, *spec| { + spec.* = .{ .key = fills.blockKey(artifact.artifact_id, artifact.checksum, entry.offset, entry.len, &entry.checksum), .len = entry.len }; + } + var batch = try cache.graph_metric_blocks.acquire(cache.alloc, alloc, specs, session.io, session.cancellation); + defer batch.deinit(); + const missing = try alloc.alloc(bool, selected.len); + defer alloc.free(missing); + @memset(missing, false); + for (selected, batch.items, missing) |entry, item, *miss| { + try session.checkCancellation(); + if (!item.producer) continue; + var id_buf: [64]u8 = undefined; + const id = try metricBlockId(&id_buf, kind, entry.block_index); + if (try session.readCachedAuthenticatedBlockAlloc(alloc, metric_index, id, entry.offset, entry.len, &entry.checksum)) |bytes| { + defer alloc.free(bytes); + @memcpy(item.buffer(), bytes); + } else miss.* = true; + } + var runs: usize = 0; + var prior_missing = false; + for (missing) |miss| { + if (miss and !prior_missing) runs += 1; + prior_missing = miss; + } + // Reuse newly shared hits without weakening the already-reserved budget. + // If splitting needs unavailable requests, retain the admitted full range. + var full_range = false; + if (runs > 1) { + session.reserveGraphMetricRanges(runs - 1, 0) catch |err| switch (err) { + error.GraphMetricQueryBudgetExceeded => full_range = true, + else => return err, + }; + } + var start: usize = 0; + while (start < selected.len) { + if (!full_range and !missing[start]) { + start += 1; + continue; + } + var end = if (full_range) selected.len else start + 1; + if (!full_range) while (end < selected.len and missing[end]) : (end += 1) {}; + const bytes = try fetchCanonicalRunAlloc(alloc, session, metric_index, selected[start..end]); + defer session.alloc.free(bytes); + for (selected[start..end], batch.items[start..end]) |entry, item| { + if (item.producer) { + const offset: usize = @intCast(entry.offset - selected[start].offset); + @memcpy(item.buffer(), bytes[offset..][0..entry.len]); + } + } + start = end; + } + // Fetch temporaries have retired before allocating the contiguous result. + const output = try session.alloc.alloc(u8, range.len); + errdefer session.alloc.free(output); + // All newly produced units are authenticated before waiters see them. + try session.checkCancellation(); + batch.publish(session.io); + const limit = runtime_mod.max_authenticated_publication_blocks; + var publications: [limit]runtime_mod.AuthenticatedBlockPublication = undefined; + var ids: [limit][64]u8 = undefined; + var count: usize = 0; + for (selected, batch.items, missing) |entry, item, miss| { + const offset: usize = @intCast(entry.offset - range.offset); + @memcpy(output[offset..][0..entry.len], item.bytes()); + if (!miss) continue; + publications[count] = .{ + .block_id = try metricBlockId(&ids[count], kind, entry.block_index), + .offset = entry.offset, + .contents = item.bytes(), + .checksum = entry.checksum, + }; + count += 1; + if (count == limit) { + try session.cacheAuthenticatedBlocks(metric_index, publications[0..count]); + count = 0; + } + } + if (count != 0) try session.cacheAuthenticatedBlocks(metric_index, publications[0..count]); + try session.checkCancellation(); + return output; +} + +fn fetchCanonicalRunAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, metric_index: usize, entries: []const metric_segment.codec.RoutingEntry) ![]u8 { + const subranges = try alloc.alloc(runtime_mod.AuthenticatedSubrange, entries.len); + defer alloc.free(subranges); + var len: usize = 0; + for (entries, subranges) |entry, *subrange| { + subrange.* = .{ .relative_offset = len, .len = entry.len, .checksum = entry.checksum }; + len = std.math.add(usize, len, entry.len) catch return error.InvalidGraphMetricSegment; + } + return session.fetchArtifactAuthenticatedRangeUncachedAlloc(metric_index, entries[0].offset, len, subranges) catch |err| switch (err) { + error.ArtifactIntegrityMismatch => error.InvalidGraphMetricSegment, + else => |other| other, + }; +} + +const FetchedMetricRanges = struct { + payloads: [][]u8, + memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}, + + fn deinit(self: *@This(), alloc: Allocator) void { + for (self.payloads) |payload| if (payload.len > 0) std.heap.smp_allocator.free(payload); + alloc.free(self.payloads); + self.memory.deinit(); + } +}; + +fn fetchMetricRangeWorker( + child: *runtime_mod.QuerySession, + metric_index: usize, + segment_version: u16, + entries: []const metric_segment.codec.RoutingEntry, + range: ScoreFetchRange, + output: *[]u8, + failure: *?anyerror, + kind: MetricRangeKind, +) void { + const fetched = fetchMetricRangeAlloc( + std.heap.smp_allocator, + child, + metric_index, + segment_version, + entries, + range, + kind, + ) catch |err| { + failure.* = err; + return; + }; + // The joined batch owns the reservation for every child output. + std.debug.assert(fetched.memory.budget == null); + output.* = fetched.bytes; +} + +/// Fetch independent immutable routing or score ranges with bounded fanout. +/// Every child borrows the pinned manifest and charges the same synchronized +/// request budget, so parallelism changes latency without weakening admission. +fn fetchMetricRangeBatchAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + metric_index: usize, + segment_version: u16, + entries: []const metric_segment.codec.RoutingEntry, + ranges: []const ScoreFetchRange, + kind: MetricRangeKind, +) !FetchedMetricRanges { + if (ranges.len > max_parallel_metric_range_requests) + return error.GraphMetricQueryBudgetExceeded; + var requested_bytes: usize = 0; + for (ranges) |range| { + requested_bytes = std.math.add(usize, requested_bytes, range.len) catch + return error.GraphMetricQueryBudgetExceeded; + } + if (requested_bytes > max_parallel_metric_range_bytes) + return error.GraphMetricQueryBudgetExceeded; + var memory_bytes: usize = 0; + for (ranges) |range| memory_bytes = std.math.add(usize, memory_bytes, try transportMemoryBytes(range.len, range.last_block -| range.first_block + 1)) catch return error.GraphMetricQueryBudgetExceeded; + var memory: runtime_mod.GraphMetricReadBudget.Reservation = .{}; + if (session.graph_metric_transport_credit == 0) { + memory = try session.reserveGraphMetricMemory(memory_bytes); + } else if (memory_bytes > session.graph_metric_transport_credit) return error.GraphMetricQueryBudgetExceeded; + errdefer memory.deinit(); + const payloads = try alloc.alloc([]u8, ranges.len); + @memset(payloads, @constCast((&[_]u8{})[0..])); + errdefer { + for (payloads) |payload| if (payload.len > 0) std.heap.smp_allocator.free(payload); + alloc.free(payloads); + } + + var children: [max_parallel_metric_range_requests]runtime_mod.QuerySession = undefined; + var failures: [max_parallel_metric_range_requests]?anyerror = @splat(null); + if (session.io) |io| { + var group: std.Io.Group = .init; + for (ranges, 0..) |range, index| { + children[index] = session.forkGraphMetricRead(std.heap.smp_allocator); + children[index].graph_metric_transport_credit = transportMemoryBytes(range.len, range.last_block -| range.first_block + 1) catch unreachable; + group.async(io, fetchMetricRangeWorker, .{ + &children[index], metric_index, segment_version, entries, range, &payloads[index], &failures[index], kind, + }); + } + const await_result = group.await(io); + for (children[0..ranges.len]) |*child| child.deinit(); + try await_result; + } else { + for (ranges, 0..) |range, index| { + children[index] = session.forkGraphMetricRead(std.heap.smp_allocator); + children[index].graph_metric_transport_credit = transportMemoryBytes(range.len, range.last_block -| range.first_block + 1) catch unreachable; + fetchMetricRangeWorker( + &children[index], + metric_index, + segment_version, + entries, + range, + &payloads[index], + &failures[index], + kind, + ); + } + for (children[0..ranges.len]) |*child| child.deinit(); + } + for (failures[0..ranges.len]) |failure| if (failure) |err| return err; + return .{ .payloads = payloads, .memory = memory }; +} + +fn fetchRankedScoreBlocksAlloc( + alloc: Allocator, + session: *runtime_mod.QuerySession, + metric_index: usize, + entries: []const metric_segment.codec.RankedRoutingEntry, + first_block: usize, +) !OwnedMetricRange { + if (entries.len == 0) return .{ .bytes = try session.alloc.alloc(u8, 0) }; + const canonical = try alloc.alloc(metric_segment.codec.RoutingEntry, entries.len); + defer alloc.free(canonical); + var len: usize = 0; + for (entries, canonical, 0..) |entry, *block, i| { + if (entry.len == 0 or entry.len > metric_segment.codec.max_ranked_score_block_bytes) return error.InvalidGraphMetricSegment; + block.* = .{ .first_node_id = "", .block_index = first_block + i, .offset = entry.offset, .len = entry.len, .checksum = entry.checksum }; + len = std.math.add(usize, len, entry.len) catch return error.InvalidGraphMetricSegment; + } + return fetchMetricRangeAlloc(alloc, session, metric_index, metric_segment.wire_version, canonical, .{ + .first_block = 0, + .last_block = canonical.len - 1, + .offset = canonical[0].offset, + .len = len, + }, .ranked); +} + +fn planAllScoreFetchRangesAlloc( + alloc: Allocator, + entries: []const metric_segment.codec.RoutingEntry, +) ![]ScoreFetchRange { + var ranges = std.ArrayListUnmanaged(ScoreFetchRange).empty; + errdefer ranges.deinit(alloc); + for (entries, 0..) |entry, block_index| { + const entry_end = std.math.add(u64, entry.offset, entry.len) catch return error.InvalidGraphMetricSegment; + if (ranges.items.len == 0) { + try ranges.append(alloc, .{ + .first_block = block_index, + .last_block = block_index, + .offset = entry.offset, + .len = entry.len, + }); + continue; + } + const range = &ranges.items[ranges.items.len - 1]; + const range_end = std.math.add(u64, range.offset, range.len) catch return error.InvalidGraphMetricSegment; + const combined_len = entry_end - range.offset; + if (entry.offset != range_end or combined_len > coalesced_score_window_bytes) { + try ranges.append(alloc, .{ + .first_block = block_index, + .last_block = block_index, + .offset = entry.offset, + .len = entry.len, + }); + } else { + range.last_block = block_index; + range.len = std.math.cast(usize, combined_len) orelse return error.GraphMetricQueryBudgetExceeded; + } + } + if (ranges.items.len > max_top_score_range_requests) return error.GraphMetricQueryBudgetExceeded; + return try ranges.toOwnedSlice(alloc); +} + +pub fn openAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, graph_index_name: []const u8, metric_name: []const u8) !metric_segment.Segment { + return try loadVerifiedAlloc(alloc, session, graph_index_name, metric_name); +} + +pub fn topAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, graph_index_name: []const u8, metric_name: []const u8, top_k: usize) !Result { + return try topWithLimitsAlloc(alloc, session, graph_index_name, metric_name, top_k, .{}); +} + +pub fn topWithLimitsAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, graph_index_name: []const u8, metric_name: []const u8, top_k: usize, limits: Limits) !Result { + if (top_k > limits.max_top_k or top_k > metric_segment.codec.max_persisted_top_entries or + limits.max_top_k == 0 or limits.max_result_bytes == 0) + { + return error.GraphMetricQueryBudgetExceeded; + } + + try session.checkCancellation(); + const specs = try session.graphMetricSpecs(); + const config = findConfig(specs, graph_index_name, metric_name) orelse return error.MetricNotConfigured; + const graph_index = session.findNamedArtifactIndex(.graph_segment, graph_index_name) orelse return error.GraphSegmentNotFound; + const graph_artifact = session.artifactRef(graph_index).?; + const artifact_name = try metric_segment.artifactNameAlloc(alloc, graph_index_name, metric_name); + defer alloc.free(artifact_name); + const metric_index = session.findNamedArtifactIndex(.graph_metric_segment, artifact_name) orelse return error.MetricNotReady; + const metric_artifact = session.artifactRef(metric_index) orelse return error.InvalidGraphMetricSegment; + const control_len = metric_artifact.graph_metric_control_len; + var control_range = try fetchControlAlloc(session, metric_index, metric_artifact, control_len); + defer control_range.deinit(session.alloc); + const control = try metric_segment.decodeControl(control_range.bytes, config.edge_filter); + try validateControl(control.header, graph_artifact, metric_artifact, config); + if (control.header.materialization_state == .rejected) { + recordRejectionDiagnostic(session, graph_index_name, metric_name, control.header.materializer_fingerprint); + return error.GraphMetricMaterializationRejected; + } + // A zero-cardinality request still authenticates control/provenance but + // never pays for a potentially large routing footer. + if (top_k == 0 or control.score_count == 0) { + const scores = try alloc.alloc(Score, 0); + errdefer alloc.free(scores); + var edge_filter = try config.edge_filter.cloneAlloc(alloc); + errdefer edge_filter.deinit(alloc); + return .{ + .scores = scores, + .config_fingerprint = control.header.config_fingerprint, + .converged = control.header.converged, + .iterations_completed = control.header.iterations_completed, + .delta = control.header.delta, + .edge_filter = edge_filter, + .metadata_version = control.header.version, + .published_generation = if (metric_artifact.published_generation != 0) metric_artifact.published_generation else session.manifest.version, + .edge_generation = if (metric_artifact.edge_generation != 0) metric_artifact.edge_generation else session.manifest.version, + .computed_at_ms = if (metric_artifact.computed_at_ms != 0) metric_artifact.computed_at_ms else @divTrunc(session.manifest.built_at_ns, std.time.ns_per_ms), + }; + } + // Wire-v8 has an independently authenticated routing root. Cold top-K + // never fetches, decodes or retains the primary point-lookup index. + const footer_len = try routingFooterLen(metric_artifact, control.header.version); + const footer_offset = metric_artifact.byte_len - footer_len; + const expected_top_count = @min(@as(usize, control.score_count), metric_segment.codec.max_persisted_top_entries); + const expected_ranked_blocks = expected_top_count / metric_segment.codec.ranked_score_block_entries + + @intFromBool(expected_top_count % metric_segment.codec.ranked_score_block_entries != 0); + const root_len = metric_segment.codec.routingRootLen(control.score_count); + if (root_len > footer_len) return error.InvalidGraphMetricSegment; + var routing_lease = try acquireRouting(session, metric_index, metric_artifact, control.header.version, metric_artifact.byte_len - root_len, root_len, expected_ranked_blocks, root_len, true, null); + defer routing_lease.deinit(); + const routing = routing_lease.entry.routing; + if (routing.entries.len != 0 or routing.footer_offset != footer_offset or routing.primary_data_offset != control.score_data_offset) { + return error.InvalidGraphMetricSegment; + } + + if (routing.top_score_count != expected_top_count or routing.ranked_entries.len != expected_ranked_blocks) { + return error.InvalidGraphMetricSegment; + } + const result_count = @min(top_k, @as(usize, control.score_count)); + if (result_count > expected_top_count) return error.InvalidGraphMetricSegment; + { + const required_blocks = result_count / metric_segment.codec.ranked_score_block_entries + + @intFromBool(result_count % metric_segment.codec.ranked_score_block_entries != 0); + var result_budget = try TopResultBudget.init(session, result_count, limits.max_result_bytes); + const scores = try alloc.alloc(Score, result_count); + var initialized: usize = 0; + errdefer { + for (scores[0..initialized]) |*score| score.deinit(alloc); + alloc.free(scores); + } + var block_cursor: usize = 0; + var boundary_validator = RankedScoreBoundaryValidator{}; + while (block_cursor < required_blocks) { + var range_end = block_cursor; + var range_bytes: usize = 0; + while (range_end < required_blocks) : (range_end += 1) { + const entry_len = routing.ranked_entries[range_end].len; + if (range_bytes > 0 and (range_bytes >= ranked_fetch_window_bytes or entry_len > ranked_fetch_window_bytes - range_bytes)) break; + range_bytes = std.math.add(usize, range_bytes, entry_len) catch return error.GraphMetricQueryBudgetExceeded; + } + const range_entries = routing.ranked_entries[block_cursor..range_end]; + var fetched = try fetchRankedScoreBlocksAlloc(alloc, session, metric_index, range_entries, block_cursor); + defer fetched.deinit(session.alloc); + const ranked_payload = fetched.bytes; + for (range_entries) |entry| { + try session.checkCancellation(); + const relative_offset = std.math.cast(usize, entry.offset -| range_entries[0].offset) orelse return error.InvalidGraphMetricSegment; + if (relative_offset > ranked_payload.len or entry.len > ranked_payload.len - relative_offset) return error.InvalidGraphMetricSegment; + const decoded = try metric_segment.codec.decodeRankedScoreBlockWithCancellation( + ranked_payload[relative_offset..][0..entry.len], + session.cancellation, + ); + const expected_block_count = @min( + metric_segment.codec.ranked_score_block_entries, + expected_top_count - initialized, + ); + try session.chargeGraphMetricDecode(1, expected_block_count); + if (decoded.len != expected_block_count) return error.InvalidGraphMetricSegment; + try boundary_validator.observeBlock(decoded); + for (decoded.scores[0..@min(decoded.len, result_count - initialized)]) |score| { + try result_budget.addNode(score.nodeIdLen(decoded.node_prefix)); + scores[initialized] = .{ .node_id = try score.dupeNodeAlloc(alloc, decoded.node_prefix), .value = score.value }; + initialized += 1; + } + } + block_cursor = range_end; + } + if (initialized != result_count) return error.InvalidGraphMetricSegment; + var edge_filter = try config.edge_filter.cloneAlloc(alloc); + errdefer edge_filter.deinit(alloc); + return .{ + .scores = scores, + .config_fingerprint = control.header.config_fingerprint, + .converged = control.header.converged, + .iterations_completed = control.header.iterations_completed, + .delta = control.header.delta, + .edge_filter = edge_filter, + .metadata_version = control.header.version, + .published_generation = if (metric_artifact.published_generation != 0) metric_artifact.published_generation else session.manifest.version, + .edge_generation = if (metric_artifact.edge_generation != 0) metric_artifact.edge_generation else session.manifest.version, + .computed_at_ms = if (metric_artifact.computed_at_ms != 0) metric_artifact.computed_at_ms else @divTrunc(session.manifest.built_at_ns, std.time.ns_per_ms), + }; + } +} + +fn findConfig( + specs: []const graph_metric_config.IndexSpec, + graph_index_name: []const u8, + metric_name: []const u8, +) ?graph_mod.GraphMetricConfig { + for (specs) |spec| { + if (!std.mem.eql(u8, spec.index_name, graph_index_name)) continue; + for (spec.configs) |config| if (std.mem.eql(u8, config.name, metric_name)) return config; + return null; + } + return null; +} + +fn validateControl( + header: metric_segment.codec.Header, + graph_artifact: anytype, + metric_artifact: manifest_mod.ArtifactRef, + config: graph_mod.GraphMetricConfig, +) !void { + if (header.kind != config.kind or header.config_fingerprint != lake_graph_metric.configFingerprint(config)) { + return error.MetricStale; + } + if (!lake_graph_metric.metricSourceMatches(header, graph_artifact, metric_artifact)) return error.MetricStale; + if (header.materializer_fingerprint != graph_metric_policy.materializerFingerprint(.{})) { + return error.GraphMetricPolicyStale; + } +} + +fn loadVerifiedAlloc(alloc: Allocator, session: *runtime_mod.QuerySession, graph_index_name: []const u8, metric_name: []const u8) !metric_segment.Segment { + try session.checkCancellation(); + const specs = try session.graphMetricSpecs(); + const config = findConfig(specs, graph_index_name, metric_name) orelse return error.MetricNotConfigured; + const graph_index = session.findNamedArtifactIndex(.graph_segment, graph_index_name) orelse return error.GraphSegmentNotFound; + const graph_artifact = session.artifactRef(graph_index).?; + const name = try metric_segment.artifactNameAlloc(alloc, graph_index_name, metric_name); + defer alloc.free(name); + const metric_index = session.findNamedArtifactIndex(.graph_metric_segment, name) orelse return error.MetricNotReady; + const metric_artifact = session.artifactRef(metric_index) orelse return error.InvalidGraphMetricSegment; + const artifact_len = std.math.cast(usize, metric_artifact.byte_len) orelse return error.GraphMetricQueryBudgetExceeded; + try session.chargeGraphMetricRange(artifact_len); + try session.chargeGraphMetricDecode(1, std.math.divCeil(usize, artifact_len, 12) catch return error.GraphMetricQueryBudgetExceeded); + const retained_bytes = std.math.mul(usize, artifact_len, 2) catch return error.GraphMetricQueryBudgetExceeded; + try session.chargeGraphMetricRetained(retained_bytes); + const payload = try session.fetchArtifactAlloc(metric_index); + defer session.alloc.free(payload); + var segment = try metric_segment.decodeAllocWithCancellation(alloc, payload, session.cancellation); + errdefer segment.deinit(alloc); + // The authenticated payload owns its schema version. Manifest provenance + // is optional and must not override the decoded wire contract. + segment.published_generation = if (metric_artifact.published_generation != 0) metric_artifact.published_generation else session.manifest.version; + segment.edge_generation = if (metric_artifact.edge_generation != 0) metric_artifact.edge_generation else session.manifest.version; + segment.computed_at_ms = if (metric_artifact.computed_at_ms != 0) metric_artifact.computed_at_ms else @divTrunc(session.manifest.built_at_ns, std.time.ns_per_ms); + if (segment.kind != config.kind or segment.config_fingerprint != lake_graph_metric.configFingerprint(config) or + !segment.edge_filter.equivalent(config.edge_filter)) return error.MetricStale; + if (!lake_graph_metric.metricSourceMatches(segment, graph_artifact, metric_artifact)) return error.MetricStale; + if (segment.materializer_fingerprint != graph_metric_policy.materializerFingerprint(.{})) return error.GraphMetricPolicyStale; + if (segment.materialization_state == .rejected) return switch (segment.rejection_reason) { + .build_budget_exceeded => { + recordRejectionDiagnostic(session, graph_index_name, metric_name, segment.materializer_fingerprint); + return error.GraphMetricMaterializationRejected; + }, + .none => error.InvalidGraphMetricSegment, + }; + return segment; +} + +test "serverless graph metric column reads bound shape and accept empty dependencies" { + const alloc = std.testing.allocator; + var unused_session: runtime_mod.QuerySession = undefined; + var empty = try scoreColumnsAlloc(alloc, &unused_session, "graph_idx", &.{}, &.{}); + defer empty.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), empty.columns.len); + + const too_many: [max_point_score_columns + 1][]const u8 = @splat("rank"); + try std.testing.expectError( + error.GraphMetricQueryBudgetExceeded, + scoreColumnsAlloc(alloc, &unused_session, "graph_idx", &too_many, &.{}), + ); +} + +test "serverless graph metric point admission precedes output and candidate allocations" { + const alloc = std.testing.allocator; + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + session.graph_metric_read_budget.limits.max_retained_bytes = 0; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, scoresAlloc(failing.allocator(), &session, "g", "m", &.{"a"})); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, scoreColumnsAlloc(failing.allocator(), &session, "g", &.{ "m", "n" }, &.{"a"})); + const oversized = try alloc.alloc([]const u8, (Limits{}).max_point_scores + 1); + defer alloc.free(oversized); + session.graph_metric_read_budget = .{}; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, scoresAlloc(failing.allocator(), &session, "g", "m", oversized)); + try std.testing.expectError(error.InvalidGraphMetricNodeId, scoresAlloc(failing.allocator(), &session, "g", "m", &.{""})); + try std.testing.expectError(error.InvalidGraphMetricNodeId, scoreColumnsAlloc(failing.allocator(), &session, "g", &.{ "m", "n" }, &.{""})); + try std.testing.expectEqual(@as(u64, 0), session.graph_metric_read_budget.retained_bytes); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); +} + +test "serverless graph metric candidate order shares duplicate rows across sparse block spans" { + const alloc = std.testing.allocator; + const ids: []const []const u8 = &.{ "z", "a", "m", "a", "0", "x", "n" }; + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + const order = try candidateOrderAlloc(alloc, &session, ids); + defer alloc.free(order); + try std.testing.expectEqualSlices(u32, &.{ 4, 1, 3, 2, 6, 5, 0 }, order); + var entries = [_]metric_segment.codec.RoutingEntry{ + .{ .first_node_id = "a", .offset = 0, .len = 10 }, + .{ .first_node_id = "m", .offset = 10, .len = 10 }, + .{ .first_node_id = "z", .offset = 20, .len = 10 }, + }; + var blocks = CandidateBlocks{ .node_ids = ids, .order = order, .routing = .{ .entries = &entries, .footer_offset = 0, .ranked_entries = &.{}, .top_score_count = 0 } }; + try std.testing.expectEqual(TouchedBlock{ .block_index = 0, .first_pending = 1, .pending_count = 2 }, blocks.next().?); + try std.testing.expectEqual(TouchedBlock{ .block_index = 1, .first_pending = 3, .pending_count = 3 }, blocks.next().?); + try std.testing.expectEqual(TouchedBlock{ .block_index = 2, .first_pending = 6, .pending_count = 1 }, blocks.next().?); + try std.testing.expect(blocks.next() == null); + session.graph_metric_read_budget = .{ .limits = .{ .max_retained_bytes = ids.len * (@sizeOf(u32) + @sizeOf(u64)) - 1 } }; + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, candidateOrderAlloc(failing.allocator(), &session, ids)); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); + session.graph_metric_read_budget = .{ .limits = .{ .max_retained_bytes = 1 } }; + const touched = [_]TouchedBlock{.{ .block_index = 0, .first_pending = 0, .pending_count = 1 }}; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planSparseScoreFetchRangesWithBudgetAlloc(failing.allocator(), &entries, &touched, 0, 128, .{ .session = &session })); +} + +test "serverless graph metric candidate spans match per-row routing with prefixes and absent IDs" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const ids = try arena.allocator().alloc([]const u8, 1024); + for (ids, 0..) |*id, i| id.* = try std.fmt.allocPrint(arena.allocator(), "prefix/{d:0>4}", .{(i * 7919) % 713}); + var entries = [_]metric_segment.codec.RoutingEntry{ + .{ .first_node_id = "prefix/0001", .offset = 0, .len = 10 }, + .{ .first_node_id = "prefix/0234", .offset = 10, .len = 10 }, + .{ .first_node_id = "prefix/0555", .offset = 20, .len = 10 }, + }; + const routing = metric_segment.codec.RoutingIndex{ .entries = &entries, .footer_offset = 0, .ranked_entries = &.{}, .top_score_count = 0 }; + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + const order = try candidateOrderAlloc(alloc, &session, ids); + defer alloc.free(order); + for (order[1..], order[0 .. order.len - 1]) |row, previous| { + const compared = std.mem.order(u8, ids[previous], ids[row]); + try std.testing.expect(compared == .lt or (compared == .eq and previous < row)); + } + var mapped: [1024]?usize = @splat(null); + var blocks = CandidateBlocks{ .node_ids = ids, .order = order, .routing = routing }; + while (blocks.next()) |block| for (order[block.first_pending..][0..block.pending_count]) |row| { + try std.testing.expect(mapped[row] == null); + mapped[row] = block.block_index; + }; + for (ids, mapped) |id, actual| try std.testing.expectEqual(routing.findIndex(id), actual); +} + +test "serverless graph metric candidate prefix keys preserve binary ties and allocation failure ownership" { + const Runner = struct { + fn run(alloc: Allocator) !void { + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + const cases = [_][]const []const u8{ + &.{ "sameabcdefghz", "sameabcdefgha", "sameabcdefgha\x00", "sameabcdefgh", "sameabcdefgha", "sameabcdefgi" }, + &.{ "\x00", "\x00\x00", "\x00a", "a", "a\x00", "a\x00\x00", "\xff", "abcdefghz", "abcdefgha", "abcdefgh" }, + }; + for (cases) |ids| { + var rows: [257][]const u8 = undefined; + for (&rows, 0..) |*id, i| id.* = ids[(i * 7) % ids.len]; + const order = try candidateOrderAlloc(alloc, &session, &rows); + defer alloc.free(order); + var seen: [257]bool = @splat(false); + for (order) |row| { + try std.testing.expect(!seen[row]); + seen[row] = true; + } + for (order[1..], order[0 .. order.len - 1]) |row, previous| { + const compared = std.mem.order(u8, rows[previous], rows[row]); + try std.testing.expect(compared == .lt or (compared == .eq and previous < row)); + } + } + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "serverless graph metric transport admission bounds cache copies and adapts batch width" { + const alloc = std.testing.allocator; + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + const range = ScoreFetchRange{ .first_block = 0, .last_block = 0, .offset = 0, .len = 4096 }; + const ranges = [_]ScoreFetchRange{range} ** 8; + const peak = try transportMemoryBytes(range.len, 1); + session.graph_metric_read_budget.limits.max_retained_bytes = peak; + try std.testing.expectEqual(@as(usize, 1), try admittedMetricRangeBatchEnd(&session, &ranges, 0)); + var owned = OwnedMetricRange{ .bytes = try alloc.alloc(u8, range.len), .memory = try session.reserveGraphMetricMemory(peak) }; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, admittedMetricRangeBatchEnd(&session, &ranges, 0)); + // Admission must fail before touching artifact metadata or issuing I/O. + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, fetchMetricRangeAlloc(alloc, &session, 0, metric_segment.wire_version, &.{}, range, .score)); + owned.deinit(alloc); + try std.testing.expectEqual(@as(u64, 0), session.graph_metric_read_budget.retained_bytes); + try std.testing.expectEqual(@as(usize, 1), try admittedMetricRangeBatchEnd(&session, &ranges, 0)); + session.graph_metric_read_budget.limits.max_retained_bytes = 2 * peak; + try std.testing.expectEqual(@as(usize, 2), try admittedMetricRangeBatchEnd(&session, &ranges, 0)); + // Children consume only their pre-reserved share, not siblings' capacity. + session.graph_metric_transport_credit = peak; + try std.testing.expectEqual(@as(usize, 1), try admittedMetricRangeBatchEnd(&session, &ranges, 0)); +} + +test "serverless graph metric top result admission bounds descriptors and incremental node ownership" { + var session = runtime_mod.QuerySession{ .alloc = std.testing.allocator, .artifacts = undefined, .manifest = undefined }; + const base = 2 * @sizeOf(Score); + session.graph_metric_read_budget = .{ .limits = .{ .max_retained_bytes = base - 1 } }; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, TopResultBudget.init(&session, 2, 1024)); + try std.testing.expectEqual(@as(u64, 0), session.graph_metric_read_budget.retained_bytes); + session.graph_metric_read_budget = .{ .limits = .{ .max_retained_bytes = base + 10 } }; + var budget = try TopResultBudget.init(&session, 2, base + 20); + try budget.addNode(6); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.addNode(5)); + try std.testing.expectEqual(base + 6, budget.bytes); + try std.testing.expectEqual(@as(u64, base + 6), session.graph_metric_read_budget.retained_bytes); + try budget.addNode(4); + session.graph_metric_read_budget = .{}; + budget = try TopResultBudget.init(&session, 2, base + 2); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.addNode(3)); + try std.testing.expectEqual(@as(u64, base), session.graph_metric_read_budget.retained_bytes); +} + +test "serverless graph metric top score transfer preserves ownership across allocation failures" { + const Runner = struct { + fn run(alloc: Allocator) !void { + const scores = try alloc.alloc(Score, 1); + const node = alloc.dupe(u8, "owned-node") catch |err| { + alloc.free(scores); + return err; + }; + scores[0] = .{ .node_id = node, .value = 0.5 }; + var result = Result{ .scores = scores, .config_fingerprint = 1, .converged = true, .iterations_completed = 1, .delta = 0, .edge_filter = .{}, .metadata_version = metric_segment.wire_version, .published_generation = 1, .edge_generation = 1, .computed_at_ms = 1 }; + defer result.deinit(alloc); + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + session.graph_metric_read_budget.limits.max_retained_bytes = 0; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, result.takePublicScoresAlloc(alloc, &session)); + try std.testing.expect(result.owns_scores); + session.graph_metric_read_budget = .{}; + const moved = try result.takePublicScoresAlloc(alloc, &session); + defer { + for (moved) |*score| score.deinit(alloc); + alloc.free(moved); + } + try std.testing.expectEqual(node.ptr, moved[0].node.ptr); + try std.testing.expectEqualStrings("owned-node", moved[0].node); + try std.testing.expectEqual(@as(f64, 0.5), moved[0].score); + try std.testing.expectError(error.GraphMetricScoresAlreadyTaken, result.takePublicScoresAlloc(alloc, &session)); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, Runner.run, .{}); +} + +test "serverless graph metric top limit cannot exceed the persisted ranked tier" { + const alloc = std.testing.allocator; + var unused_session: runtime_mod.QuerySession = undefined; + const oversized = metric_segment.codec.max_persisted_top_entries + 1; + try std.testing.expectError( + error.GraphMetricQueryBudgetExceeded, + topWithLimitsAlloc(alloc, &unused_session, "graph_idx", "rank", oversized, .{ .max_top_k = oversized }), + ); +} + +test "serverless graph metric routing batches coalesce selected pages within byte bounds" { + const alloc = std.testing.allocator; + var entries: [5]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, i| entry.* = .{ .block_index = i * metric_segment.codec.routing_page_entries, .first_node_id = "", .offset = i * 1024, .len = 1024 }; + const ranges = try planRoutingPageRangesAlloc(alloc, &entries, &.{ 0, 0, 1, 3, 4 }); + defer alloc.free(ranges); + try std.testing.expectEqual(@as(usize, 2), ranges.len); + try std.testing.expectEqual(@as(usize, 2048), ranges[0].len); + try std.testing.expectEqual(@as(usize, 3), ranges[1].first_block); + try std.testing.expectError(error.InvalidGraphMetricSegment, planRoutingPageRangesAlloc(alloc, &entries, &.{ 2, 1 })); + try std.testing.expectError(error.InvalidGraphMetricSegment, planRoutingPageRangesAlloc(alloc, &entries, &.{5})); + entries[0].len = coalesced_score_window_bytes; + entries[1].offset = entries[0].len; + const bounded = try planRoutingPageRangesAlloc(alloc, &entries, &.{ 0, 1 }); + defer alloc.free(bounded); + try std.testing.expectEqual(@as(usize, 2), bounded.len); + entries[0].offset = std.math.maxInt(u64); + try std.testing.expectError(error.InvalidGraphMetricSegment, planRoutingPageRangesAlloc(alloc, &entries, &.{0})); +} + +test "serverless graph metric joint admission gives uneven columns their actual request costs" { + const alloc = std.testing.allocator; + var heavy: [8]metric_segment.codec.RoutingEntry = undefined; + for (&heavy, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .block_index = i * 128, .offset = i * 16 * 1024 * 1024, .len = 1024 }; + const counts: [8]usize = @splat(1); + var inputs: [8]RangePlanningColumn = undefined; + inputs[0] = .{ .entries = &heavy, .counts = &counts, .score_data_offset = 0 }; + for (inputs[1..]) |*input| input.* = .{ .entries = heavy[0..1], .counts = counts[0..1], .score_data_offset = 0 }; + // Metadata/routing cost 25, scores cost 15: forty requests total. + // No column quota may reject the eight disconnected exact score ranges. + const ranges = try planScoreColumnsRangesAlloc(alloc, &inputs, 15); + defer { + for (ranges) |column| alloc.free(column); + alloc.free(ranges); + } + try std.testing.expectEqual(@as(usize, 8), ranges[0].len); + for (ranges[1..]) |column| try std.testing.expectEqual(@as(usize, 1), column.len); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsRangesAlloc(alloc, &inputs, 14)); + const Runner = struct { + fn run(a: Allocator, columns: []const RangePlanningColumn) !void { + const result = try planScoreColumnsRangesAlloc(a, columns, 15); + defer a.free(result); + for (result) |column| a.free(column); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{@as([]const RangePlanningColumn, &inputs)}); +} + +test "serverless graph metric joint coalescing spends overfetch only where it saves most" { + const alloc = std.testing.allocator; + var expensive: [4]metric_segment.codec.RoutingEntry = undefined; + var cheap: [4]metric_segment.codec.RoutingEntry = undefined; + for (&expensive, &cheap, 0..) |*large, *small, i| { + large.* = .{ .first_node_id = "node", .offset = i * 1024 * 1024, .len = 1024 * 1024 }; + small.* = .{ .first_node_id = "node", .offset = i * 1024, .len = 1024 }; + } + const inputs = [_]RangePlanningColumn{ + .{ .entries = &expensive, .counts = &.{ 1, 0, 0, 1 }, .score_data_offset = 0 }, + .{ .entries = &cheap, .counts = &.{ 1, 1, 1, 1 }, .score_data_offset = 0 }, + }; + const ranges = try planScoreColumnsRangesAlloc(alloc, &inputs, 3); + defer { + for (ranges) |column| alloc.free(column); + alloc.free(ranges); + } + try std.testing.expectEqual(@as(usize, 2), ranges[0].len); + try std.testing.expectEqual(@as(usize, 1), ranges[1].len); + try std.testing.expectEqual(@as(usize, 4096), ranges[1][0].len); + const Runner = struct { + fn run(a: Allocator, columns: []const RangePlanningColumn) !void { + const result = try planScoreColumnsRangesAlloc(a, columns, 3); + defer a.free(result); + for (result) |column| a.free(column); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{@as([]const RangePlanningColumn, &inputs)}); +} + +test "serverless graph metric joint admission finds the byte-feasible alternative to greedy coalescing" { + const alloc = std.testing.allocator; + const mib = 1024 * 1024; + var a: [8]metric_segment.codec.RoutingEntry = undefined; + var b: [4]metric_segment.codec.RoutingEntry = undefined; + for (&a, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .offset = i * mib, .len = mib }; + for (&b, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .offset = i * mib, .len = mib }; + const inputs = [_]RangePlanningColumn{ + .{ .entries = &a, .counts = &.{ 1, 1, 1, 0, 0, 0, 0, 1 }, .score_data_offset = 0 }, + .{ .entries = &b, .counts = &.{ 1, 0, 0, 1 }, .score_data_offset = 0 }, + }; + // Contiguous misses cost no overfetch: A needs two runs, as does B. + const result = try planScoreColumnsWithinBudgetAlloc(alloc, &inputs, 5, 9 * mib, .{}); + defer { + for (result) |ranges| alloc.free(ranges); + alloc.free(result); + } + try std.testing.expectEqual(@as(usize, 2), result[0].len); + try std.testing.expectEqual(@as(usize, 2), result[1].len); + try std.testing.expectEqual(@as(usize, mib), result[1][0].len); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsWithinBudgetAlloc(alloc, &inputs, 5, 6 * mib - 1, .{})); + const Runner = struct { + fn run(failing: Allocator, columns: []const RangePlanningColumn) !void { + const ranges = try planScoreColumnsWithinBudgetAlloc(failing, columns, 5, 9 * 1024 * 1024, .{}); + defer failing.free(ranges); + for (ranges) |column| failing.free(column); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{@as([]const RangePlanningColumn, &inputs)}); +} + +test "serverless graph metric bounded admission matches exhaustive range partitions" { + const alloc = std.testing.allocator; + for (0..32) |seed| { + var entries: [3][4]metric_segment.codec.RoutingEntry = undefined; + var counts: [3][4]usize = undefined; + var inputs: [3]RangePlanningColumn = undefined; + var touched: [12]struct { column: usize, offset: usize, end: usize } = undefined; + var touched_count: usize = 0; + for (&entries, &counts, &inputs, 0..) |*column, *hits, *input, i| { + const mask = (seed + i * 5) % 15 + 1; + const block_bytes = (i + 1) * 1024; + for (column, hits, 0..) |*entry, *count, j| { + entry.* = .{ .first_node_id = "node", .offset = j * block_bytes, .len = block_bytes }; + count.* = @intFromBool(mask & (@as(usize, 1) << @intCast(j)) != 0); + if (count.* != 0) { + touched[touched_count] = .{ .column = i, .offset = j * block_bytes, .end = (j + 1) * block_bytes }; + touched_count += 1; + } + } + input.* = .{ .entries = column, .counts = hits, .score_data_offset = 0 }; + } + for (1..13) |limit| { + var minimum: usize = std.math.maxInt(usize); + for (0..@as(usize, 1) << @intCast(touched_count - 1)) |mask| { + var reads: usize = 0; + var bytes: usize = 0; + var first: usize = 0; + for (touched[0..touched_count], 0..) |block, i| { + const split = i + 1 == touched_count or touched[i + 1].column != block.column or mask & (@as(usize, 1) << @intCast(i)) != 0; + if (split) { + reads += 1; + bytes += block.end - touched[first].offset; + first = i + 1; + } + } + if (reads <= limit) minimum = @min(minimum, bytes); + } + if (minimum == std.math.maxInt(usize)) { + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsRangesAlloc(alloc, &inputs, limit)); + continue; + } + const result = try planScoreColumnsWithinBudgetAlloc(alloc, &inputs, limit, minimum, .{}); + defer { + for (result) |ranges| alloc.free(ranges); + alloc.free(result); + } + var bytes: usize = 0; + var reads: usize = 0; + for (result) |ranges| for (ranges) |range| { + bytes += range.len; + reads += 1; + }; + try std.testing.expectEqual(minimum, bytes); + try std.testing.expect(reads <= limit); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsWithinBudgetAlloc(alloc, &inputs, limit, minimum - 1, .{})); + } + } +} + +test "serverless graph metric partial partitions cross old windows and split miss runs" { + const alloc = std.testing.allocator; + const mib = 1024 * 1024; + var entries: [16]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .offset = i * mib, .len = mib }; + const cases = [_]struct { counts: []const usize, reads: usize, bytes: usize }{ + .{ .counts = &.{ 1, 1, 0, 0, 0, 0, 1, 1 }, .reads = 3, .bytes = 4 * mib }, + .{ .counts = &.{ 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }, .reads = 1, .bytes = 8 * mib }, + // The middle six-block run must split across two seven-block reads. + .{ .counts = &.{ 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 }, .reads = 2, .bytes = 14 * mib }, + }; + for (cases) |case| { + const columns = [_]RangePlanningColumn{.{ .entries = entries[0..case.counts.len], .counts = case.counts, .score_data_offset = 0 }}; + const ranges = try planScoreColumnsWithinBudgetAlloc(alloc, &columns, case.reads, case.bytes, .{}); + defer { + for (ranges) |column| alloc.free(column); + alloc.free(ranges); + } + var bytes: usize = 0; + for (ranges[0]) |range| { + bytes += range.len; + try std.testing.expect(range.len <= coalesced_score_window_bytes); + } + try std.testing.expect(ranges[0].len <= case.reads); + try std.testing.expectEqual(case.bytes, bytes); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsWithinBudgetAlloc(alloc, &columns, case.reads, case.bytes - 1, .{})); + const Runner = struct { + fn run(a: Allocator, input: []const RangePlanningColumn, limit: usize, byte_limit: usize) !void { + const result = try planScoreColumnsWithinBudgetAlloc(a, input, limit, byte_limit, .{}); + defer a.free(result); + for (result) |column| a.free(column); + } + }; + try std.testing.checkAllAllocationFailures(alloc, Runner.run, .{ @as([]const RangePlanningColumn, &columns), case.reads, case.bytes }); + } +} + +test "serverless graph metric partition planning admits scratch and work before allocating" { + const alloc = std.testing.allocator; + var entries: [4]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .offset = i * 1024, .len = 1024 }; + const columns = [_]RangePlanningColumn{.{ .entries = &entries, .counts = &.{ 1, 0, 1, 0 }, .score_data_offset = 0 }}; + var session = runtime_mod.QuerySession{ .alloc = alloc, .artifacts = undefined, .manifest = undefined }; + session.graph_metric_read_budget.limits.max_retained_bytes = 1; + var failing = std.testing.FailingAllocator.init(alloc, .{ .fail_index = 0 }); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsWithinBudgetAlloc(failing.allocator(), &columns, 1, 4096, .{ .session = &session })); + try std.testing.expectEqual(@as(usize, 0), failing.alloc_index); + session.graph_metric_read_budget = .{ .limits = .{ .max_work_items = 8 } }; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, planScoreColumnsWithinBudgetAlloc(alloc, &columns, 1, 4096, .{ .session = &session })); + try std.testing.expectEqual(@as(u64, 8), session.graph_metric_read_budget.work_items); +} + +test "serverless graph metric contiguous sparse misses use one request without overfetch" { + const alloc = std.testing.allocator; + var entries: [64]metric_segment.codec.RoutingEntry = undefined; + var touched: [64]TouchedBlock = undefined; + for (&entries, &touched, 0..) |*entry, *block, i| { + entry.* = .{ .first_node_id = "node", .offset = i * 16 * 1024, .len = 16 * 1024 }; + block.* = .{ .block_index = i, .first_pending = i, .pending_count = 1 }; + } + const ranges = try planSparseScoreFetchRangesAlloc(alloc, &entries, &touched, 0, 124); + defer alloc.free(ranges); + try std.testing.expectEqual(@as(usize, 1), ranges.len); + try std.testing.expectEqual(@as(usize, 1024 * 1024), ranges[0].len); +} + +test "serverless graph metric cached gaps only cost overfetch when admission requires it" { + const alloc = std.testing.allocator; + var entries: [4]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, i| entry.* = .{ .first_node_id = "node", .offset = i * 1024, .len = 1024 }; + const inputs = [_]RangePlanningColumn{.{ .entries = &entries, .counts = &.{ 1, 0, 1, 0 }, .score_data_offset = 0 }}; + const bridged = try planScoreColumnsWithinBudgetAlloc(alloc, &inputs, 1, 3072, .{}); + defer { + for (bridged) |ranges| alloc.free(ranges); + alloc.free(bridged); + } + try std.testing.expectEqual(@as(usize, 1), bridged[0].len); + try std.testing.expectEqual(@as(usize, 3072), bridged[0][0].len); + const result = try planScoreColumnsRangesAlloc(alloc, &inputs, 2); + defer { + for (result) |ranges| alloc.free(ranges); + alloc.free(result); + } + try std.testing.expectEqual(@as(usize, 2), result[0].len); + for (result[0]) |range| try std.testing.expectEqual(@as(usize, 1024), range.len); + const Cancel = struct { + fn isCancelled(_: *const anyopaque) bool { + return true; + } + }; + const canceled = true; + try std.testing.expectError(error.Canceled, planScoreColumnsWithinBudgetAlloc(alloc, &inputs, 2, 4096, .{ .cancellation = .{ .ptr = &canceled, .is_cancelled_fn = Cancel.isCancelled } })); +} + +test "serverless graph metric range planning caps broad point batches" { + const alloc = std.testing.allocator; + var entries: [128]metric_segment.codec.RoutingEntry = undefined; + var counts: [128]usize = @splat(1); + for (&entries, 0..) |*entry, index| entry.* = .{ + .first_node_id = "node", + .offset = index * 64 * 1024, + .len = 64 * 1024, + }; + const broad = try planScoreFetchRangesAlloc(alloc, &entries, &counts, 0, max_score_range_requests); + defer alloc.free(broad); + try std.testing.expectEqual(@as(usize, 1), broad.len); + try std.testing.expectEqual(@as(usize, 128), broad[0].last_block - broad[0].first_block + 1); + try std.testing.expectEqual(coalesced_score_window_bytes, broad[0].len); + + // The old threshold was 32 and caused the 33rd touched block to jump from + // exact reads to full windows. Moderate point batches now stay exact. + @memset(&counts, 0); + @memset(counts[0..33], 1); + const moderate = try planScoreFetchRangesAlloc(alloc, &entries, &counts, 0, max_score_range_requests); + defer alloc.free(moderate); + try std.testing.expectEqual(@as(usize, 1), moderate.len); + try std.testing.expectEqual(@as(usize, 33 * 64 * 1024), moderate[0].len); + + @memset(&counts, 0); + counts[0] = 1; + counts[counts.len - 1] = 1; + const sparse = try planScoreFetchRangesAlloc(alloc, &entries, &counts, 0, max_score_range_requests); + defer alloc.free(sparse); + try std.testing.expectEqual(@as(usize, 2), sparse.len); + try std.testing.expectEqual(@as(usize, 64 * 1024), sparse[0].len); + try std.testing.expectEqual(@as(usize, 64 * 1024), sparse[1].len); + + var unused_session: runtime_mod.QuerySession = undefined; + const oversized_batch: [max_parallel_metric_range_requests + 1]ScoreFetchRange = @splat(.{ + .first_block = 0, + .last_block = 0, + .offset = 0, + .len = 1, + }); + try std.testing.expectError( + error.GraphMetricQueryBudgetExceeded, + fetchMetricRangeBatchAlloc(alloc, &unused_session, 0, 0, &.{}, &oversized_batch, .score), + ); +} + +test "serverless graph metric sparse planning stays proportional to touched blocks" { + const alloc = std.testing.allocator; + var entries: [4096]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, index| entry.* = .{ + .first_node_id = "node", + .offset = index * 64, + .len = 64, + }; + const touched = [_]struct { block_index: usize, first_pending: usize, pending_count: usize }{ + .{ .block_index = 3072, .first_pending = 0, .pending_count = 1 }, + }; + const ranges = try planSparseScoreFetchRangesAlloc(alloc, &entries, &touched, 0, max_score_range_requests); + defer alloc.free(ranges); + try std.testing.expectEqual(@as(usize, 1), ranges.len); + try std.testing.expectEqual(@as(usize, 3072), ranges[0].first_block); + try std.testing.expectEqual(@as(usize, 64), ranges[0].len); +} + +test "serverless graph metric transport windows trim unused boundary blocks" { + const alloc = std.testing.allocator; + var entries: [128]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, index| entry.* = .{ + .first_node_id = "node", + .offset = index * 64 * 1024, + .len = 64 * 1024, + }; + var first_counts: [128]usize = @splat(1); + var second_counts: [128]usize = @splat(1); + first_counts[0] = 0; + second_counts[1] = 0; + const first = try planScoreFetchRangesAlloc(alloc, &entries, &first_counts, 0, max_score_range_requests); + defer alloc.free(first); + const second = try planScoreFetchRangesAlloc(alloc, &entries, &second_counts, 0, max_score_range_requests); + defer alloc.free(second); + try std.testing.expectEqual(@as(usize, 1), first.len); + try std.testing.expectEqual(@as(usize, 2), second.len); + try std.testing.expectEqual(@as(u64, 64 * 1024), first[0].offset); + try std.testing.expectEqual(@as(usize, 127 * 64 * 1024), first[0].len); + try std.testing.expectEqual(@as(u64, 0), second[0].offset); + try std.testing.expectEqual(@as(usize, 64 * 1024), second[0].len); + try std.testing.expectEqual(@as(usize, 126 * 64 * 1024), second[1].len); +} + +test "serverless graph metric top range planning coalesces contiguous blocks" { + const alloc = std.testing.allocator; + var entries: [977]metric_segment.codec.RoutingEntry = undefined; + for (&entries, 0..) |*entry, index| entry.* = .{ + .first_node_id = "node", + .offset = index * 256 * 1024, + .len = 256 * 1024, + }; + const ranges = try planAllScoreFetchRangesAlloc(alloc, &entries); + defer alloc.free(ranges); + try std.testing.expectEqual(@as(usize, 31), ranges.len); + for (ranges) |range| try std.testing.expect(range.len <= coalesced_score_window_bytes); +} + +test "serverless graph metric point reads authenticate before fetching ranges" { + const alloc = std.testing.allocator; + const State = struct { + verify_calls: usize = 0, + range_calls: usize = 0, + + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedPut; + } + fn getAlloc(_: *anyopaque, _: Allocator, _: []const u8) ![]u8 { + return error.UnexpectedFullRead; + } + fn getRangeAlloc(ptr: *anyopaque, alloc_: Allocator, _: []const u8, _: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.range_calls += 1; + const payload = try alloc_.alloc(u8, len); + @memset(payload, 0); + return payload; + } + fn stat(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedStat; + } + fn verifyContent(ptr: *anyopaque, _: Allocator, _: []const u8, _: u64, _: []const u8, _: @import("../../common/cancellation.zig").CancellationToken) !void { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.verify_calls += 1; + return error.ArtifactIntegrityMismatch; + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.UnexpectedDelete; + } + + const vtable = artifacts_mod.ArtifactStore.VTable{ + .deinit = deinit, + .put = put, + .get_alloc = getAlloc, + .get_range_alloc = getRangeAlloc, + .stat = stat, + .verify_content = verifyContent, + .delete = delete, + }; + }; + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const artifact_id = "sha256:" ++ checksum; + var state = State{}; + var artifacts = artifacts_mod.ArtifactStore{ .allocator = alloc, .ptr = &state, .vtable = &State.vtable }; + defer artifacts.deinit(); + var refs = [_]manifest_mod.ArtifactRef{ + .{ .kind = .graph_segment, .name = "graph_idx", .artifact_id = artifact_id, .byte_len = 1, .checksum = checksum }, + .{ .kind = .graph_metric_segment, .name = "9:graph_idx4:rank", .artifact_id = artifact_id, .byte_len = 100, .checksum = checksum, .metadata_version = metric_segment.wire_version }, + }; + refs[1].graph_metric_control_len = @intCast(try metric_segment.controlProbeLen( + refs[1].byte_len, + refs[0].artifact_id, + refs[0].checksum, + .{}, + )); + // Session fetch buffers and result buffers deliberately have different + // owners. This exercises direct reads as well as forked column reads. + var session_arena = std.heap.ArenaAllocator.init(alloc); + defer session_arena.deinit(); + var session = runtime_mod.QuerySession{ + .alloc = session_arena.allocator(), + .artifacts = &artifacts, + .manifest = .{ + .namespace = "docs", + .version = 1, + .built_at_ns = 1, + .wal_start_lsn = 1, + .wal_end_lsn = 1, + .stats = .{ .indexes_json = @constCast("{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\"}}}}") }, + .artifacts = &refs, + }, + }; + defer session.clearGraphMetricSpecs(); + const cached_specs = try session.graphMetricSpecs(); + const cached_specs_again = try session.graphMetricSpecs(); + try std.testing.expect(cached_specs.ptr == cached_specs_again.ptr); + const node_ids = [_][]const u8{"node"}; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + try std.testing.expectEqual(@as(usize, 0), state.verify_calls); + try std.testing.expectEqual(@as(usize, 1), state.range_calls); +} + +test "serverless graph metric point and top-1025 reads authenticate bounded ranges without full scans" { + try testAuthenticatedMetricReads(metric_segment.score_block_entries + 1); +} + +test "serverless graph metric point routing reads only selected pages of a large index" { + try testAuthenticatedMetricReadsWithPrefix(2 * metric_segment.codec.routing_page_entries * metric_segment.score_block_entries + 1, "x" ** 256); +} + +test "serverless graph metric small byte indexes keep a single routing read across pages" { + try testAuthenticatedMetricReads(2 * metric_segment.codec.routing_page_entries * metric_segment.score_block_entries + 1); +} + +fn testAuthenticatedMetricReads(score_count: usize) !void { + try testAuthenticatedMetricReadsWithPrefix(score_count, ""); +} + +fn testAuthenticatedMetricReadsWithPrefix(score_count: usize, prefix: []const u8) !void { + const alloc = std.testing.allocator; + const source_checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const source_artifact_id = "sha256:" ++ source_checksum; + const config = graph_mod.GraphMetricConfig{ .name = "rank" }; + var metric = metric_segment.Segment{ + .kind = .pagerank, + .source_graph_artifact_id = try alloc.dupe(u8, source_artifact_id), + .source_graph_checksum = try alloc.dupe(u8, source_checksum), + .config_fingerprint = lake_graph_metric.configFingerprint(config), + .materializer_fingerprint = graph_metric_policy.materializerFingerprint(.{}), + .edge_filter = .{}, + .converged = true, + .iterations_completed = 2, + .delta = 0.001, + .scores = try alloc.alloc(metric_segment.Score, score_count), + }; + for (metric.scores, 0..) |*score, index| { + score.* = .{ + .node_id = try std.fmt.allocPrint(alloc, "node:{s}{d:0>8}", .{ prefix, index }), + .value = @floatFromInt(index), + }; + } + defer metric.deinit(alloc); + const payload = try metric_segment.encodeAlloc(alloc, metric); + defer alloc.free(payload); + const integrity = try metric_segment.artifactIntegrity(metric, payload); + var metric_digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(payload, &metric_digest, .{}); + const metric_checksum = std.fmt.bytesToHex(metric_digest, .lower); + var metric_artifact_id: [artifacts_mod.store.sha256_artifact_id_prefix.len + metric_checksum.len]u8 = undefined; + @memcpy(metric_artifact_id[0..artifacts_mod.store.sha256_artifact_id_prefix.len], artifacts_mod.store.sha256_artifact_id_prefix); + @memcpy(metric_artifact_id[artifacts_mod.store.sha256_artifact_id_prefix.len..], &metric_checksum); + + const State = struct { + payload: []const u8, + control_len: usize, + footer_offset: usize, + root_offset: usize, + range_calls: std.atomic.Value(usize) = .init(0), + range_bytes: std.atomic.Value(usize) = .init(0), + blocked_offset: ?u64 = null, + release_reads: std.atomic.Value(bool) = .init(true), + control_calls: std.atomic.Value(usize) = .init(0), + required_controls_before_scores: usize = 0, + verify_calls: std.atomic.Value(usize) = .init(0), + corrupt_score_reads: bool = false, + reject_point_index_reads: bool = false, + corrupt_point_index: bool = false, + corrupt_routing_page: bool = false, + corrupt_routing_root: bool = false, + + fn deinit(_: Allocator, _: *anyopaque) void {} + fn put(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedPut; + } + fn getAlloc(_: *anyopaque, _: Allocator, _: []const u8) ![]u8 { + return error.UnexpectedFullRead; + } + fn getRangeAlloc(ptr: *anyopaque, result_alloc: Allocator, _: []const u8, offset: u64, len: usize) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + _ = self.range_calls.fetchAdd(1, .monotonic); + _ = self.range_bytes.fetchAdd(len, .monotonic); + if (self.blocked_offset) |blocked| { + if (offset == blocked) while (!self.release_reads.load(.acquire)) { + try std.Options.debug_io.sleep(.fromMilliseconds(1), .awake); + }; + } + const start = std.math.cast(usize, offset) orelse return error.InvalidRange; + if (start > self.payload.len or len > self.payload.len - start) return error.InvalidRange; + const touches_point_index = start < self.root_offset and start + len > self.footer_offset; + if (start == 0) _ = self.control_calls.fetchAdd(1, .monotonic); + if (start >= self.control_len and start < self.footer_offset and + self.control_calls.load(.monotonic) < self.required_controls_before_scores) return error.ScoreReadBeforeWholeQueryPrepared; + if (self.reject_point_index_reads and touches_point_index) return error.UnexpectedPointIndexRead; + const out = try result_alloc.dupe(u8, self.payload[start..][0..len]); + if (self.corrupt_point_index and touches_point_index) out[@max(start, self.footer_offset) - start] ^= 1; + if (self.corrupt_routing_page and touches_point_index and !std.mem.startsWith(u8, self.payload[start..], "AFGD")) out[0] ^= 1; + if (self.corrupt_routing_root and start <= self.root_offset and start + len > self.root_offset) out[self.root_offset - start] ^= 1; + if (self.corrupt_score_reads and start >= self.control_len and start < self.footer_offset and out.len > @sizeOf(u32)) { + out[@sizeOf(u32)] ^= 0x01; + } + return out; + } + fn stat(_: *anyopaque, _: Allocator, _: []const u8) !artifacts_mod.ArtifactMetadata { + return error.UnexpectedStat; + } + fn verifyContent(ptr: *anyopaque, _: Allocator, _: []const u8, _: u64, _: []const u8, _: @import("../../common/cancellation.zig").CancellationToken) !void { + const self: *@This() = @ptrCast(@alignCast(ptr)); + _ = self.verify_calls.fetchAdd(1, .monotonic); + return error.UnexpectedFullVerification; + } + fn delete(_: *anyopaque, _: []const u8) !void { + return error.UnexpectedDelete; + } + + const vtable = artifacts_mod.ArtifactStore.VTable{ + .deinit = deinit, + .put = put, + .get_alloc = getAlloc, + .get_range_alloc = getRangeAlloc, + .stat = stat, + .verify_content = verifyContent, + .delete = delete, + }; + }; + var state = State{ + .payload = payload, + .control_len = integrity.control_len, + .footer_offset = payload.len - integrity.routing_footer_len, + .root_offset = payload.len - metric_segment.codec.routingRootLen(metric.scores.len), + }; + var artifacts = artifacts_mod.ArtifactStore{ .allocator = alloc, .ptr = &state, .vtable = &State.vtable }; + defer artifacts.deinit(); + var refs = [_]manifest_mod.ArtifactRef{ + .{ .kind = .graph_segment, .name = "graph_idx", .artifact_id = source_artifact_id, .byte_len = 1, .checksum = source_checksum }, + .{ + .kind = .graph_metric_segment, + .name = "9:graph_idx4:rank", + .artifact_id = &metric_artifact_id, + .byte_len = payload.len, + .checksum = &metric_checksum, + .metadata_version = metric_segment.wire_version, + .materializer_fingerprint = metric.materializer_fingerprint, + .graph_metric_control_len = integrity.control_len, + .graph_metric_routing_footer_len = integrity.routing_footer_len, + .graph_metric_control_checksum = integrity.control_checksum, + .graph_metric_routing_checksum = integrity.routing_checksum, + .graph_metric_point_index_checksum = integrity.point_index_checksum, + .graph_metric_config_fingerprint = metric.config_fingerprint, + .graph_metric_source_checksum = @splat(0xaa), + }, + undefined, + }; + refs[2] = refs[1]; + refs[2].name = "9:graph_idx5:alias"; + refs[2].published_generation = 7; + refs[2].edge_generation = 6; + refs[2].computed_at_ms = 123; + var session_arena = std.heap.ArenaAllocator.init(alloc); + defer session_arena.deinit(); + var session = runtime_mod.QuerySession{ + .alloc = session_arena.allocator(), + .artifacts = &artifacts, + .manifest = .{ + .namespace = "docs", + .version = 1, + .built_at_ns = 1, + .wal_start_lsn = 1, + .wal_end_lsn = 1, + .stats = .{ .indexes_json = @constCast("{\"graph_idx\":{\"type\":\"graph\",\"metrics\":{\"rank\":{\"kind\":\"pagerank\"},\"alias\":{\"kind\":\"pagerank\"}}}}") }, + .artifacts = &refs, + }, + }; + defer session.clearGraphMetricSpecs(); + const cached_specs = try session.graphMetricSpecs(); + const cached_specs_again = try session.graphMetricSpecs(); + try std.testing.expect(cached_specs.ptr == cached_specs_again.ptr); + const last_id = metric.scores[score_count - 1].node_id; + const last_value: f64 = @floatFromInt(score_count - 1); + const node_ids = [_][]const u8{last_id}; + + { + // Cold construction transfers only the decoded lease's live bytes; + // destroying it refunds memory even outside a read scratch scope. + const root_len = metric_segment.codec.routingRootLen(score_count); + var lease = try acquireRouting(&session, 1, refs[1], metric_segment.wire_version, payload.len - root_len, root_len, 40, root_len, true, null); + defer lease.deinit(); + try std.testing.expectEqual(@as(u64, lease.entry.bytes()), session.graph_metric_read_budget.retained_bytes); + } + try std.testing.expectEqual(@as(u64, 0), session.graph_metric_read_budget.retained_bytes); + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + + { + var aliases = try scoreColumnsAlloc(alloc, &session, "graph_idx", &.{ "rank", "alias" }, &node_ids); + defer aliases.deinit(alloc); + try std.testing.expectEqualSlices(?f64, aliases.columns[0].scores, aliases.columns[1].scores); + try std.testing.expectEqual(@as(u64, 7), aliases.columns[1].published_generation); + try std.testing.expectEqual(@as(u64, 6), aliases.columns[1].edge_generation); + try std.testing.expectEqual(@as(u64, 123), aliases.columns[1].computed_at_ms); + const detached = try aliases.columns[1].takeScores(); + defer alloc.free(detached); + detached[0] = null; + try std.testing.expectEqual(@as(?f64, last_value), aliases.columns[0].scores[0]); + const is_paged = score_count > metric_segment.codec.routing_page_entries * metric_segment.score_block_entries and integrity.routing_footer_len > 64 * 1024; + try std.testing.expectEqual(@as(usize, if (is_paged) 5 else 3), state.range_calls.load(.monotonic)); + } + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + // An alias with conflicting immutable metadata must be validated on its + // own physical plan, not hidden behind the first column's authentication. + refs[2].graph_metric_control_checksum[0] ^= 1; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoreColumnsAlloc(alloc, &session, "graph_idx", &.{ "rank", "alias" }, &node_ids)); + refs[2].graph_metric_control_checksum[0] ^= 1; + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + + var empty_points = try scoresAlloc(alloc, &session, "graph_idx", "rank", &.{}); + defer empty_points.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), empty_points.scores.len); + // Only the authenticated control/status probe is required; routing and + // score data remain untouched. + try std.testing.expectEqual(@as(usize, 1), state.range_calls.load(.monotonic)); + + state.range_calls.store(0, .monotonic); + var result = try scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids); + defer result.deinit(alloc); + try std.testing.expectEqual(@as(?f64, last_value), result.scores[0]); + try std.testing.expectEqual(@as(usize, 0), state.verify_calls.load(.monotonic)); + const paged = score_count > metric_segment.codec.routing_page_entries * metric_segment.score_block_entries and integrity.routing_footer_len > 64 * 1024; + try std.testing.expectEqual(@as(usize, if (paged) 5 else 3), state.range_calls.load(.monotonic)); + + // Exercise disconnected selected pages, duplicate candidates, and misses. + session.graph_metric_read_budget = .{}; + const mixed_ids = [_][]const u8{ last_id, metric.scores[0].node_id, last_id, "before", "zzzz" }; + var mixed = try scoresAlloc(alloc, &session, "graph_idx", "rank", &mixed_ids); + defer mixed.deinit(alloc); + try std.testing.expectEqualSlices(?f64, &.{ last_value, 0, last_value, null, null }, mixed.scores); + session.graph_metric_read_budget = .{}; + + var io_impl = std.Io.Threaded.init(std.heap.page_allocator, .{}); + defer io_impl.deinit(); + session.setIo(io_impl.io()); + if (paged) { + // Sixty distinct score blocks across three routing pages, requested + // twice but admitted/fetched/decoded as one physical column. + var broad_ids: [60][]const u8 = undefined; + for (&broad_ids, 0..) |*id, i| id.* = metric.scores[(i * 128 / 59) * metric_segment.score_block_entries].node_id; + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + var broad = try scoreColumnsAlloc(alloc, &session, "graph_idx", &.{ "rank", "rank" }, &broad_ids); + defer broad.deinit(alloc); + for (broad.columns) |column| for (column.scores, 0..) |score, i| { + try std.testing.expectEqual(@as(?f64, @floatFromInt((i * 128 / 59) * metric_segment.score_block_entries)), score); + }; + try std.testing.expectEqual(@as(u64, 66), session.graph_metric_read_budget.range_requests); + try std.testing.expectEqual(@as(usize, 66), state.range_calls.load(.monotonic)); + session.graph_metric_read_budget = .{}; + } + const column_names = [_][]const u8{ "rank", "rank", "rank" }; + state.control_calls.store(0, .monotonic); + state.required_controls_before_scores = 1; + var columns = try scoreColumnsAlloc(alloc, &session, "graph_idx", &column_names, &node_ids); + state.required_controls_before_scores = 0; + defer columns.deinit(alloc); + try std.testing.expectEqual(@as(usize, column_names.len), columns.columns.len); + for (columns.columns) |column| try std.testing.expectEqual(@as(?f64, last_value), column.scores[0]); + + // All routing fits, but the complete score plan does not. Admission must + // reject before any column starts score I/O, including earlier cohorts. + const metadata_requests: u64 = if (paged) 4 else 2; + session.graph_metric_read_budget = .{ .limits = .{ .max_range_requests = metadata_requests } }; + state.range_calls.store(0, .monotonic); + state.control_calls.store(0, .monotonic); + state.required_controls_before_scores = 1; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, scoreColumnsAlloc(alloc, &session, "graph_idx", &column_names, &node_ids)); + state.required_controls_before_scores = 0; + try std.testing.expectEqual(@as(usize, 1), state.control_calls.load(.monotonic)); + try std.testing.expectEqual(metadata_requests, state.range_calls.load(.monotonic)); + session.graph_metric_read_budget = .{}; + + const AllocationRunner = struct { + fn run(failing_alloc: Allocator, active_session: *runtime_mod.QuerySession, lookup_id: []const u8) !void { + // Each injected run is one logical request. Workers share this + // budget, but never the intentionally non-thread-safe allocator. + active_session.graph_metric_read_budget = .{}; + const names = [_][]const u8{ "rank", "alias", "rank" }; + const ids = [_][]const u8{lookup_id}; + var direct = try scoresAlloc(failing_alloc, active_session, "graph_idx", "rank", &ids); + defer direct.deinit(failing_alloc); + var loaded = try scoreColumnsAlloc(failing_alloc, active_session, "graph_idx", &names, &ids); + defer loaded.deinit(failing_alloc); + } + }; + try std.testing.checkAllAllocationFailures(alloc, AllocationRunner.run, .{ &session, last_id }); + + state.range_calls.store(0, .monotonic); + state.reject_point_index_reads = true; + session.graph_metric_read_budget = .{}; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, topWithLimitsAlloc(alloc, &session, "graph_idx", "rank", 1, .{ .max_result_bytes = @sizeOf(Score) - 1 })); + try std.testing.expectEqual(@as(usize, 2), state.range_calls.load(.monotonic)); + state.range_calls.store(0, .monotonic); + session.graph_metric_read_budget = .{}; + var top_one = try topWithLimitsAlloc(alloc, &session, "graph_idx", "rank", 1, .{}); + defer top_one.deinit(alloc); + try std.testing.expectEqualStrings(last_id, top_one.scores[0].node_id); + try std.testing.expectEqual(@as(usize, 3), state.range_calls.load(.monotonic)); + const routing_retained = session.graph_metric_read_budget.retained_bytes - @sizeOf(Score) - last_id.len; + session.graph_metric_read_budget = .{ .limits = .{ .max_retained_bytes = routing_retained } }; + state.range_calls.store(0, .monotonic); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, topAlloc(alloc, &session, "graph_idx", "rank", 1)); + // Routing and control reservations have retired; only escaping output + // remains charged. This smaller limit rejects before any control I/O. + try std.testing.expectEqual(@as(usize, 0), state.range_calls.load(.monotonic)); + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + var top = try topWithLimitsAlloc(alloc, &session, "graph_idx", "rank", metric_segment.score_block_entries + 1, .{}); + defer top.deinit(alloc); + try std.testing.expectEqual(@as(usize, metric_segment.score_block_entries + 1), top.scores.len); + try std.testing.expectEqualStrings(last_id, top.scores[0].node_id); + try std.testing.expectEqual(last_value, top.scores[0].value); + try std.testing.expectEqualStrings(metric.scores[score_count - top.scores.len].node_id, top.scores[top.scores.len - 1].node_id); + try std.testing.expectEqual(@as(usize, 3), state.range_calls.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 0), state.verify_calls.load(.monotonic)); + + state.reject_point_index_reads = false; + state.corrupt_point_index = true; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + state.corrupt_point_index = false; + state.corrupt_routing_page = true; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + state.corrupt_routing_page = false; + state.corrupt_routing_root = true; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + try std.testing.expectError(error.InvalidGraphMetricSegment, topWithLimitsAlloc(alloc, &session, "graph_idx", "rank", 1, .{})); + state.corrupt_routing_root = false; + state.corrupt_score_reads = true; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + try std.testing.expectError(error.InvalidGraphMetricSegment, topWithLimitsAlloc( + alloc, + &session, + "graph_idx", + "rank", + metric_segment.score_block_entries + 1, + .{}, + )); + try std.testing.expectEqual(@as(usize, 0), state.verify_calls.load(.monotonic)); + + state.corrupt_score_reads = false; + refs[1].graph_metric_routing_footer_len = @intCast(metric_segment.codec.routingRootLen(score_count) - 1); + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + refs[1].graph_metric_routing_footer_len = integrity.routing_footer_len; + state.range_calls.store(0, .monotonic); + session.graph_metric_read_budget = .{ .limits = .{ + .max_range_requests = 2, + .max_range_bytes = std.math.maxInt(u64), + .max_decoded_blocks = std.math.maxInt(u64), + .max_work_items = std.math.maxInt(u64), + .max_retained_bytes = std.math.maxInt(u64), + } }; + try std.testing.expectError( + error.GraphMetricQueryBudgetExceeded, + scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids), + ); + // Control and routing are fetched; the score range is rejected before it + // can reach the backend. The counter is shared by every later metric read + // performed through this pinned request session. + try std.testing.expectEqual(@as(usize, 2), state.range_calls.load(.monotonic)); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const cache_root = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/routing", .{tmp.sub_path}); + defer alloc.free(cache_root); + var cache = try @import("cache.zig").QueryCache.init(alloc, cache_root); + defer cache.deinit(); + session.cache = &cache; + defer session.cache = null; + session.graph_metric_read_budget = .{}; + var first_cached = try scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids); + defer first_cached.deinit(alloc); + // Every routing level is leased decoded, including a large-index page. + // A warm sparse lookup admits only the score block's decode. + session.graph_metric_read_budget = .{ .limits = .{ .max_decoded_blocks = 1 } }; + var second_cached = try scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids); + defer second_cached.deinit(alloc); + try std.testing.expectEqual(@as(?f64, last_value), second_cached.scores[0]); + try std.testing.expectEqual(@as(u64, if (paged) 3 else 1), cache.graph_metric_routing.hits); + try std.testing.expectEqual(@as(u64, 1), session.graph_metric_read_budget.decoded_blocks); + // Scoped stage reads release routing, planner, and output reservations. + // Repeated warm stages must not accumulate a fictitious retained heap. + session.graph_metric_read_budget = .{}; + for (0..8) |_| { + var scoped_columns = try scoreColumnsScopedAlloc(alloc, &session, "graph_idx", &.{"rank"}, &node_ids); + try std.testing.expectEqual(@as(?f64, last_value), scoped_columns.columns[0].scores[0]); + try std.testing.expectEqual(@as(u64, node_ids.len * @sizeOf(?f64) + @sizeOf(PointScoresResult)), session.graph_metric_read_budget.retained_bytes); + scoped_columns.deinit(alloc); + try std.testing.expectEqual(@as(u64, 0), session.graph_metric_read_budget.retained_bytes); + } + try std.testing.expect(session.graph_metric_read_budget.work_items > 0); + if (score_count == metric_segment.score_block_entries + 1) { + var ranked = try metric_segment.codec.decodeRoutingRootAlloc(alloc, payload[state.root_offset..], payload.len, metric_segment.wire_version, .none); + defer ranked.deinit(alloc); + const io = io_impl.io(); + state.blocked_offset = ranked.ranked_entries[0].offset; + state.release_reads.store(false, .release); + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + var children = [_]runtime_mod.QuerySession{ session.forkGraphMetricRead(std.heap.smp_allocator), session.forkGraphMetricRead(std.heap.smp_allocator) }; + defer for (&children) |*child| child.deinit(); + for (&children) |*child| child.io = io; + const Worker = struct { + fn run(child: *runtime_mod.QuerySession, entries: []const metric_segment.codec.RankedRoutingEntry, output: *?OwnedMetricRange, failure: *?anyerror) void { + output.* = fetchRankedScoreBlocksAlloc(std.heap.smp_allocator, child, 1, entries, 0) catch |err| { + failure.* = err; + return; + }; + } + }; + var outputs: [2]?OwnedMetricRange = @splat(null); + defer for (&outputs) |*output| if (output.*) |*range| range.deinit(std.heap.smp_allocator); + var failures: [2]?anyerror = @splat(null); + var group: std.Io.Group = .init; + // Always unblock and join before destroying any worker-owned state. + defer { + state.release_reads.store(true, .release); + group.await(io) catch {}; + state.blocked_offset = null; + } + group.async(io, Worker.run, .{ &children[0], ranked.ranked_entries[0..1], &outputs[0], &failures[0] }); + for (0..1000) |_| { + if (state.range_calls.load(.monotonic) != 0) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + group.async(io, Worker.run, .{ &children[1], ranked.ranked_entries[0..1], &outputs[1], &failures[1] }); + for (0..1000) |_| { + if (cache.graph_metric_blocks.snapshot().waiters != 0) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + const shared = cache.graph_metric_blocks.snapshot().waiters; + state.release_reads.store(true, .release); + try group.await(io); + try std.testing.expectEqual(@as(usize, 1), shared); + for (failures) |failure| if (failure) |err| return err; + try std.testing.expectEqual(@as(usize, 1), state.range_calls.load(.monotonic)); + try std.testing.expectEqualSlices(u8, outputs[0].?.bytes, outputs[1].?.bytes); + for (&outputs) |*output| { + output.*.?.deinit(std.heap.smp_allocator); + output.* = null; + } + + // Changing K must download only the newly needed canonical block. + state.range_bytes.store(0, .monotonic); + session.graph_metric_read_budget = .{}; + var top_two = try topAlloc(alloc, &session, "graph_idx", "rank", 257); + defer top_two.deinit(alloc); + // Root/control may still require a disk read but no object-store read. + // The small-index point path has not leased the ranked-only root yet. + const root_cost = metric_segment.codec.routingRootLen(score_count); + try std.testing.expect(state.range_bytes.load(.monotonic) <= ranked.ranked_entries[1].len + root_cost); + try std.testing.expect(state.range_bytes.load(.monotonic) >= ranked.ranked_entries[1].len); + state.range_bytes.store(0, .monotonic); + session.graph_metric_read_budget = .{}; + var warm_top_one = try topAlloc(alloc, &session, "graph_idx", "rank", 1); + defer warm_top_one.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), state.range_bytes.load(.monotonic)); + cache.drainGraphMetricPersistence(); + cache.graph_metric_blocks.deinit(); + cache.graph_metric_blocks = .{}; + session.graph_metric_read_budget = .{}; + var disk_top = try topAlloc(alloc, &session, "graph_idx", "rank", 257); + defer disk_top.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), state.range_bytes.load(.monotonic)); + } + if (paged) { + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + const adjacent = [_][]const u8{ metric.scores[0].node_id, metric.scores[64 * metric_segment.score_block_entries].node_id }; + { + const io = io_impl.io(); + // Leave one fill slot: a two-page request must publish its owned + // page before waiting for capacity or another request's page. + var held: [63]usize = undefined; + for (&held, 0..) |*slot, i| { + var key: [32]u8 = @splat(0); + key[0] = 0xff; + key[1] = @intCast(i); + slot.* = cache.graph_metric_routing.begin(key).fill; + } + defer for (held) |slot| cache.graph_metric_routing.finish(slot, io); + const misses_before = cache.graph_metric_routing.snapshot().misses; + var root_view = try metric_segment.codec.decodeRoutingRootAlloc(alloc, payload[state.root_offset..], payload.len, metric_segment.wire_version, .none); + defer root_view.deinit(alloc); + const directory_start = state.root_offset - root_view.directory_len; + const directory_view = try metric_segment.codec.decodePointDirectoryAlloc(alloc, payload[directory_start..state.root_offset], directory_start, state.footer_offset, std.math.divCeil(usize, score_count, metric_segment.score_block_entries) catch unreachable, .none); + defer alloc.free(directory_view); + state.blocked_offset = directory_view[0].offset; + state.release_reads.store(false, .release); + var children = [_]runtime_mod.QuerySession{ session.forkGraphMetricRead(std.heap.smp_allocator), session.forkGraphMetricRead(std.heap.smp_allocator) }; + defer for (&children) |*child| child.deinit(); + for (&children) |*child| child.io = io; + const control = try metric_segment.decodeControl(payload, config.edge_filter); + const Worker = struct { + fn run(child: *runtime_mod.QuerySession, ref: manifest_mod.ArtifactRef, ctl: metric_segment.codec.Control, ids: []const []const u8, output: *?PointRouting, failure: *?anyerror) void { + output.* = loadPointRouting(std.heap.smp_allocator, child, 1, ref, ctl, ref.byte_len - ref.graph_metric_routing_footer_len, std.math.divCeil(usize, ctl.score_count, metric_segment.score_block_entries) catch unreachable, ids, &.{ 0, 1 }) catch |err| { + failure.* = err; + return; + }; + } + }; + var outputs: [2]?PointRouting = @splat(null); + defer for (&outputs) |*output| if (output.*) |*routing| routing.deinit(); + var failures: [2]?anyerror = @splat(null); + var group: std.Io.Group = .init; + defer { + state.release_reads.store(true, .release); + group.await(io) catch {}; + state.blocked_offset = null; + } + group.async(io, Worker.run, .{ &children[0], refs[1], control, &adjacent, &outputs[0], &failures[0] }); + for (0..1000) |_| { + if (state.range_calls.load(.monotonic) != 0) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + group.async(io, Worker.run, .{ &children[1], refs[1], control, &adjacent, &outputs[1], &failures[1] }); + for (0..1000) |_| { + if (cache.graph_metric_routing.snapshot().waiters != 0) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + const waiters = cache.graph_metric_routing.snapshot().waiters; + state.release_reads.store(true, .release); + try group.await(io); + try std.testing.expect(waiters != 0); + for (failures) |failure| if (failure) |err| return err; + try std.testing.expectEqual(@as(u64, 2), cache.graph_metric_routing.snapshot().misses - misses_before); + try std.testing.expectEqual(@as(u64, 2), session.graph_metric_read_budget.decoded_blocks); + try std.testing.expectEqual(@as(usize, 2), state.range_calls.load(.monotonic)); + for (outputs[0].?.page_leases, outputs[1].?.page_leases) |left, right| try std.testing.expect(left.?.entry == right.?.entry); + } + state.range_calls.store(0, .monotonic); + var pair = try scoresAlloc(alloc, &session, "graph_idx", "rank", &adjacent); + defer pair.deinit(alloc); + // Both decoded pages are shared; only two exact score misses remain. + try std.testing.expectEqual(@as(usize, 2), state.range_calls.load(.monotonic)); + session.graph_metric_read_budget = .{}; + state.range_calls.store(0, .monotonic); + const overlap = [_][]const u8{ adjacent[1], last_id }; + var overlapping = try scoresAlloc(alloc, &session, "graph_idx", "rank", &overlap); + defer overlapping.deinit(alloc); + try std.testing.expectEqualSlices(?f64, &.{ @floatFromInt(64 * metric_segment.score_block_entries), last_value }, overlapping.scores); + var single = try scoresAlloc(alloc, &session, "graph_idx", "rank", adjacent[1..]); + defer single.deinit(alloc); + // Coalesced [0,1], overlapping [1,2], and exact [1] share pages. + try std.testing.expectEqual(@as(usize, 0), state.range_calls.load(.monotonic)); + + // Score blocks are also canonical cache units. A broad miss set fits + // one bounded transport despite interspersed cache hits. + var broad_ids: [129][]const u8 = undefined; + for (&broad_ids, 0..) |*id, i| id.* = metric.scores[i * metric_segment.score_block_entries].node_id; + session.graph_metric_read_budget = .{ .limits = .{ .max_range_requests = 3 } }; + state.corrupt_score_reads = true; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &broad_ids)); + state.corrupt_score_reads = false; + session.graph_metric_read_budget = .{ .limits = .{ .max_range_requests = 3 } }; + state.range_calls.store(0, .monotonic); + var broad_cached = try scoresAlloc(alloc, &session, "graph_idx", "rank", &broad_ids); + defer broad_cached.deinit(alloc); + // The corrupt response did not publish any of its score blocks. + try std.testing.expectEqual(@as(usize, 1), state.range_calls.load(.monotonic)); + for (broad_cached.scores, 0..) |score, i| try std.testing.expectEqual(@as(?f64, @floatFromInt(i * metric_segment.score_block_entries)), score); + state.range_calls.store(0, .monotonic); + // Only the control read is charged. Cached scores need no network + // admission even when a new candidate set changes the routing pages. + session.graph_metric_read_budget = .{ .limits = .{ .max_range_requests = 1, .max_range_bytes = integrity.control_len } }; + var crossed = try scoresAlloc(alloc, &session, "graph_idx", "rank", broad_ids[63..66]); + defer crossed.deinit(alloc); + for (crossed.scores, 63..) |score, i| try std.testing.expectEqual(@as(?f64, @floatFromInt(i * metric_segment.score_block_entries)), score); + session.graph_metric_read_budget = .{ .limits = .{ .max_range_requests = 1, .max_range_bytes = integrity.control_len } }; + var one_page = try scoresAlloc(alloc, &session, "graph_idx", "rank", broad_ids[96..97]); + defer one_page.deinit(alloc); + try std.testing.expectEqual(@as(?f64, @floatFromInt(96 * metric_segment.score_block_entries)), one_page.scores[0]); + try std.testing.expectEqual(@as(usize, 0), state.range_calls.load(.monotonic)); + } + try std.testing.checkAllAllocationFailures(alloc, AllocationRunner.run, .{ &session, last_id }); + if (score_count == metric_segment.score_block_entries + 1) { + const broken_root = try std.fmt.allocPrint(alloc, "{s}-unavailable", .{cache_root}); + defer alloc.free(broken_root); + var broken_cache = try @import("cache.zig").QueryCache.init(alloc, broken_root); + defer broken_cache.deinit(); + const blocks_path = try std.fs.path.join(alloc, &.{ broken_root, ".blocks" }); + defer alloc.free(blocks_path); + // A file where the cache needs a directory makes both reads and + // publication fail deterministically, independent of host permissions. + const blocker = try std.Io.Dir.cwd().createFile(io_impl.io(), blocks_path, .{}); + blocker.close(io_impl.io()); + session.cache = &broken_cache; + defer session.cache = &cache; + session.graph_metric_read_budget = .{}; + var cold_scores = try scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids); + defer cold_scores.deinit(alloc); + try std.testing.expectEqual(@as(?f64, last_value), cold_scores.scores[0]); + var cold_top = try topAlloc(alloc, &session, "graph_idx", "rank", 257); + defer cold_top.deinit(alloc); + broken_cache.drainGraphMetricPersistence(); + try std.testing.expect(broken_cache.statsSnapshot().graph_metric_persistence_failures > 0); + state.range_calls.store(0, .monotonic); + session.graph_metric_read_budget = .{}; + var warm_scores = try scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids); + defer warm_scores.deinit(alloc); + var warm_top = try topAlloc(alloc, &session, "graph_idx", "rank", 257); + defer warm_top.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), state.range_calls.load(.monotonic)); + // Optional retention cannot weaken origin authentication on a miss. + broken_cache.graph_metric_blocks.deinit(); + broken_cache.graph_metric_blocks = .{}; + state.corrupt_score_reads = true; + defer state.corrupt_score_reads = false; + session.graph_metric_read_budget = .{}; + try std.testing.expectError(error.InvalidGraphMetricSegment, scoresAlloc(alloc, &session, "graph_idx", "rank", &node_ids)); + } +} + +test "serverless graph metric ranked blocks reject cross-boundary inversions and duplicates" { + var validator = RankedScoreBoundaryValidator{}; + var first = metric_segment.codec.DecodedScoreBlock{}; + first.scores[0] = .{ .node_suffix = "a", .value = 10 }; + first.scores[1] = .{ .node_suffix = "b", .value = 9 }; + first.len = 2; + try validator.observeBlock(first); + + var valid = metric_segment.codec.DecodedScoreBlock{}; + valid.scores[0] = .{ .node_suffix = "c", .value = 9 }; + valid.scores[1] = .{ .node_suffix = "d", .value = 8 }; + valid.len = 2; + try validator.observeBlock(valid); + + var inverted_validator = RankedScoreBoundaryValidator{}; + try inverted_validator.observeBlock(first); + var inverted = metric_segment.codec.DecodedScoreBlock{}; + inverted.scores[0] = .{ .node_suffix = "c", .value = 11 }; + inverted.len = 1; + try std.testing.expectError(error.InvalidGraphMetricSegment, inverted_validator.observeBlock(inverted)); + + var duplicate_validator = RankedScoreBoundaryValidator{}; + try duplicate_validator.observeBlock(first); + var duplicate = metric_segment.codec.DecodedScoreBlock{}; + duplicate.scores[0] = .{ .node_suffix = "b", .value = 9 }; + duplicate.len = 1; + try std.testing.expectError(error.InvalidGraphMetricSegment, duplicate_validator.observeBlock(duplicate)); +} diff --git a/zig/pkg/antfly/src/serverless/query/graph_metric_routing_cache.zig b/zig/pkg/antfly/src/serverless/query/graph_metric_routing_cache.zig new file mode 100644 index 0000000000..d6526a76b3 --- /dev/null +++ b/zig/pkg/antfly/src/serverless/query/graph_metric_routing_cache.zig @@ -0,0 +1,487 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Bounded, process-local cache of authenticated immutable routing indexes. +//! Leases keep borrowed node IDs alive. Eviction never invalidates a reader; +//! pinned entries remain charged and admission bypasses a saturated cache. +const std = @import("std"); +const codec = @import("../graph_metric_segment/codec.zig"); +const CancellationToken = @import("../../api/operation.zig").CancellationToken; +const Allocator = std.mem.Allocator; + +pub const Entry = struct { + key: [32]u8, + alloc: Allocator, + footer: []u8, + routing: codec.RoutingIndex, + references: usize = 0, + resident: bool = false, + // Metadata gets a separate LRU so streaming decoded pages cannot evict + // hot roots/directories ahead of unused pages. Both share the byte limit. + class: enum(u1) { metadata, page } = .metadata, + hash_next: ?*Entry = null, + older: ?*Entry = null, + newer: ?*Entry = null, + + pub fn bytes(self: *const Entry) usize { + return @sizeOf(Entry) + self.footer.len + self.routing.entries.len * @sizeOf(codec.RoutingEntry) + + self.routing.ranked_entries.len * @sizeOf(codec.RankedRoutingEntry); + } + + pub fn destroy(self: *Entry) void { + const alloc = self.alloc; + self.routing.deinit(alloc); + alloc.free(self.footer); + alloc.destroy(self); + } +}; + +pub const Lease = struct { + entry: *Entry, + cache: ?*Cache = null, + + pub fn deinit(self: *Lease) void { + if (self.cache) |cache| { + cache.lock(); + defer cache.mu.unlock(); + cache.releaseLocked(self.entry); + } else self.entry.destroy(); + self.* = undefined; + } +}; + +pub const Cache = struct { + mu: std.atomic.Mutex = .unlocked, + // Intrusive buckets have no entry-count ceiling or hidden map allocations. + // Every variable-size allocation is included in Entry.bytes(). + buckets: [1024]?*Entry = @splat(null), + oldest: [2]?*Entry = @splat(null), + newest: [2]?*Entry = @splat(null), + retained_bytes: usize = 0, + hits: u64 = 0, + misses: u64 = 0, + // Fixed-capacity ownership table: fetch/decode happens outside the lock. + // Releasing a failed/canceled fill permits a live waiter to take over. + fills: [64]?Fill = @splat(null), + wake_epoch: std.atomic.Value(u32) = .init(0), + + const Fill = struct { + key: [32]u8, + waiters: usize = 0, + finished: bool = false, + result: ?*Entry = null, + }; + + pub const Lookup = union(enum) { hit: Lease, fill: usize, wait: Waiter, saturated: u32 }; + + /// A registered waiter pins the fill slot and its eventual result, even + /// when the LRU cannot retain it or the producer drops its lease first. + /// Registration happens once, not on every timed cancellation check. + pub const Waiter = struct { + cache: *Cache, + index: usize, + + pub fn deinit(self: *Waiter) void { + const cache = self.cache; + cache.lock(); + defer cache.mu.unlock(); + const fill = &cache.fills[self.index].?; + std.debug.assert(fill.waiters > 0); + fill.waiters -= 1; + cache.retireFillLocked(self.index); + self.* = undefined; + } + + pub fn awaitResult(self: *Waiter, io: ?std.Io, cancellation: CancellationToken) !?Lease { + const cache = self.cache; + while (true) { + try cancellation.check(); + cache.lock(); + const fill = cache.fills[self.index].?; + const epoch = cache.wake_epoch.load(.acquire); + if (fill.finished) { + const result = if (fill.result) |entry| blk: { + entry.references += 1; + cache.hits +|= 1; + break :blk Lease{ .entry = entry, .cache = cache }; + } else null; + cache.mu.unlock(); + return result; + } + cache.mu.unlock(); + try cache.awaitFill(io, cancellation, epoch); + } + } + }; + + pub fn begin(self: *Cache, key: [32]u8) Lookup { + self.lock(); + defer self.mu.unlock(); + if (self.acquireLocked(key)) |lease| return .{ .hit = lease }; + var empty: ?usize = null; + for (&self.fills, 0..) |*fill, index| { + if (fill.*) |*existing| { + if (std.mem.eql(u8, &existing.key, &key)) { + if (existing.finished and existing.result == null) + return .{ .saturated = self.wake_epoch.load(.acquire) }; + existing.waiters += 1; + return .{ .wait = .{ .cache = self, .index = index } }; + } + } else empty = index; + } + const index = empty orelse return .{ .saturated = self.wake_epoch.load(.acquire) }; + self.fills[index] = .{ .key = key }; + self.misses +|= 1; + return .{ .fill = index }; + } + + pub fn finish(self: *Cache, index: usize, io: ?std.Io) void { + self.lock(); + std.debug.assert(self.fills[index] != null); + self.fills[index].?.finished = true; + self.retireFillLocked(index); + _ = self.wake_epoch.fetchAdd(1, .release); + self.mu.unlock(); + (io orelse std.Options.debug_io).futexWake(u32, &self.wake_epoch.raw, std.math.maxInt(u32)); + } + + fn retireFillLocked(self: *Cache, index: usize) void { + const fill = self.fills[index].?; + if (!fill.finished or fill.waiters != 0) return; + if (fill.result) |entry| self.releaseLocked(entry); + self.fills[index] = null; + // Saturation waiters also use a bounded timeout; changing the epoch + // ensures a newly available slot cannot suffer a lost notification. + _ = self.wake_epoch.fetchAdd(1, .release); + } + + fn releaseLocked(self: *Cache, entry: *Entry) void { + std.debug.assert(entry.references > 0); + entry.references -= 1; + if (entry.references == 0) { + if (entry.resident) self.append(entry) else entry.destroy(); + } + } + + pub fn publish(self: *Cache, index: usize, entry: *Entry, max_bytes: usize) Lease { + self.lock(); + defer self.mu.unlock(); + const lease = self.adoptLocked(entry, max_bytes); + const fill = &self.fills[index].?; + std.debug.assert(!fill.finished and fill.result == null); + std.debug.assert(std.mem.eql(u8, &fill.key, &lease.entry.key)); + lease.entry.references += 1; + fill.result = lease.entry; + return lease; + } + + /// Notification-driven waiting avoids extra cold-read latency. The bounded + /// timeout observes request-token cancellation independently of Io task + /// cancellation. Epoch comparison prevents lost wakes; no lock spans I/O. + pub fn awaitFill(self: *Cache, io: ?std.Io, cancellation: CancellationToken, observed_epoch: u32) !void { + try cancellation.check(); + if (self.wake_epoch.load(.acquire) != observed_epoch) return; + try (io orelse std.Options.debug_io).futexWaitTimeout(u32, &self.wake_epoch.raw, observed_epoch, .{ + .duration = .{ .raw = .fromMilliseconds(10), .clock = .awake }, + }); + try cancellation.check(); + } + + pub fn snapshot(self: *Cache) struct { hits: u64, misses: u64, bytes: usize, waiters: usize } { + self.lock(); + defer self.mu.unlock(); + var waiters: usize = 0; + for (self.fills) |fill| if (fill) |active| { + waiters += active.waiters; + }; + return .{ .hits = self.hits, .misses = self.misses, .bytes = self.retained_bytes, .waiters = waiters }; + } + + fn lock(self: *Cache) void { + @import("antfly_platform").sync.lockYielding(&self.mu); + } + + pub fn acquire(self: *Cache, key: [32]u8) ?Lease { + self.lock(); + defer self.mu.unlock(); + const lease = self.acquireLocked(key); + if (lease == null) self.misses +|= 1; + return lease; + } + + fn acquireLocked(self: *Cache, key: [32]u8) ?Lease { + if (self.find(key)) |entry| { + if (entry.references == 0) self.unlink(entry); + entry.references += 1; + self.hits +|= 1; + return .{ .entry = entry, .cache = self }; + } + return null; + } + + /// Takes ownership even on bypass. Decode and object-store I/O happen + /// before this call, never while holding the cache mutex. + pub fn adopt(self: *Cache, entry: *Entry, max_bytes: usize) Lease { + self.lock(); + defer self.mu.unlock(); + return self.adoptLocked(entry, max_bytes); + } + + fn adoptLocked(self: *Cache, entry: *Entry, max_bytes: usize) Lease { + if (self.find(entry.key)) |existing| { + if (existing.references == 0) self.unlink(existing); + existing.references += 1; + entry.destroy(); + return .{ .entry = existing, .cache = self }; + } + const needed = entry.bytes(); + entry.references = 1; + if (needed > max_bytes) return .{ .entry = entry, .cache = self }; + while (self.retained_bytes > max_bytes - needed) { + const old = self.victim(1) orelse self.victim(0) orelse return .{ .entry = entry, .cache = self }; + self.retained_bytes -= old.bytes(); + self.unlink(old); + var link = &self.buckets[bucket(old.key)]; + while (link.*.? != old) link = &link.*.?.hash_next; + link.* = old.hash_next; + old.destroy(); + } + entry.resident = true; + const slot = &self.buckets[bucket(entry.key)]; + entry.hash_next = slot.*; + slot.* = entry; + self.retained_bytes += needed; + return .{ .entry = entry, .cache = self }; + } + + fn bucket(key: [32]u8) usize { + return std.mem.readInt(u64, key[0..8], .little) % 1024; + } + + fn find(self: *Cache, key: [32]u8) ?*Entry { + var next = self.buckets[bucket(key)]; + while (next) |entry| : (next = entry.hash_next) { + if (std.mem.eql(u8, &entry.key, &key)) return entry; + } + return null; + } + + fn victim(self: *Cache, class: usize) ?*Entry { + // Only unpinned entries enter the LRU. Admission must remain O(1) + // when a large query pins thousands of decoded pages. + return self.oldest[class]; + } + + fn unlink(self: *Cache, entry: *Entry) void { + const class = @intFromEnum(entry.class); + if (entry.older) |older| older.newer = entry.newer else self.oldest[class] = entry.newer; + if (entry.newer) |newer| newer.older = entry.older else self.newest[class] = entry.older; + } + + fn append(self: *Cache, entry: *Entry) void { + const class = @intFromEnum(entry.class); + entry.older = self.newest[class]; + entry.newer = null; + if (self.newest[class]) |newest| newest.newer = entry else self.oldest[class] = entry; + self.newest[class] = entry; + } + + /// The owning QueryCache outlives all query sessions and their leases. + pub fn deinit(self: *Cache) void { + for (self.fills) |fill| std.debug.assert(fill == null); + for (self.buckets) |head| { + var next = head; + while (next) |entry| { + next = entry.hash_next; + std.debug.assert(entry.references == 0); + entry.destroy(); + } + } + self.* = undefined; + } +}; + +fn testEntry(key: u8) !*Entry { + const alloc = std.testing.allocator; + const entry = try alloc.create(Entry); + errdefer alloc.destroy(entry); + const footer = try alloc.dupe(u8, "immutable footer"); + errdefer alloc.free(footer); + const entries = try alloc.alloc(codec.RoutingEntry, 0); + errdefer alloc.free(entries); + const ranked = try alloc.alloc(codec.RankedRoutingEntry, 0); + entry.* = .{ + .key = @splat(key), + .alloc = alloc, + .footer = footer, + .routing = .{ .entries = entries, .ranked_entries = ranked, .footer_offset = 1, .top_score_count = 0 }, + }; + return entry; +} + +test "serverless graph metric routing cache bounds pinned memory and deduplicates concurrent fills" { + var cache = Cache{}; + defer cache.deinit(); + const first = try testEntry(1); + const limit = first.bytes(); + var pinned = cache.adopt(first, limit); + var duplicate = cache.adopt(try testEntry(1), limit); + try std.testing.expect(pinned.entry == duplicate.entry); + try std.testing.expectEqual(limit, cache.retained_bytes); + duplicate.deinit(); + var bypass = cache.adopt(try testEntry(2), limit); + try std.testing.expect(!bypass.entry.resident); + try std.testing.expectEqualStrings("immutable footer", pinned.entry.footer); + bypass.deinit(); + pinned.deinit(); + var replacement = cache.adopt(try testEntry(2), limit); + defer replacement.deinit(); + try std.testing.expect(cache.acquire(@splat(1)) == null); + var hit = cache.acquire(@splat(2)).?; + defer hit.deinit(); + try std.testing.expectEqual(limit, cache.retained_bytes); +} + +test "serverless graph metric routing cache retains wide multi-metric working sets by bytes" { + var cache = Cache{}; + defer cache.deinit(); + // Sixteen metrics, each with root + directory + three decoded pages. + // The former 64-slot LRU missed all 80 entries on every sequential query. + const limit = 1024 * 1024; + for (0..80) |i| { + const entry = try testEntry(@intCast(i)); + entry.class = if (i % 5 < 2) .metadata else .page; + var lease = cache.adopt(entry, limit); + lease.deinit(); + } + for (0..4) |_| { + var leases: [80]Lease = undefined; + var count: usize = 0; + defer for (leases[0..count]) |*lease| lease.deinit(); + for (&leases, 0..) |*lease, i| { + lease.* = cache.acquire(@splat(@intCast(i))) orelse return error.TestUnexpectedResult; + count += 1; + } + } + try std.testing.expectEqual(@as(u64, 320), cache.snapshot().hits); + try std.testing.expectEqual(@as(u64, 0), cache.snapshot().misses); + try std.testing.expect(cache.retained_bytes < limit); +} + +test "serverless graph metric routing cache evicts pages before metadata and preserves pins" { + var cache = Cache{}; + defer cache.deinit(); + const root = try testEntry(1); + const limit = root.bytes() * 3; + var metadata = cache.adopt(root, limit); + metadata.deinit(); + const first = try testEntry(2); + first.class = .page; + var pinned = cache.adopt(first, limit); + defer pinned.deinit(); + for (3..100) |i| { + const entry = try testEntry(@intCast(i)); + entry.class = .page; + var lease = cache.adopt(entry, limit); + lease.deinit(); + } + var hit = cache.acquire(@splat(1)) orelse return error.TestUnexpectedResult; + defer hit.deinit(); + try std.testing.expectEqualStrings("immutable footer", pinned.entry.footer); + try std.testing.expectEqual(limit, cache.retained_bytes); +} + +test "serverless graph metric routing cache coalesces misses and releases canceled fills" { + var cache = Cache{}; + defer cache.deinit(); + const leader = cache.begin(@splat(1)).fill; + var canceled_waiter = cache.begin(@splat(1)).wait; + try std.testing.expectEqual(@as(u64, 1), cache.snapshot().misses); + var canceled = std.atomic.Value(bool).init(true); + try std.testing.expectError(error.Canceled, canceled_waiter.awaitResult(null, CancellationToken.fromAtomic(&canceled))); + canceled_waiter.deinit(); + // Canceling a waiter does not disturb its leader. A failed leader releases + // ownership so another live caller can independently retry. + var failed_waiter = cache.begin(@splat(1)).wait; + cache.finish(leader, null); + try std.testing.expect(try failed_waiter.awaitResult(null, .none) == null); + failed_waiter.deinit(); + const replacement = cache.begin(@splat(1)).fill; + var filled = cache.publish(replacement, try testEntry(1), 4096); + defer filled.deinit(); + cache.finish(replacement, null); + var shared = cache.begin(@splat(1)).hit; + defer shared.deinit(); + try std.testing.expect(shared.entry == filled.entry); + + var fills: [64]usize = undefined; + for (&fills, 0..) |*fill, i| fill.* = cache.begin(@splat(@intCast(i + 2))).fill; + try std.testing.expect(cache.begin(@splat(100)) == .saturated); + for (fills) |fill| cache.finish(fill, null); +} + +test "serverless graph metric routing cache wakes a concurrent waiter without duplicating a fill" { + var io_impl = std.Io.Threaded.init(std.testing.allocator, .{}); + defer io_impl.deinit(); + const io = io_impl.io(); + var cache = Cache{}; + defer cache.deinit(); + const leader = cache.begin(@splat(1)).fill; + var started: std.Io.Event = .unset; + const Waiter = struct { + fn run(runtime: std.Io, target: *Cache, ready: *std.Io.Event) !void { + var registered = target.begin(@splat(1)).wait; + defer registered.deinit(); + ready.set(runtime); + var lease = (try registered.awaitResult(runtime, .none)) orelse return error.TestUnexpectedResult; + defer lease.deinit(); + try std.testing.expectEqualStrings("immutable footer", lease.entry.footer); + } + }; + var waiter = try io.concurrent(Waiter.run, .{ io, &cache, &started }); + defer _ = waiter.cancel(io) catch {}; + try started.wait(io); + var filled = cache.publish(leader, try testEntry(1), 0); + defer filled.deinit(); + cache.finish(leader, io); + try waiter.await(io); + try std.testing.expectEqual(@as(u64, 1), cache.snapshot().misses); +} + +test "serverless graph metric routing cache shares bypass after producer release and canceled waiters" { + var cache = Cache{}; + defer cache.deinit(); + inline for (.{ false, true }) |pinned_pressure| { + var pinned = cache.adopt(try testEntry(2), 4096); + const leader = cache.begin(@splat(1)).fill; + var first = cache.begin(@splat(1)).wait; + var canceled = cache.begin(@splat(1)).wait; + var last = cache.begin(@splat(1)).wait; + var producer = cache.publish(leader, try testEntry(1), if (pinned_pressure) pinned.entry.bytes() else 0); + try std.testing.expect(!producer.entry.resident); + cache.finish(leader, null); + producer.deinit(); + canceled.deinit(); + var a = (try first.awaitResult(null, .none)).?; + first.deinit(); + var b = (try last.awaitResult(null, .none)).?; + last.deinit(); + try std.testing.expect(a.entry == b.entry); + a.deinit(); + try std.testing.expectEqualStrings("immutable footer", b.entry.footer); + b.deinit(); + pinned.deinit(); + try std.testing.expect(cache.acquire(@splat(1)) == null); + } +} diff --git a/zig/pkg/antfly/src/serverless/query/graph_reader.zig b/zig/pkg/antfly/src/serverless/query/graph_reader.zig index db9fa618e4..ba7c044123 100644 --- a/zig/pkg/antfly/src/serverless/query/graph_reader.zig +++ b/zig/pkg/antfly/src/serverless/query/graph_reader.zig @@ -20,6 +20,74 @@ const manifest_mod = @import("../manifest/mod.zig"); const request_mod = @import("request.zig"); const runtime_mod = @import("runtime.zig"); const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +const work_budget_mod = @import("../../graph/work_budget.zig"); + +/// Rows retained here supply stable borrowed identities to BFS parent/queue +/// state. Only visited adjacency is decoded, with one shared I/O/work/memory +/// allowance for the entire query, including dictionary and routing reads. +const GraphSource = struct { + budget: work_budget_mod.WorkBudget, + allocation: work_budget_mod.RetainedAllocator, + paged: ?graph_segment_mod.AdjacencyReader = null, + segment: graph_segment_mod.Segment = .{ .adjacencies = &.{} }, + index: graph_segment_mod.AdjacencyIndex = .{}, + rows: std.StringHashMapUnmanaged(graph_segment_mod.Adjacency) = .empty, + remaining_bytes: u64 = 512 * 1024 * 1024, + remaining_edges: usize, + edge_types: []const []const u8, + direction: request_mod.GraphQueryDirection, + + fn init(self: *GraphSource, alloc: Allocator, session: *runtime_mod.QuerySession, artifact_index: usize, edge_types: []const []const u8, direction: request_mod.GraphQueryDirection, max_edges: usize) !void { + self.* = .{ .budget = work_budget_mod.WorkBudget.initWithLimits(.{ .max_retained_state_bytes = 256 * 1024 * 1024 }), .allocation = undefined, .remaining_edges = max_edges, .edge_types = edge_types, .direction = direction }; + self.allocation = .{ .backing = alloc, .budget = &self.budget }; + errdefer self.deinit(); + const admitted = self.allocation.allocator(); + const source = session.artifactRef(artifact_index) orelse return error.GraphSegmentNotFound; + self.paged = graph_segment_mod.AdjacencyReader.init(admitted, session.artifacts, source, session.cancellation, &self.remaining_bytes) catch |err| return self.translate(err); + if (self.paged != null) return; + // Current-wire sources may omit the optional bounded accelerator. + var lease = work_budget_mod.RetainedLease.init(&self.budget, std.math.cast(usize, source.byte_len) orelse return error.GraphTraversalQueryBudgetExceeded) catch |err| return self.translate(err); + defer lease.deinit(); + if (source.byte_len > self.remaining_bytes) return error.GraphTraversalQueryBudgetExceeded; + self.remaining_bytes -= source.byte_len; + const payload = try session.fetchArtifactAlloc(artifact_index); + defer alloc.free(payload); + self.segment = graph_segment_mod.decodeAllocWithCancellation(admitted, payload, session.cancellation) catch |err| return self.translate(err); + self.index = graph_segment_mod.AdjacencyIndex.initWithCancellation(admitted, self.segment, session.cancellation) catch |err| return self.translate(err); + } + + fn translate(self: *GraphSource, err: anyerror) anyerror { + if ((err == error.OutOfMemory and self.allocation.denied) or err == error.GraphMetricBuildBudgetExceeded or err == error.QueryCandidateBudgetExceeded) return error.GraphTraversalQueryBudgetExceeded; + return err; + } + + fn deinit(self: *GraphSource) void { + const alloc = self.allocation.allocator(); + var rows = self.rows.valueIterator(); + while (rows.next()) |row| row.deinit(alloc); + self.rows.deinit(alloc); + self.index.deinit(alloc); + self.segment.deinit(alloc); + if (self.paged) |*paged| paged.deinit(); + std.debug.assert(self.allocation.live_bytes == 0); + } + + fn contains(self: *GraphSource, key: []const u8) !bool { + if (self.paged) |*paged| return paged.containsNode(key) catch |err| return self.translate(err); + return self.index.find(self.segment, key) != null; + } + + fn find(self: *GraphSource, key: []const u8) !?graph_segment_mod.Adjacency { + if (self.paged) |*paged| { + if (self.rows.get(key)) |row| return row; + var row = (paged.adjacency(key, self.edge_types, self.direction, self.remaining_edges, &self.remaining_edges) catch |err| return self.translate(err)) orelse return null; + errdefer row.deinit(self.allocation.allocator()); + self.rows.put(self.allocation.allocator(), row.node_id, row) catch |err| return self.translate(err); + return row; + } + return self.index.find(self.segment, key); + } +}; pub const Neighbor = struct { doc_id: []u8, @@ -215,14 +283,16 @@ pub fn neighborsWithLimitsAlloc( } if (req.limit == 0) return try alloc.alloc(Neighbor, 0); const graph_index = findGraphArtifactIndex(session, req.index_name) orelse return error.GraphSegmentNotFound; - const payload = try session.fetchArtifactAlloc(graph_index); - defer alloc.free(payload); - var segment = try graph_segment_mod.decodeAlloc(alloc, payload); - defer graph_segment_mod.freeSegment(alloc, &segment); - - var adjacency_index = try graph_segment_mod.AdjacencyIndex.init(alloc, segment); - defer adjacency_index.deinit(alloc); - const adjacency = adjacency_index.find(segment, req.doc_id) orelse return try alloc.alloc(Neighbor, 0); + var source: GraphSource = undefined; + source.init(alloc, session, graph_index, req.edge_types orelse &.{}, req.direction, limits.max_edges_scanned) catch |err| switch (err) { + error.GraphTraversalQueryBudgetExceeded => return error.GraphNeighborQueryBudgetExceeded, + else => return err, + }; + defer source.deinit(); + const adjacency = (source.find(req.doc_id) catch |err| switch (err) { + error.GraphTraversalQueryBudgetExceeded => return error.GraphNeighborQueryBudgetExceeded, + else => return err, + }) orelse return try alloc.alloc(Neighbor, 0); return try selectNeighborsAlloc(alloc, session, adjacency, req, limits); } @@ -249,14 +319,10 @@ pub fn traverseWithLimitsAlloc( } if (req.limit == 0) return try alloc.alloc(TraversalNode, 0); const graph_index = findGraphArtifactIndex(session, req.index_name) orelse return error.GraphSegmentNotFound; - const payload = try session.fetchArtifactAlloc(graph_index); - defer alloc.free(payload); - var segment = try graph_segment_mod.decodeAlloc(alloc, payload); - defer graph_segment_mod.freeSegment(alloc, &segment); - - var adjacency_index = try graph_segment_mod.AdjacencyIndex.init(alloc, segment); - defer adjacency_index.deinit(alloc); - if (adjacency_index.find(segment, req.start_doc_id) == null) return try alloc.alloc(TraversalNode, 0); + var source: GraphSource = undefined; + try source.init(alloc, session, graph_index, req.edge_types orelse &.{}, req.direction, limits.max_edges_scanned); + defer source.deinit(); + if (!try source.contains(req.start_doc_id)) return try alloc.alloc(TraversalNode, 0); var queue = std.ArrayListUnmanaged(QueueItem).empty; defer queue.deinit(alloc); @@ -287,7 +353,7 @@ pub fn traverseWithLimitsAlloc( if (out.items.len >= req.limit) break; } if (item.depth == req.max_depth) continue; - const adjacency = adjacency_index.find(segment, item.doc_id) orelse continue; + const adjacency = try source.find(item.doc_id) orelse continue; if (req.direction == .out or req.direction == .both) { try enqueueEdgesAlloc(alloc, &queue, &seen, &parents, adjacency.out_edges, item, .out, req, &budget); } @@ -320,15 +386,11 @@ pub fn shortestPathWithLimitsAlloc( return error.GraphTraversalQueryBudgetExceeded; } const graph_index = findGraphArtifactIndex(session, req.index_name) orelse return error.GraphSegmentNotFound; - const payload = try session.fetchArtifactAlloc(graph_index); - defer alloc.free(payload); - var segment = try graph_segment_mod.decodeAlloc(alloc, payload); - defer graph_segment_mod.freeSegment(alloc, &segment); - - var adjacency_index = try graph_segment_mod.AdjacencyIndex.init(alloc, segment); - defer adjacency_index.deinit(alloc); - if (adjacency_index.find(segment, req.start_doc_id) == null) return null; - if (adjacency_index.find(segment, req.end_doc_id) == null) return null; + var source: GraphSource = undefined; + try source.init(alloc, session, graph_index, req.edge_types orelse &.{}, req.direction, limits.max_edges_scanned); + defer source.deinit(); + if (!try source.contains(req.start_doc_id)) return null; + if (!try source.contains(req.end_doc_id)) return null; if (std.mem.eql(u8, req.start_doc_id, req.end_doc_id)) { const result_bytes = std.math.add(usize, @sizeOf(ShortestPath) + @sizeOf([]u8), req.start_doc_id.len) catch @@ -361,7 +423,7 @@ pub fn shortestPathWithLimitsAlloc( if (cursor % 64 == 0) try session.checkCancellation(); const item = queue.items[cursor]; if (item.depth >= req.max_depth) continue; - const adjacency = adjacency_index.find(segment, item.doc_id) orelse continue; + const adjacency = try source.find(item.doc_id) orelse continue; if (req.direction == .out or req.direction == .both) { if (try enqueueShortestPathEdgesAlloc(alloc, &queue, &seen, &parents, adjacency.out_edges, item, .out, req, &budget)) |depth| { found_depth = depth; diff --git a/zig/pkg/antfly/src/serverless/query/lake_cache.zig b/zig/pkg/antfly/src/serverless/query/lake_cache.zig index d8eee56d85..cca4e92420 100644 --- a/zig/pkg/antfly/src/serverless/query/lake_cache.zig +++ b/zig/pkg/antfly/src/serverless/query/lake_cache.zig @@ -85,7 +85,7 @@ pub fn classifyArtifact(kind: artifact_ref.ArtifactKind) CacheClass { .row_fragment_stats => .row_fragment_stats, .algebraic_segment => .algebraic_segment, .external_base_source => .external_metadata, - .text_segment, .vector_segment, .sparse_segment, .graph_segment => .search_sidecar, + .text_segment, .vector_segment, .sparse_segment, .graph_segment, .graph_metric_segment => .search_sidecar, .doc_values, .stored_fields, .mutation_segment, .document_segment => .other, }; } diff --git a/zig/pkg/antfly/src/serverless/query/lake_explain.zig b/zig/pkg/antfly/src/serverless/query/lake_explain.zig index 3cca191b9d..ecf1bb59d1 100644 --- a/zig/pkg/antfly/src/serverless/query/lake_explain.zig +++ b/zig/pkg/antfly/src/serverless/query/lake_explain.zig @@ -353,7 +353,7 @@ fn accountArtifact(accounting: *ArtifactAccounting, artifact: artifact_ref.Artif .row_fragment_stats => accounting.row_fragment_stats_count += 1, .algebraic_segment => accounting.algebraic_segment_count += 1, .external_base_source => accounting.external_metadata_count += 1, - .text_segment, .vector_segment, .sparse_segment, .graph_segment => accounting.search_sidecar_count += 1, + .text_segment, .vector_segment, .sparse_segment, .graph_segment, .graph_metric_segment => accounting.search_sidecar_count += 1, .doc_values, .stored_fields, .mutation_segment, .document_segment => {}, } } diff --git a/zig/pkg/antfly/src/serverless/query/mod.zig b/zig/pkg/antfly/src/serverless/query/mod.zig index 40c4b2185a..5fe6042468 100644 --- a/zig/pkg/antfly/src/serverless/query/mod.zig +++ b/zig/pkg/antfly/src/serverless/query/mod.zig @@ -21,6 +21,8 @@ pub const plan = @import("plan.zig"); pub const cache = @import("cache.zig"); pub const indexed_reader = @import("indexed_reader.zig"); pub const graph_reader = @import("graph_reader.zig"); +pub const graph_metric_reader = @import("graph_metric_reader.zig"); +pub const graph_metric_routing_cache = @import("graph_metric_routing_cache.zig"); pub const lake_rows = @import("lake_rows.zig"); pub const lake_explain = @import("lake_explain.zig"); pub const lake_sidecar_selection = @import("lake_sidecar_selection.zig"); @@ -58,6 +60,16 @@ pub const freeGraphNeighbors = graph_reader.freeNeighbors; pub const freeGraphTraversalNodes = graph_reader.freeTraversalNodes; pub const freeGraphPathHops = graph_reader.freePathHops; pub const freeGraphShortestPath = graph_reader.freeShortestPath; +pub const GraphMetricScore = graph_metric_reader.Score; +pub const GraphMetricResult = graph_metric_reader.Result; +pub const GraphMetricPointScoresResult = graph_metric_reader.PointScoresResult; +pub const GraphMetricPointScoreColumnsResult = graph_metric_reader.PointScoreColumnsResult; +pub const graphMetricScoreAlloc = graph_metric_reader.scoreAlloc; +pub const graphMetricScoresAlloc = graph_metric_reader.scoresAlloc; +pub const graphMetricScoreColumnsAlloc = graph_metric_reader.scoreColumnsAlloc; +pub const openGraphMetricAlloc = graph_metric_reader.openAlloc; +pub const graphMetricTopAlloc = graph_metric_reader.topAlloc; +pub const graphMetricTopWithLimitsAlloc = graph_metric_reader.topWithLimitsAlloc; pub const QuerySearchExecutionStats = indexed_reader.SearchExecutionStats; pub const parseSearchPlanAlloc = plan.parseSearchPlanAlloc; pub const parseGraphNeighborsPlanAlloc = plan.parseGraphNeighborsPlanAlloc; diff --git a/zig/pkg/antfly/src/serverless/query/runtime.zig b/zig/pkg/antfly/src/serverless/query/runtime.zig index cf8e60a56e..5fc130b5bb 100644 --- a/zig/pkg/antfly/src/serverless/query/runtime.zig +++ b/zig/pkg/antfly/src/serverless/query/runtime.zig @@ -19,11 +19,13 @@ const artifacts_mod = @import("../artifacts/mod.zig"); const catalog_mod = @import("../catalog/mod.zig"); const manifest_mod = @import("../manifest/mod.zig"); const graph_segment_mod = @import("../graph_segment/mod.zig"); +const graph_metric_config = @import("../build/graph_metric_config.zig"); const cache_mod = @import("cache.zig"); const bounded_decode = @import("../bounded_decode.zig"); const graph_reader = @import("graph_reader.zig"); const request_mod = @import("request.zig"); -const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; +const operation = @import("../../api/operation.zig"); +const CancellationToken = operation.CancellationToken; pub const QueryExecutionMetrics = struct { total_queries: u64 = 0, @@ -47,6 +49,207 @@ pub const NamespaceQueryExecutionMetrics = struct { } }; +pub const AuthenticatedSubrange = cache_mod.AuthenticatedSubrange; +pub const AuthenticatedBlockPublication = cache_mod.AuthenticatedBlockPublication; +pub const max_authenticated_publication_blocks = cache_mod.max_authenticated_publication_blocks; + +pub const GraphMetricReadLimits = struct { + /// Shared by every graph-metric surface in one pinned request. These are + /// deliberately aggregate limits, not per metric. + max_range_requests: u64 = 128, + max_range_bytes: u64 = 256 * 1024 * 1024, + max_decoded_blocks: u64 = 16 * 1024, + max_work_items: u64 = 32 * 1024 * 1024, + max_retained_bytes: u64 = 128 * 1024 * 1024, +}; + +pub const GraphMetricRangeCapacity = struct { requests: u64, bytes: u64 }; + +pub const GraphMetricReadBudget = struct { + mutex: std.atomic.Mutex = .unlocked, + limits: GraphMetricReadLimits = .{}, + range_requests: u64 = 0, + range_bytes: u64 = 0, + decoded_blocks: u64 = 0, + work_items: u64 = 0, + retained_bytes: u64 = 0, + + /// Move-only live-memory ownership. Unlike cumulative I/O/work admission, + /// scratch and replaced outputs release their capacity when destroyed. + /// The shared request budget must outlive every reservation. + pub const Reservation = struct { + budget: ?*GraphMetricReadBudget = null, + bytes: usize = 0, + + /// A read scope can collect conservative scratch/cache-lease charges + /// from concurrent children. Only its owner may split or destroy it, + /// after those children have joined. + pub fn grow(self: *@This(), bytes: usize) !void { + const budget = self.budget orelse return error.GraphMetricQueryBudgetExceeded; + lockAtomic(&budget.mutex); + defer budget.mutex.unlock(); + const owned = std.math.add(usize, self.bytes, bytes) catch return error.GraphMetricQueryBudgetExceeded; + const retained = try checkedCharge(budget.retained_bytes, bytes, budget.limits.max_retained_bytes); + self.bytes = owned; + budget.retained_bytes = retained; + } + + pub fn deinit(self: *@This()) void { + if (self.budget) |budget| { + lockAtomic(&budget.mutex); + std.debug.assert(budget.retained_bytes >= self.bytes); + budget.retained_bytes -= self.bytes; + budget.mutex.unlock(); + } + self.* = .{}; + } + + pub fn split(self: *@This(), bytes: usize) @This() { + std.debug.assert(bytes <= self.bytes); + self.bytes -= bytes; + return .{ .budget = self.budget, .bytes = bytes }; + } + + /// Move two exclusively owned reservations into one without dropping + /// admission between construction and publication. Not for shared + /// grow-only scopes until their workers have joined. + pub fn absorb(self: *@This(), other: *@This()) void { + std.debug.assert(self.budget == other.budget); + self.bytes += other.bytes; + other.* = .{}; + } + + pub fn shrinkTo(self: *@This(), bytes: usize) void { + std.debug.assert(bytes <= self.bytes); + var released = self.split(self.bytes - bytes); + released.deinit(); + } + + /// Escaping public output keeps its request charge, but must not keep + /// a pointer to a query session that can already have been destroyed. + pub fn detach(self: *@This()) void { + self.* = .{}; + } + }; + + pub fn reserveRetained(self: *@This(), bytes: usize) !Reservation { + try self.chargeRetained(bytes); + return .{ .budget = self, .bytes = bytes }; + } + + pub fn remainingMemory(self: *@This()) usize { + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + return std.math.cast(usize, self.limits.max_retained_bytes -| self.retained_bytes) orelse std.math.maxInt(usize); + } + + fn checkedCharge(current: u64, amount: u64, limit: u64) !u64 { + const next = std.math.add(u64, current, amount) catch return error.GraphMetricQueryBudgetExceeded; + if (next > limit) return error.GraphMetricQueryBudgetExceeded; + return next; + } + + pub fn chargeRange(self: *@This(), bytes: usize) !void { + return self.reserveRanges(1, bytes); + } + + pub fn reserveRanges(self: *@This(), requests: usize, bytes: usize) !void { + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + const next_requests = try checkedCharge(self.range_requests, @intCast(requests), self.limits.max_range_requests); + const next_bytes = try checkedCharge(self.range_bytes, @intCast(bytes), self.limits.max_range_bytes); + self.range_requests = next_requests; + self.range_bytes = next_bytes; + } + + pub fn remainingRequests(self: *@This()) u64 { + return self.remainingRanges().requests; + } + + pub fn remainingRanges(self: *@This()) GraphMetricRangeCapacity { + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + return .{ .requests = self.limits.max_range_requests -| self.range_requests, .bytes = self.limits.max_range_bytes -| self.range_bytes }; + } + + pub fn chargeDecode(self: *@This(), blocks: usize, work_items: usize) !void { + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + const next_blocks = try checkedCharge(self.decoded_blocks, @intCast(blocks), self.limits.max_decoded_blocks); + const next_items = try checkedCharge(self.work_items, @intCast(work_items), self.limits.max_work_items); + self.decoded_blocks = next_blocks; + self.work_items = next_items; + } + + pub fn chargeRetained(self: *@This(), bytes: usize) !void { + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + self.retained_bytes = try checkedCharge(self.retained_bytes, @intCast(bytes), self.limits.max_retained_bytes); + } +}; + +test "serverless graph metric request budget composes reads and rejects charges atomically" { + var budget = GraphMetricReadBudget{ .limits = .{ + .max_range_requests = 2, + .max_range_bytes = 10, + .max_decoded_blocks = 2, + .max_work_items = 10, + .max_retained_bytes = 10, + } }; + + try budget.chargeRange(6); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.chargeRange(5)); + try std.testing.expectEqual(@as(u64, 1), budget.range_requests); + try std.testing.expectEqual(@as(u64, 6), budget.range_bytes); + try budget.chargeRange(4); + + try budget.chargeDecode(1, 6); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.chargeDecode(2, 1)); + try std.testing.expectEqual(@as(u64, 1), budget.decoded_blocks); + try std.testing.expectEqual(@as(u64, 6), budget.work_items); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.chargeDecode(1, 5)); + try std.testing.expectEqual(@as(u64, 1), budget.decoded_blocks); + try std.testing.expectEqual(@as(u64, 6), budget.work_items); + + try budget.chargeRetained(10); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.chargeRetained(1)); + try std.testing.expectEqual(@as(u64, 10), budget.retained_bytes); +} + +test "serverless graph metric memory reservations transfer release and preserve work charges" { + var budget = GraphMetricReadBudget{ .limits = .{ .max_retained_bytes = 64 } }; + try budget.chargeDecode(1, 10); + var scratch = try budget.reserveRetained(48); + var output = scratch.split(16); + scratch.deinit(); + try std.testing.expectEqual(@as(u64, 16), budget.retained_bytes); + for (0..100) |_| { + var replacement = try budget.reserveRetained(48); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.reserveRetained(1)); + replacement.deinit(); + } + output.deinit(); + try std.testing.expectEqual(@as(u64, 0), budget.retained_bytes); + try std.testing.expectEqual(@as(u64, 10), budget.work_items); + var escaping = try budget.reserveRetained(8); + escaping.detach(); + escaping.deinit(); + try std.testing.expectEqual(@as(u64, 8), budget.retained_bytes); +} + +test "serverless graph metric score-plan reservation is atomic across ranges and bytes" { + var budget = GraphMetricReadBudget{ .limits = .{ .max_range_requests = 3, .max_range_bytes = 10 } }; + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.reserveRanges(2, 11)); + try std.testing.expectEqual(@as(u64, 0), budget.range_requests); + try std.testing.expectEqual(@as(u64, 0), budget.range_bytes); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.reserveRanges(4, 5)); + try std.testing.expectEqual(@as(u64, 3), budget.remainingRequests()); + try budget.reserveRanges(3, 10); + try std.testing.expectEqual(@as(u64, 0), budget.remainingRequests()); + try std.testing.expectError(error.GraphMetricQueryBudgetExceeded, budget.chargeRange(1)); + try std.testing.expectEqual(@as(u64, 10), budget.range_bytes); +} + pub const QueryRuntime = struct { alloc: Allocator, artifacts: *artifacts_mod.ArtifactStore, @@ -164,13 +367,48 @@ pub const QuerySession = struct { artifacts: *artifacts_mod.ArtifactStore, cache: ?*cache_mod.QueryCache = null, manifest: manifest_mod.Manifest, + owns_manifest: bool = true, + io: ?std.Io = null, cancellation: CancellationToken = .none, + diagnostics: ?*operation.RequestDiagnostics = null, + graph_metric_specs: ?[]graph_metric_config.IndexSpec = null, + owns_graph_metric_specs: bool = true, + graph_metric_read_budget: GraphMetricReadBudget = .{}, + graph_metric_read_budget_shared: ?*GraphMetricReadBudget = null, + // Borrowed read-lifetime scratch reservation, propagated to joined child + // reads. Output reservations are independent and may outlive this scope. + graph_metric_retained_scope: ?*GraphMetricReadBudget.Reservation = null, + // Pre-admitted transport workspace owned by a joined parent execution. + graph_metric_transport_credit: usize = 0, pub fn deinit(self: *QuerySession) void { - self.manifest.deinit(self.alloc); + if (self.owns_graph_metric_specs) self.clearGraphMetricSpecs(); + if (self.owns_manifest) self.manifest.deinit(self.alloc); self.* = undefined; } + /// Lazily parses graph metric configuration once for this pinned request. + /// QuerySession is request-owned and, like its other mutable caches, must + /// not be accessed concurrently without external synchronization. + pub fn graphMetricSpecs(self: *QuerySession) ![]const graph_metric_config.IndexSpec { + if (self.graph_metric_specs == null) { + self.graph_metric_specs = try graph_metric_config.parseIndexSpecsAlloc( + self.alloc, + self.manifest.stats.indexes_json, + ); + } + return self.graph_metric_specs.?; + } + + pub fn clearGraphMetricSpecs(self: *QuerySession) void { + if (!self.owns_graph_metric_specs) { + self.graph_metric_specs = null; + return; + } + if (self.graph_metric_specs) |specs| graph_metric_config.freeIndexSpecs(self.alloc, specs); + self.graph_metric_specs = null; + } + pub fn namespace(self: *const QuerySession) []const u8 { return self.manifest.namespace; } @@ -192,10 +430,83 @@ pub const QuerySession = struct { self.cancellation = cancellation; } + pub fn setDiagnostics(self: *QuerySession, diagnostics: ?*operation.RequestDiagnostics) void { + self.diagnostics = diagnostics; + } + + pub fn setIo(self: *QuerySession, io: ?std.Io) void { + self.io = io; + } + + /// Create a non-owning view of the pinned request for a concurrent range + /// fetch. Callers provide a thread-safe allocator; the manifest, cache, + /// cancellation token, and aggregate graph budget remain shared. + pub fn forkGraphMetricRead(self: *QuerySession, alloc: Allocator) QuerySession { + return .{ + .alloc = alloc, + .artifacts = self.artifacts, + .cache = self.cache, + .manifest = self.manifest, + .owns_manifest = false, + .io = self.io, + .cancellation = self.cancellation, + .diagnostics = null, + .graph_metric_specs = self.graph_metric_specs, + .owns_graph_metric_specs = false, + .graph_metric_read_budget_shared = self.effectiveGraphMetricReadBudget(), + .graph_metric_retained_scope = self.graph_metric_retained_scope, + .graph_metric_transport_credit = self.graph_metric_transport_credit, + }; + } + + fn effectiveGraphMetricReadBudget(self: *QuerySession) *GraphMetricReadBudget { + return self.graph_metric_read_budget_shared orelse &self.graph_metric_read_budget; + } + + pub fn recordGraphMetricRejection( + self: *QuerySession, + graph_index_name: []const u8, + metric_name: []const u8, + materializer_fingerprint: u64, + ) void { + const diagnostics = self.diagnostics orelse return; + diagnostics.recordGraphMetricRejection(graph_index_name, metric_name, materializer_fingerprint); + } + pub fn checkCancellation(self: *const QuerySession) !void { return self.cancellation.check(); } + pub fn chargeGraphMetricRange(self: *QuerySession, bytes: usize) !void { + return self.effectiveGraphMetricReadBudget().chargeRange(bytes); + } + + pub fn graphMetricRangeBudget(self: *QuerySession) GraphMetricRangeCapacity { + return self.effectiveGraphMetricReadBudget().remainingRanges(); + } + + pub fn reserveGraphMetricRanges(self: *QuerySession, requests: usize, bytes: usize) !void { + try self.checkCancellation(); + return self.effectiveGraphMetricReadBudget().reserveRanges(requests, bytes); + } + + pub fn chargeGraphMetricDecode(self: *QuerySession, blocks: usize, work_items: usize) !void { + return self.effectiveGraphMetricReadBudget().chargeDecode(blocks, work_items); + } + + pub fn chargeGraphMetricRetained(self: *QuerySession, bytes: usize) !void { + if (self.graph_metric_retained_scope) |scope| return scope.grow(bytes); + return self.effectiveGraphMetricReadBudget().chargeRetained(bytes); + } + + pub fn reserveGraphMetricMemory(self: *QuerySession, bytes: usize) !GraphMetricReadBudget.Reservation { + return self.effectiveGraphMetricReadBudget().reserveRetained(bytes); + } + + pub fn graphMetricMemoryAvailable(self: *QuerySession) usize { + return self.effectiveGraphMetricReadBudget().remainingMemory(); + } + pub fn findArtifactIndex(self: *const QuerySession, kind: manifest_mod.ArtifactKind) ?usize { for (self.manifest.artifacts, 0..) |artifact, idx| { if (artifact.kind == kind) return idx; @@ -237,14 +548,48 @@ pub const QuerySession = struct { return result; } + /// Authenticates an artifact against the manifest without materializing it + /// in the query allocator. Object and filesystem backends use their + /// bounded identity caches after the first full verification. + pub fn verifyArtifact(self: *QuerySession, index: usize) !void { + try self.checkCancellation(); + const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; + try validateArtifactForQuery(artifact); + try self.artifacts.verifyContentWithCancellationUsingAllocator( + self.alloc, + artifact.artifact_id, + artifact.byte_len, + artifact.checksum, + self.cancellation, + ); + try self.checkCancellation(); + } + pub fn fetchArtifactRangeAlloc(self: *QuerySession, index: usize, offset: u64, len: usize) ![]u8 { try self.checkCancellation(); const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; try validateArtifactRange(artifact, offset, len); const result = if (self.cache) |cache| - try cache.getRangeOrFetchAllocWithCancellationUsingAllocator(self.alloc, self.artifacts, artifact.artifact_id, offset, len, self.cancellation) + try cache.getVerifiedRangeOrFetchAllocWithCancellationUsingAllocator( + self.alloc, + self.artifacts, + artifact.artifact_id, + artifact.byte_len, + artifact.checksum, + offset, + len, + self.cancellation, + ) else - try self.artifacts.getRangeAllocWithCancellationUsingAllocator(self.alloc, artifact.artifact_id, offset, len, self.cancellation); + try self.artifacts.getVerifiedRangeAllocWithCancellationUsingAllocator( + self.alloc, + artifact.artifact_id, + artifact.byte_len, + artifact.checksum, + offset, + len, + self.cancellation, + ); errdefer self.alloc.free(result); try self.checkCancellation(); return result; @@ -255,10 +600,169 @@ pub const QuerySession = struct { const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; try validateArtifactRange(artifact, offset, len); const result = if (self.cache) |cache| - try cache.getBlockOrFetchRangeAllocWithCancellationUsingAllocator(self.alloc, self.artifacts, artifact.artifact_id, block_id, offset, len, self.cancellation) + try cache.getVerifiedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + self.alloc, + self.artifacts, + artifact.artifact_id, + block_id, + artifact.byte_len, + artifact.checksum, + offset, + len, + self.cancellation, + ) else - try self.artifacts.getRangeAllocWithCancellationUsingAllocator(self.alloc, artifact.artifact_id, offset, len, self.cancellation); + try self.artifacts.getVerifiedRangeAllocWithCancellationUsingAllocator( + self.alloc, + artifact.artifact_id, + artifact.byte_len, + artifact.checksum, + offset, + len, + self.cancellation, + ); + errdefer self.alloc.free(result); + try self.checkCancellation(); + return result; + } + + pub fn fetchArtifactAuthenticatedBlockAlloc( + self: *QuerySession, + index: usize, + block_id: []const u8, + offset: u64, + len: usize, + checksum: *const [std.crypto.hash.sha2.Sha256.digest_length]u8, + ) ![]u8 { + try self.checkCancellation(); + const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; + try validateArtifactRange(artifact, offset, len); + if (len == 0) return error.InvalidArtifactRange; + const result = if (self.cache) |cache| + try cache.getAuthenticatedBlockOrFetchRangeAllocWithCancellationUsingAllocator( + self.alloc, + self.artifacts, + artifact.artifact_id, + block_id, + artifact.byte_len, + artifact.checksum, + checksum, + offset, + len, + self.cancellation, + ) + else blk: { + const bytes = try self.artifacts.getRangeAllocWithCancellationUsingAllocator( + self.alloc, + artifact.artifact_id, + offset, + len, + self.cancellation, + ); + errdefer self.alloc.free(bytes); + if (bytes.len != len) return error.ArtifactIntegrityMismatch; + var actual: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &actual, .{}); + if (!std.mem.eql(u8, &actual, checksum)) return error.ArtifactIntegrityMismatch; + break :blk bytes; + }; + errdefer self.alloc.free(result); + try self.checkCancellation(); + return result; + } + + pub fn readCachedAuthenticatedBlockAlloc(self: *QuerySession, alloc: Allocator, index: usize, block_id: []const u8, offset: u64, len: usize, checksum: *const [32]u8) !?[]u8 { + var lease = (try self.readCachedAuthenticatedBlockLease(alloc, index, block_id, offset, len, checksum)) orelse return null; + if (lease == .owned) return lease.owned.data; + defer lease.deinit(); + return try alloc.dupe(u8, lease.bytes()); + } + + pub fn readCachedAuthenticatedBlockLease(self: *QuerySession, alloc: Allocator, index: usize, block_id: []const u8, offset: u64, len: usize, checksum: *const [32]u8) !?cache_mod.AuthenticatedBlockLease { + try self.checkCancellation(); + const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; + try validateArtifactRange(artifact, offset, len); + const cache = self.cache orelse return null; + return cache.readAuthenticatedBlockIfPresentLease(alloc, artifact.artifact_id, block_id, artifact.byte_len, artifact.checksum, checksum, offset, len, self.cancellation) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => return err, + // A damaged/unavailable local cache is a miss, never authority. + // The origin read still authenticates against the manifest digest. + else => null, + }; + } + + pub fn cacheAuthenticatedBlocks(self: *QuerySession, index: usize, blocks: []const AuthenticatedBlockPublication) !void { + const cache = self.cache orelse return; + const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; + cache.retainAuthenticatedBlocks(artifact.artifact_id, artifact.byte_len, artifact.checksum, blocks); + } + + /// Fetches a bounded range and authenticates every byte against digests + /// rooted in the published manifest or an already-authenticated routing + /// footer. Subranges must exactly and contiguously cover the response. + pub fn fetchArtifactAuthenticatedRangeAlloc( + self: *QuerySession, + index: usize, + offset: u64, + len: usize, + subranges: []const AuthenticatedSubrange, + ) ![]u8 { + return self.fetchAuthenticatedRangeAlloc(index, offset, len, subranges, true); + } + + pub fn fetchArtifactAuthenticatedRangeUncachedAlloc(self: *QuerySession, index: usize, offset: u64, len: usize, subranges: []const AuthenticatedSubrange) ![]u8 { + return self.fetchAuthenticatedRangeAlloc(index, offset, len, subranges, false); + } + + fn fetchAuthenticatedRangeAlloc(self: *QuerySession, index: usize, offset: u64, len: usize, subranges: []const AuthenticatedSubrange, retain_range: bool) ![]u8 { + try self.checkCancellation(); + const artifact = self.artifactRef(index) orelse return error.ArtifactNotFound; + try validateArtifactRange(artifact, offset, len); + if (len == 0 or subranges.len == 0) return error.InvalidArtifactRange; + var covered: usize = 0; + for (subranges) |subrange| { + if (subrange.len == 0 or subrange.relative_offset != covered) return error.InvalidArtifactRange; + covered = std.math.add(usize, covered, subrange.len) catch return error.InvalidArtifactRange; + if (covered > len) return error.InvalidArtifactRange; + } + if (covered != len) return error.InvalidArtifactRange; + + if (if (retain_range) self.cache else null) |cache| { + const result = try cache.getAuthenticatedRangeOrFetchAllocWithCancellationUsingAllocator( + self.alloc, + self.artifacts, + artifact.artifact_id, + artifact.byte_len, + artifact.checksum, + offset, + len, + subranges, + self.cancellation, + ); + errdefer self.alloc.free(result); + try self.checkCancellation(); + return result; + } + + const result = try self.artifacts.getRangeAllocWithCancellationUsingAllocator( + self.alloc, + artifact.artifact_id, + offset, + len, + self.cancellation, + ); errdefer self.alloc.free(result); + if (result.len != len) return error.ArtifactIntegrityMismatch; + for (subranges, 0..) |subrange, subrange_index| { + if (subrange_index % 64 == 0) try self.checkCancellation(); + var actual: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash( + result[subrange.relative_offset..][0..subrange.len], + &actual, + .{}, + ); + if (!std.mem.eql(u8, &actual, &subrange.checksum)) return error.ArtifactIntegrityMismatch; + } try self.checkCancellation(); return result; } diff --git a/zig/pkg/antfly/src/serverless/runtime/bootstrap.zig b/zig/pkg/antfly/src/serverless/runtime/bootstrap.zig index d39797fb9c..77dde1472a 100644 --- a/zig/pkg/antfly/src/serverless/runtime/bootstrap.zig +++ b/zig/pkg/antfly/src/serverless/runtime/bootstrap.zig @@ -56,6 +56,8 @@ pub const BootstrapConfig = struct { query_cache_dir: ?[]const u8 = null, query_cache_max_bytes: u64 = 4 * 1024 * 1024 * 1024, query_cache_payload_max_bytes: u64 = 64 * 1024 * 1024, + /// Only the current wire is supported while serverless is unreleased. + manifest_write_version: u16 = manifest_mod.codec.wire_version, embedding_indexes_json: ?[]const u8 = null, sparse_embedding_index_name: []const u8 = search_sources.default_sparse_embedding_index_name, chunk_embedding_index_name: []const u8 = search_sources.default_chunk_embedding_index_name, @@ -76,6 +78,10 @@ pub const BootstrapConfig = struct { query_max_concurrent_requests: u32 = common_config.default_query_max_concurrent_requests, graph_execution_limits: @import("../../graph/work_budget.zig").Limits = .{}, write_max_concurrent_requests: u32 = common_config.default_write_max_concurrent_requests, + /// CPU fanout available to one graph-metric kernel. Work is scheduled on + /// the shared std.Io backend, so deployments can align this with their + /// runtime capacity instead of materializers creating private threads. + graph_metric_max_parallelism: u8 = @intCast(build_mod.default_graph_metric_compute_parallelism), }; pub const RuntimeStatus = api_mod.RuntimeStatusResult; @@ -487,6 +493,7 @@ pub const OwnedStack = struct { try ensureConfiguredBucket(client, target.bucket, shouldCreateGcsBucket(cfg.gcs_options[1])); break :blk try manifest_object_store.ObjectStore.initWithClient(alloc, client, target.bucket, target.prefix); } else try manifest_object_store.ObjectStore.initRemoteUriWithS3Options(alloc, cfg.manifests_uri, cfg.s3_options[1]); + try self.manifests_impl.setWriteVersion(cfg.manifest_write_version); self.manifests = self.manifests_impl.manifestStore(); errdefer self.manifests.deinit(); @@ -546,6 +553,8 @@ pub const OwnedStack = struct { errdefer self.catalog_store.deinit(); self.builder = build_mod.Builder.init(alloc, &self.artifacts, &self.manifests, &self.progress, &self.wal); + self.builder.setIo(io); + try self.builder.setGraphMetricMaxParallelism(cfg.graph_metric_max_parallelism); self.catalog = catalog_mod.CatalogService.init(alloc, &self.artifacts, &self.manifests, &self.progress, &self.wal, &self.builder, &self.catalog_store); self.external_source_object_store_resolver = .{}; self.external_source_object_store_resolver.configure(cfg.node_config, cfg.secret_store); @@ -567,7 +576,13 @@ pub const OwnedStack = struct { self.query_cache = null; self.query = query_mod.QueryRuntime.init(alloc, &self.artifacts, &self.manifests, &self.progress); } + var query_cache_owned = self.query_cache != null; + errdefer if (query_cache_owned) self.query_cache.?.deinit(); + var query_owned = true; + errdefer if (query_owned) self.query.deinit(); self.status = try runtimeStatusAlloc(alloc, cfg); + var status_owned = true; + errdefer if (status_owned) self.status.deinit(alloc); self.runtime = runtime_manager.ManagedRuntime.initWithIo(alloc, io, .{ .tick_interval_ms = cfg.tick_interval_ms, .role = cfg.role, @@ -582,6 +597,8 @@ pub const OwnedStack = struct { .remote_content = cfg.remote_content, }, }, &self.catalog, build_mod.Pruner.init(alloc, &self.artifacts, &self.manifests, &self.progress, &self.wal)); + var runtime_owned = true; + errdefer if (runtime_owned) self.runtime.deinit(); const generated_lease_owner = if (cfg.work_lease_owner_id == null) try workLeaseOwnerIdAlloc(alloc, io) else @@ -595,6 +612,12 @@ pub const OwnedStack = struct { ); self.runtime.setCompactor(build_mod.Compactor.init(alloc, &self.artifacts, &self.manifests, &self.progress)); var enricher = enrichment_mod.SparseEnricher.init(alloc, &self.artifacts, &self.manifests, &self.progress, &self.wal); + var enricher_owned = true; + errdefer if (enricher_owned) enricher.deinit(); + var pending_query_embedder: ?managed_embedder.ManagedEmbedder = null; + errdefer if (pending_query_embedder) |*query_embedder| query_embedder.deinit(); + var pending_dense_query_index_name: ?[]u8 = null; + errdefer if (pending_dense_query_index_name) |index_name| alloc.free(index_name); if (cfg.embedding_indexes_json) |indexes_json| { const embedder_options = managed_embedder.InitOptions{ .io = io, @@ -603,28 +626,48 @@ pub const OwnedStack = struct { .remote_content = cfg.remote_content, .provider_runtime = &self.embedding_provider_runtime, }; - var query_embedder = try managed_embedder.ManagedEmbedder.initFromIndexesJsonWithOptions(alloc, indexes_json, embedder_options); - errdefer query_embedder.deinit(); + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, indexes_json, .{}); + defer parsed.deinit(); + var query_embedder = try managed_embedder.ManagedEmbedder.initDenseFromIndexValueObjectWithOptions(alloc, parsed.value, embedder_options); + var query_embedder_owned = true; + errdefer if (query_embedder_owned) query_embedder.deinit(); if (query_embedder.hasDenseEntries()) { - self.managed_query_embedder = query_embedder; - self.dense_query_index_name = try alloc.dupe(u8, cfg.chunk_embedding_index_name); + pending_query_embedder = query_embedder; + query_embedder_owned = false; + pending_dense_query_index_name = try alloc.dupe(u8, cfg.chunk_embedding_index_name); } else { query_embedder.deinit(); - self.managed_query_embedder = null; - self.dense_query_index_name = null; + query_embedder_owned = false; } - if (try managed_embedder.ManagedEmbedder.createSparseEmbedderWithOptions(alloc, indexes_json, embedder_options)) |sparse_embedder| { + if (try managed_embedder.ManagedEmbedder.createSparseEmbedderFromIndexValueWithOptions(alloc, parsed.value, embedder_options)) |sparse_embedder| { + var embedder_owned = true; + errdefer if (embedder_owned) sparse_embedder.deinit(alloc); try enricher.setSparseEmbedder(sparse_embedder, cfg.sparse_embedding_index_name); + embedder_owned = false; } - if (try managed_embedder.ManagedEmbedder.createDenseEmbedderWithOptions(alloc, indexes_json, embedder_options)) |dense_embedder| { + if (try managed_embedder.ManagedEmbedder.createDenseEmbedderFromIndexValueWithOptions(alloc, parsed.value, embedder_options)) |dense_embedder| { + var embedder_owned = true; + errdefer if (embedder_owned) dense_embedder.deinit(alloc); try enricher.setChunkEmbedder(dense_embedder, cfg.chunk_embedding_index_name, cfg.chunk_embedding_dimensions); + embedder_owned = false; } - } else { - self.managed_query_embedder = null; - self.dense_query_index_name = null; } - self.sparse_query_index_name = try alloc.dupe(u8, cfg.sparse_embedding_index_name); + const sparse_query_index_name = try alloc.dupe(u8, cfg.sparse_embedding_index_name); + var sparse_query_index_name_owned = true; + errdefer if (sparse_query_index_name_owned) alloc.free(sparse_query_index_name); + + // Commit the interdependent runtime resources only after every + // fallible allocation and provider construction has succeeded. This + // keeps init failure cleanup local and prevents partially initialized + // fields from leaking or being observed by the handler. + self.managed_query_embedder = pending_query_embedder; + pending_query_embedder = null; + self.dense_query_index_name = pending_dense_query_index_name; + pending_dense_query_index_name = null; + self.sparse_query_index_name = sparse_query_index_name; + sparse_query_index_name_owned = false; self.runtime.setEnricher(enricher); + enricher_owned = false; self.handler = api_mod.HttpHandler.init(alloc, &self.api, &self.catalog, &self.manifests, &self.progress, &self.query, &self.status); try self.handler.setGraphExecutionLimits(cfg.graph_execution_limits); self.handler.setIo(io); @@ -655,6 +698,10 @@ pub const OwnedStack = struct { self.handler.setManagedDenseQueryEmbedder(query_embedder, self.dense_query_index_name.?); } self.handler.setRuntimeMetrics(&self.runtime); + runtime_owned = false; + status_owned = false; + query_owned = false; + query_cache_owned = false; } pub fn deinit(self: *OwnedStack) void { @@ -694,6 +741,11 @@ fn workLeaseOwnerIdAlloc(alloc: Allocator, io: std.Io) ![]u8 { pub fn validateConfig(alloc: Allocator, cfg: BootstrapConfig) !void { if (cfg.tick_interval_ms == 0) return error.InvalidTickInterval; + if (cfg.graph_metric_max_parallelism == 0 or cfg.graph_metric_max_parallelism > build_mod.max_graph_metric_compute_parallelism) + return error.InvalidGraphMetricBuildOptions; + if (cfg.manifest_write_version != manifest_mod.codec.wire_version) { + return error.UnsupportedManifestWriteVersion; + } if (cfg.work_lease_ttl_ms == 0) return error.InvalidWorkLeaseTtl; if (cfg.work_lease_owner_id) |owner_id| { if (std.mem.trim(u8, owner_id, &std.ascii.whitespace).len == 0) { @@ -881,6 +933,62 @@ test "runtime bootstrap assembles serverless stack from uri config" { try std.testing.expect(std.mem.indexOf(u8, resp.body, "\"doc_id\":\"doc-a\"") != null); } +test "serverless runtime bootstrap failure unwinds query cache and runtime ownership" { + const alloc = std.testing.allocator; + + var artifacts_buf: [256]u8 = undefined; + var manifests_buf: [256]u8 = undefined; + var wal_buf: [256]u8 = undefined; + var progress_buf: [256]u8 = undefined; + var catalog_buf: [256]u8 = undefined; + var cache_buf: [256]u8 = undefined; + const artifacts_root = tmpPath(&artifacts_buf, "bootstrap-failure-artifacts"); + const manifests_root = tmpPath(&manifests_buf, "bootstrap-failure-manifests"); + const wal_root = tmpPath(&wal_buf, "bootstrap-failure-wal"); + const progress_root = tmpPath(&progress_buf, "bootstrap-failure-progress"); + const catalog_root = tmpPath(&catalog_buf, "bootstrap-failure-catalog"); + const cache_root = tmpPath(&cache_buf, "bootstrap-failure-cache"); + defer cleanupTmp(artifacts_root); + defer cleanupTmp(manifests_root); + defer cleanupTmp(wal_root); + defer cleanupTmp(progress_root); + defer cleanupTmp(catalog_root); + defer cleanupTmp(cache_root); + + const artifacts_uri = try std.fmt.allocPrint(alloc, "file://{s}", .{std.mem.span(artifacts_root)}); + defer alloc.free(artifacts_uri); + const manifests_uri = try std.fmt.allocPrint(alloc, "file://{s}", .{std.mem.span(manifests_root)}); + defer alloc.free(manifests_uri); + const wal_uri = try std.fmt.allocPrint(alloc, "file://{s}", .{std.mem.span(wal_root)}); + defer alloc.free(wal_uri); + const progress_uri = try std.fmt.allocPrint(alloc, "file://{s}", .{std.mem.span(progress_root)}); + defer alloc.free(progress_uri); + const catalog_uri = try std.fmt.allocPrint(alloc, "file://{s}", .{std.mem.span(catalog_root)}); + defer alloc.free(catalog_uri); + + const base = BootstrapConfig{ + .artifacts_uri = artifacts_uri, + .manifests_uri = manifests_uri, + .wal_uri = wal_uri, + .progress_uri = progress_uri, + .catalog_uri = catalog_uri, + .query_cache_dir = std.mem.span(cache_root), + .query_cache_max_bytes = 1024 * 1024, + .query_cache_payload_max_bytes = 256 * 1024, + .tick_interval_ms = 1, + }; + var invalid = base; + invalid.embedding_indexes_json = "{"; + + var failed_stack: OwnedStack = undefined; + try std.testing.expectError(error.UnexpectedEndOfInput, failed_stack.init(alloc, invalid, std.testing.io)); + + // A clean retry exercises file/lease release as well as allocator cleanup. + var retry_stack: OwnedStack = undefined; + try retry_stack.init(alloc, base, std.testing.io); + retry_stack.deinit(); +} + test "runtime bootstrap wires foreign registry into public join handler" { const alloc = std.testing.allocator; @@ -1305,6 +1413,29 @@ test "runtime bootstrap requires bounded query cache budgets" { try std.testing.expectError(error.QueryCachePayloadExceedsBudget, validateConfig(alloc, invalid)); } +test "runtime bootstrap requires bounded graph metric parallelism and current manifest wire" { + const alloc = std.testing.allocator; + const base = BootstrapConfig{ + .artifacts_uri = "file:///tmp/antfly-artifacts", + .manifests_uri = "file:///tmp/antfly-manifests", + .wal_uri = "file:///tmp/antfly-wal", + .progress_uri = "file:///tmp/antfly-progress", + .catalog_uri = "file:///tmp/antfly-catalog", + }; + try validateConfig(alloc, base); + var invalid = base; + invalid.graph_metric_max_parallelism = 0; + try std.testing.expectError(error.InvalidGraphMetricBuildOptions, validateConfig(alloc, invalid)); + invalid.graph_metric_max_parallelism = 17; + try std.testing.expectError(error.InvalidGraphMetricBuildOptions, validateConfig(alloc, invalid)); + try std.testing.expectEqual(manifest_mod.codec.wire_version, base.manifest_write_version); + invalid = base; + for ([_]u16{ 12, manifest_mod.codec.wire_version - 1, manifest_mod.codec.wire_version + 1 }) |version| { + invalid.manifest_write_version = version; + try std.testing.expectError(error.UnsupportedManifestWriteVersion, validateConfig(alloc, invalid)); + } +} + test "runtime bootstrap bucket provisioning fails closed" { const alloc = std.testing.allocator; var memory = objectstore.MemoryClient.init(alloc); diff --git a/zig/pkg/antfly/src/serverless/runtime/manager.zig b/zig/pkg/antfly/src/serverless/runtime/manager.zig index be86d83e88..8481f084e1 100644 --- a/zig/pkg/antfly/src/serverless/runtime/manager.zig +++ b/zig/pkg/antfly/src/serverless/runtime/manager.zig @@ -14,7 +14,9 @@ const std = @import("std"); const platform_sync = @import("antfly_platform").sync; +const platform_time = @import("antfly_platform").time; const Allocator = std.mem.Allocator; +const CancellationToken = @import("../../common/cancellation.zig").CancellationToken; const api_mod = @import("../api/mod.zig"); const build_mod = @import("../build/mod.zig"); const catalog_mod = @import("../catalog/mod.zig"); @@ -195,14 +197,22 @@ pub const ManagedRuntime = struct { } pub fn runOnce(self: *ManagedRuntime) !RuntimeRunStats { + return try self.runOnceWithCancellation(.none); + } + + pub fn runOnceWithCancellation(self: *ManagedRuntime, cancellation: CancellationToken) !RuntimeRunStats { if (self.cfg.role == .query_only or self.cfg.role == .api_only) return RuntimeRunStats{}; - lockAtomic(&self.run_mu); + var pass = self.cancellationToken(); + pass.cooperative = cancellation; + var bridge = maintenance_cancellation.GraphBridge{ .maintenance = pass }; + try pass.check(); + try lockAtomicWithCancellation(&self.run_mu, bridge.token()); defer self.run_mu.unlock(); var stats = RuntimeRunStats{}; if (self.cfg.publish_enabled) { try self.chargeWork(.publish_round, 1); - const publish_stats = try self.publisher.runOnceUntil(&self.stop_requested); + const publish_stats = try self.publisher.runOnceWithCancellation(bridge.token()); stats.published_namespaces = publish_stats.published_namespaces; stats.publish_head_conflicts = publish_stats.head_conflicts; stats.work_lease_conflicts += publish_stats.lease_conflicts; @@ -214,6 +224,7 @@ pub const ManagedRuntime = struct { defer self.catalog.freeNamespaces(self.alloc, namespaces); for (namespaces) |namespace| { + try cancellation.check(); try self.cancellationCheckpoint(); const policy = self.catalog.getPolicy(namespace.name) catch |err| switch (err) { error.FileNotFound => continue, @@ -224,103 +235,7 @@ pub const ManagedRuntime = struct { else => return err, }; defer status.deinit(self.alloc); - var effective_policy = policy; - effective_policy.enrichment_enabled = status.enrichment_enabled; - effective_policy.chunk_preview_enabled = status.chunk_preview_enabled; - effective_policy.chunk_embeddings_enabled = status.chunk_embeddings_enabled; - effective_policy.rerank_terms_enabled = status.rerank_terms_enabled; - - if (self.enricher) |*enricher| { - try self.cancellationCheckpoint(); - const maybe_table_record = self.catalog.getTableForNamespaceAlloc(self.alloc, namespace.name) catch |err| switch (err) { - error.FileNotFound => null, - else => return err, - }; - if (maybe_table_record) |table_record| { - var table = table_record; - defer table.deinit(self.alloc); - - if (try managed_embedder.ManagedEmbedder.createSparseEmbedderWithOptions(self.alloc, table.indexes_json, self.cfg.embedder_options)) |sparse_embedder| { - var parsed = try std.json.parseFromSlice(std.json.Value, self.alloc, table.indexes_json, .{}); - defer parsed.deinit(); - const sparse_name = firstSparseIndexNameFromIndexesJson(parsed.value) orelse "serverless_sparse"; - try enricher.setSparseEmbedder(sparse_embedder, sparse_name); - } else { - enricher.clearSparseEmbedder(); - } - - if (try managed_embedder.ManagedEmbedder.createDenseEmbedderWithOptions(self.alloc, table.indexes_json, self.cfg.embedder_options)) |dense_embedder| { - var parsed = try std.json.parseFromSlice(std.json.Value, self.alloc, table.indexes_json, .{}); - defer parsed.deinit(); - const dims = denseDimsFromIndexesJson(parsed.value) orelse 8; - const dense_name = firstDenseIndexNameFromIndexesJson(parsed.value) orelse "serverless_chunk"; - try enricher.setChunkEmbedder(dense_embedder, dense_name, dims); - } else { - enricher.clearChunkEmbedder(); - } - } - - if (self.cfg.enrichment_enabled and status.enrichment_active_stage != null) { - const stage_spec = enrichment_mod.builtinPipelineForPolicy(effective_policy).stageSpec(status.enrichment_active_stage.?) orelse continue; - var held_enrichment_lease: ?build_mod.work_lease.HeldLease = null; - var can_enrich = true; - if (self.work_lease_provider) |provider| { - held_enrichment_lease = try build_mod.work_lease.acquireHeld( - provider, - self.io, - namespace.name, - self.work_lease_owner_id orelse return error.MissingLeaseOwner, - self.work_lease_ttl_ns, - ); - if (held_enrichment_lease == null) { - stats.enrichment_conflicts += 1; - stats.work_lease_conflicts += 1; - can_enrich = false; - } else if (held_enrichment_lease.?.acquisition.took_over) { - stats.work_lease_takeovers += 1; - } - } - defer if (held_enrichment_lease) |*lease| { - _ = lease.release() catch {}; - }; - - if (can_enrich) { - try self.chargeWork(.enrichment_round, 1); - const enrichment_cancellation = if (held_enrichment_lease) |*lease| - lease.cancellation(self.cancellationToken()) - else - self.cancellationToken(); - const maybe_enrichment = enricher.runNamespaceWithConfigUntil(namespace.name, .{ - .batch_size = policy.enrichment_batch_size, - .pipeline_version = stage_spec.pipeline_version, - .stage = status.enrichment_active_stage.?, - .model_preference = stage_spec.model_preference, - .failure_policy = policy.enrichment_failure_policy, - }, enrichment_cancellation) catch |err| switch (err) { - error.FileNotFound => null, - error.EnrichmentProgressChanged => blk: { - stats.enrichment_conflicts += 1; - break :blk null; - }, - error.WorkLeaseLost => blk: { - stats.enrichment_conflicts += 1; - stats.work_lease_conflicts += 1; - break :blk null; - }, - else => return err, - }; - if (maybe_enrichment) |enrichment| { - stats.enriched_namespaces += enrichment.enriched_namespaces; - stats.enriched_documents += enrichment.enriched_documents; - stats.enrichment_wal_appends += enrichment.wal_appends; - stats.enrichment_model_documents += enrichment.model_documents; - stats.enrichment_fallback_documents += enrichment.fallback_documents; - stats.enrichment_failed_documents += enrichment.failed_documents; - stats.enrichment_stage_failures += enrichment.stage_failures; - } - } - } - } + if (!try self.enrichNamespaceWithStatus(namespace.name, policy, status, cancellation, &stats)) continue; if (self.compactor) |*compactor| { try self.cancellationCheckpoint(); @@ -351,9 +266,9 @@ pub const ManagedRuntime = struct { else null; const compaction_cancellation = if (held_lease) |*lease| - lease.cancellation(self.cancellationToken()) + lease.cancellation(pass) else - self.cancellationToken(); + pass; try self.chargeWork(.compaction_round, 1); var compacted = compactor.compactHeadGuardedUntil( namespace.name, @@ -383,7 +298,7 @@ pub const ManagedRuntime = struct { var result = self.pruner.pruneNamespaceUntil( namespace.name, policy.keep_latest_versions, - self.cancellationToken(), + pass, ) catch |err| switch (err) { error.FileNotFound => continue, else => return err, @@ -401,6 +316,40 @@ pub const ManagedRuntime = struct { return stats; } + /// Runs only the request-visible background materialization stage for one + /// namespace. It deliberately excludes publication, compaction, pruning, + /// and catalog-wide enumeration; those remain owned by the background + /// runtime and must not be amplified by synchronous HTTP requests. + pub fn runNamespaceMaterializationOnceWithCancellation( + self: *ManagedRuntime, + namespace: []const u8, + cancellation: CancellationToken, + ) !RuntimeRunStats { + if (!self.supportsSynchronousMaterialization()) return error.MaterializationUnavailable; + if (namespace.len == 0) return error.InvalidNamespace; + try cancellation.check(); + try lockAtomicWithCancellation(&self.run_mu, cancellation); + defer self.run_mu.unlock(); + + const policy = try self.catalog.getPolicy(namespace); + var status = try self.catalog.buildStatus(namespace); + defer status.deinit(self.alloc); + var stats = RuntimeRunStats{}; + if (!try self.enrichNamespaceWithStatus(namespace, policy, status, cancellation, &stats)) { + return error.MaterializationUnavailable; + } + try cancellation.check(); + self.recordStats(stats); + return stats; + } + + pub fn supportsSynchronousMaterialization(self: *const ManagedRuntime) bool { + return self.cfg.role != .query_only and + self.cfg.role != .api_only and + self.cfg.enrichment_enabled and + self.enricher != null; + } + pub fn metricsSnapshot(self: *ManagedRuntime) RuntimeRunStats { lockAtomic(&self.stats_mu); defer self.stats_mu.unlock(); @@ -415,6 +364,129 @@ pub const ManagedRuntime = struct { self.enricher = enricher; } + /// Returns false when the namespace disappeared or its active stage is no + /// longer representable, matching the full-sweep behavior of skipping the + /// remainder of that namespace for this tick. + fn enrichNamespaceWithStatus( + self: *ManagedRuntime, + namespace: []const u8, + policy: catalog_mod.NamespacePolicy, + status: catalog_mod.BuildStatus, + cancellation: CancellationToken, + stats: *RuntimeRunStats, + ) !bool { + try cancellation.check(); + if (!self.cfg.enrichment_enabled or status.enrichment_active_stage == null) return true; + const enricher = if (self.enricher) |*value| value else return true; + + var effective_policy = policy; + effective_policy.enrichment_enabled = status.enrichment_enabled; + effective_policy.chunk_preview_enabled = status.chunk_preview_enabled; + effective_policy.chunk_embeddings_enabled = status.chunk_embeddings_enabled; + effective_policy.rerank_terms_enabled = status.rerank_terms_enabled; + const stage = status.enrichment_active_stage.?; + const stage_spec = enrichment_mod.builtinPipelineForPolicy(effective_policy).stageSpec(stage) orelse return false; + + var held_lease: ?build_mod.work_lease.HeldLease = null; + if (self.work_lease_provider) |provider| { + held_lease = try build_mod.work_lease.acquireHeld(provider, self.io, namespace, self.work_lease_owner_id orelse return error.MissingLeaseOwner, self.work_lease_ttl_ns); + if (held_lease == null) { + stats.enrichment_conflicts += 1; + stats.work_lease_conflicts += 1; + return true; + } + if (held_lease.?.acquisition.took_over) stats.work_lease_takeovers += 1; + } + defer if (held_lease) |*lease| { + _ = lease.release() catch {}; + }; + var maintenance = self.cancellationToken(); + maintenance.cooperative = cancellation; + if (held_lease) |*lease| maintenance = lease.cancellation(maintenance); + try self.chargeWork(.enrichment_round, 1); + try maintenance.check(); + + const maybe_table_record = self.catalog.getTableForNamespaceAlloc(self.alloc, namespace) catch |err| switch (err) { + error.FileNotFound => null, + else => return err, + }; + if (maybe_table_record) |table_record| { + var table = table_record; + defer table.deinit(self.alloc); + + // Only model-backed stages need an embedder. Parse once and build + // only the relevant provider set so deterministic stages avoid + // configuration churn and an unrelated provider cannot block the + // active stage. + switch (stage) { + .lexical_sparse => { + var parsed = try std.json.parseFromSlice(std.json.Value, self.alloc, table.indexes_json, .{}); + defer parsed.deinit(); + if (try managed_embedder.ManagedEmbedder.createSparseEmbedderFromIndexValueWithOptions(self.alloc, parsed.value, self.cfg.embedder_options)) |sparse_embedder| { + const sparse_name = firstSparseIndexNameFromIndexesJson(parsed.value) orelse "serverless_sparse"; + var embedder_owned = true; + errdefer if (embedder_owned) sparse_embedder.deinit(self.alloc); + try enricher.setSparseEmbedder(sparse_embedder, sparse_name); + embedder_owned = false; + } else { + enricher.clearSparseEmbedder(); + } + }, + .chunk_embeddings => { + var parsed = try std.json.parseFromSlice(std.json.Value, self.alloc, table.indexes_json, .{}); + defer parsed.deinit(); + if (try managed_embedder.ManagedEmbedder.createDenseEmbedderFromIndexValueWithOptions(self.alloc, parsed.value, self.cfg.embedder_options)) |dense_embedder| { + const dims = denseDimsFromIndexesJson(parsed.value) orelse 8; + const dense_name = firstDenseIndexNameFromIndexesJson(parsed.value) orelse "serverless_chunk"; + var embedder_owned = true; + errdefer if (embedder_owned) dense_embedder.deinit(self.alloc); + try enricher.setChunkEmbedder(dense_embedder, dense_name, dims); + embedder_owned = false; + } else { + enricher.clearChunkEmbedder(); + } + }, + .chunk_preview, .rerank_terms => {}, + } + } else { + // Namespace-only records predate the table catalog. Preserve their + // fallback enrichment behavior without retaining a previous + // namespace's managed embedder configuration. + enricher.clearSparseEmbedder(); + enricher.clearChunkEmbedder(); + } + + try cancellation.check(); + const enrichment = enricher.runNamespaceWithConfigUntil(namespace, .{ + .batch_size = policy.enrichment_batch_size, + .pipeline_version = stage_spec.pipeline_version, + .stage = stage, + .model_preference = stage_spec.model_preference, + .failure_policy = policy.enrichment_failure_policy, + .cancellation = cancellation, + }, maintenance) catch |err| switch (err) { + error.FileNotFound => return false, + error.EnrichmentProgressChanged => { + stats.enrichment_conflicts += 1; + return true; + }, + error.WorkLeaseLost => { + stats.enrichment_conflicts += 1; + stats.work_lease_conflicts += 1; + return true; + }, + else => return err, + }; + stats.enriched_namespaces += enrichment.enriched_namespaces; + stats.enriched_documents += enrichment.enriched_documents; + stats.enrichment_wal_appends += enrichment.wal_appends; + stats.enrichment_model_documents += enrichment.model_documents; + stats.enrichment_fallback_documents += enrichment.fallback_documents; + stats.enrichment_failed_documents += enrichment.failed_documents; + stats.enrichment_stage_failures += enrichment.stage_failures; + return true; + } + pub fn setWorkCostPort(self: *ManagedRuntime, port: ?RuntimeWorkCostPort) void { self.work_cost_port = port; } @@ -923,7 +995,7 @@ test "managed runtime compacts head when namespace exceeds compaction threshold" try std.testing.expectEqual(manifest_mod.ArtifactKind.text_segment, compacted.artifacts[1].kind); } -test "managed runtime runs sparse enrichment for opted-in namespaces" { +test "serverless managed runtime targets request-driven enrichment without global maintenance" { const alloc = std.testing.allocator; var artifact_root_buf: [256]u8 = undefined; @@ -962,10 +1034,17 @@ test "managed runtime runs sparse enrichment for opted-in namespaces" { var builder = @import("../build/builder.zig").Builder.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store); var catalog = catalog_mod.CatalogService.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store, &builder, &catalog_store); defer catalog.deinit(); - try std.testing.expect(try catalog.ensureNamespaceWithPolicy("docs", 100, .{ - .enrichment_enabled = true, - .keep_latest_versions = 2, - })); + try std.testing.expect(try catalog.ensureTableWithDefinition( + "docs", + 100, + .{ + .enrichment_enabled = true, + .keep_latest_versions = 2, + }, + "", + "", + "{}", + )); var api = @import("../api/service.zig").Service.init(alloc, &wal_store, &builder); const batch = [_]@import("../api/types.zig").DocumentMutation{ @@ -973,14 +1052,18 @@ test "managed runtime runs sparse enrichment for opted-in namespaces" { }; var ingest = try api.ingestBatch(.{ .namespace = "docs", .timestamp_ns = 100, .mutations = &batch }); defer ingest.deinit(alloc); - var build = try builder.publishNamespace("docs"); + var build = try catalog.buildNamespace("docs"); defer build.deinit(alloc); var runtime = ManagedRuntime.init(alloc, std.testing.io, .{ .tick_interval_ms = 1 }, &catalog, build_mod.Pruner.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store)); runtime.setEnricher(enrichment_mod.SparseEnricher.init(alloc, &artifact_store, &manifest_store, &progress_store, &wal_store)); defer runtime.deinit(); - const stats = try runtime.runOnce(); + try std.testing.expect(runtime.supportsSynchronousMaterialization()); + const stats = try runtime.runNamespaceMaterializationOnceWithCancellation("docs", .none); + try std.testing.expectEqual(@as(usize, 0), stats.published_namespaces); + try std.testing.expectEqual(@as(usize, 0), stats.compacted_namespaces); + try std.testing.expectEqual(@as(usize, 0), stats.pruned_namespaces); try std.testing.expectEqual(@as(usize, 1), stats.enriched_namespaces); try std.testing.expectEqual(@as(usize, 1), stats.enriched_documents); try std.testing.expectEqual(@as(usize, 1), stats.enrichment_wal_appends); @@ -988,6 +1071,7 @@ test "managed runtime runs sparse enrichment for opted-in namespaces" { const tail = try wal_store.readFromAlloc("docs", 2); defer @import("../wal/mod.zig").freeRecords(alloc, tail); try std.testing.expectEqual(@as(usize, 1), tail.len); + try std.testing.expectEqual(@as(u64, 1), try progress_store.getHead("docs")); } var test_nonce: std.atomic.Value(u64) = .init(0); @@ -1022,3 +1106,10 @@ fn cleanupTmp(path: [*:0]const u8) void { fn lockAtomic(mutex: *std.atomic.Mutex) void { platform_sync.lockYielding(mutex); } + +fn lockAtomicWithCancellation(mutex: *std.atomic.Mutex, cancellation: CancellationToken) !void { + while (!mutex.tryLock()) { + try cancellation.check(); + platform_time.yieldBriefly(); + } +} diff --git a/zig/pkg/antfly/src/serverless/segment/sidecar_manifest.zig b/zig/pkg/antfly/src/serverless/segment/sidecar_manifest.zig index a319ed833d..fca0057a35 100644 --- a/zig/pkg/antfly/src/serverless/segment/sidecar_manifest.zig +++ b/zig/pkg/antfly/src/serverless/segment/sidecar_manifest.zig @@ -49,7 +49,9 @@ pub const Manifest = struct { if (std.mem.eql(u8, previous.name, artifact.name)) { return error.DuplicateSidecarArtifactDeclaration; } - if (std.mem.eql(u8, previous.artifact.artifact_id, artifact.artifact.artifact_id)) { + if (std.mem.eql(u8, previous.artifact.artifact_id, artifact.artifact.artifact_id) and + !artifact_ref.areGraphArtifactAliases(previous.artifact, artifact.artifact)) + { return error.DuplicateSidecarArtifactDeclaration; } } @@ -71,6 +73,7 @@ pub fn artifactKindForSidecarKind(kind: source_binding.SidecarKind) artifact_ref .sparse => .sparse_segment, .graph => .graph_segment, .algebraic => .algebraic_segment, + .graph_metric => .graph_metric_segment, }; } @@ -81,6 +84,7 @@ pub fn sidecarKindForArtifactKind(kind: artifact_ref.ArtifactKind) ?source_bindi .sparse_segment => .sparse, .graph_segment => .graph, .algebraic_segment => .algebraic, + .graph_metric_segment => .graph_metric, else => null, }; } @@ -262,6 +266,21 @@ test "sidecar manifest rejects mismatched artifact kinds and duplicate ids" { ); } +test "serverless sidecar manifest permits only consistent graph metric aliases" { + const original = DeclaredArtifact{ + .name = "1:a1:x", + .binding = .{ .sidecar_kind = .graph_metric, .source_kind = .serverless_fragment, .row_ref_kind = .serverless, .snapshot_id = "manifest-1", .schema_fingerprint = "schema-v1", .column_bindings = &.{"edges"}, .index_config_hash = "metric-config" }, + .artifact = .{ .kind = .graph_metric_segment, .name = "1:a1:x", .artifact_id = "metric", .checksum = "checksum", .byte_len = 128, .metadata_version = artifact_ref.graph_metric_segment_wire_version }, + }; + var alias = original; + alias.name = "1:b1:y"; + alias.artifact.name = alias.name; + try (Manifest{ .artifacts = &.{ original, alias } }).validate(); + alias.artifact.graph_metric_source_checksum[0] = 1; + try std.testing.expectError(error.DuplicateSidecarArtifactDeclaration, (Manifest{ .artifacts = &.{ original, alias } }).validate()); + try std.testing.expectError(error.DuplicateSidecarArtifactDeclaration, (Manifest{ .artifacts = &.{ original, original } }).validate()); +} + test "sidecar manifest validates declared artifacts against RowSource batches" { const row_refs = [_]rowsource.RowRef{ .{ .external = .{ diff --git a/zig/pkg/antfly/src/serverless/segment/source_binding.zig b/zig/pkg/antfly/src/serverless/segment/source_binding.zig index 26dbcc380c..4123e3c98a 100644 --- a/zig/pkg/antfly/src/serverless/segment/source_binding.zig +++ b/zig/pkg/antfly/src/serverless/segment/source_binding.zig @@ -24,6 +24,7 @@ pub const SidecarKind = enum(u8) { sparse = 3, graph = 4, algebraic = 5, + graph_metric = 6, }; pub const RowRefKind = enum(u8) { diff --git a/zig/pkg/antfly/src/serverless_main.zig b/zig/pkg/antfly/src/serverless_main.zig index fa368fe710..367220daa8 100644 --- a/zig/pkg/antfly/src/serverless_main.zig +++ b/zig/pkg/antfly/src/serverless_main.zig @@ -20,6 +20,7 @@ const serverless_default_max_request_bytes: usize = antfly.public_api.http_serve const serverless_default_max_connection_threads: u32 = 64; const serverless_default_query_cache_max_bytes: u64 = 4 * 1024 * 1024 * 1024; const serverless_default_query_cache_payload_max_bytes: u64 = 64 * 1024 * 1024; +const serverless_default_graph_metric_max_parallelism: u8 = @intCast(serverless.build.default_graph_metric_compute_parallelism); const CliConfig = struct { config_path: ?[]const u8 = null, @@ -32,6 +33,7 @@ const CliConfig = struct { query_cache_dir: ?[]const u8 = null, query_cache_max_bytes: ?u64 = null, query_cache_payload_max_bytes: ?u64 = null, + manifest_write_version: ?u16 = null, embedding_indexes_json: ?[]const u8 = null, sparse_embedding_index_name: ?[]const u8 = null, chunk_embedding_index_name: ?[]const u8 = null, @@ -41,6 +43,7 @@ const CliConfig = struct { health_port: ?u16 = null, max_request_bytes: ?usize = null, max_connection_threads: ?u32 = null, + graph_metric_max_parallelism: ?u8 = null, role: ?[]const u8 = null, tick_ms: ?u64 = null, publish_enabled: ?bool = null, @@ -133,6 +136,12 @@ pub fn runFromIterator( .query_cache_dir = cli.query_cache_dir orelse init.environ_map.get("ANTFLY_SERVERLESS_QUERY_CACHE_DIR"), .query_cache_max_bytes = cli.query_cache_max_bytes orelse try parseEnvIntOrDefault(init.environ_map, u64, "ANTFLY_SERVERLESS_QUERY_CACHE_MAX_BYTES", serverless_default_query_cache_max_bytes), .query_cache_payload_max_bytes = cli.query_cache_payload_max_bytes orelse try parseEnvIntOrDefault(init.environ_map, u64, "ANTFLY_SERVERLESS_QUERY_CACHE_PAYLOAD_MAX_BYTES", serverless_default_query_cache_payload_max_bytes), + .manifest_write_version = cli.manifest_write_version orelse try parseEnvIntOrDefault( + init.environ_map, + u16, + "ANTFLY_SERVERLESS_MANIFEST_WRITE_VERSION", + serverless.manifest.codec.wire_version, + ), .embedding_indexes_json = cli.embedding_indexes_json orelse init.environ_map.get("ANTFLY_SERVERLESS_EMBEDDING_INDEXES_JSON"), .sparse_embedding_index_name = cli.sparse_embedding_index_name orelse init.environ_map.get("ANTFLY_SERVERLESS_SPARSE_EMBEDDING_INDEX_NAME") orelse "serverless_sparse", .chunk_embedding_index_name = cli.chunk_embedding_index_name orelse init.environ_map.get("ANTFLY_SERVERLESS_CHUNK_EMBEDDING_INDEX_NAME") orelse "serverless_chunk", @@ -148,6 +157,12 @@ pub fn runFromIterator( .query_max_concurrent_requests = if (loaded_config) |*cfg| cfg.admission.query.max_concurrent_requests else antfly.common.config.default_query_max_concurrent_requests, .graph_execution_limits = if (loaded_config) |*cfg| cfg.graph_execution else .{}, .write_max_concurrent_requests = if (loaded_config) |*cfg| cfg.admission.write.max_concurrent_requests else antfly.common.config.default_write_max_concurrent_requests, + .graph_metric_max_parallelism = cli.graph_metric_max_parallelism orelse try parseEnvIntOrDefault( + init.environ_map, + u8, + "ANTFLY_SERVERLESS_GRAPH_METRIC_MAX_PARALLELISM", + serverless_default_graph_metric_max_parallelism, + ), }; const listener_enabled = forced_listener orelse listenerEnabledForRole(bootstrap.role); const listener = if (listener_enabled) try serverless_serverConfigFromEnv(init.environ_map, cli) else null; @@ -307,6 +322,10 @@ fn parseCli(args: *std.process.Args.Iterator) !CliConfig { cfg.query_cache_payload_max_bytes = try std.fmt.parseInt(u64, args.next() orelse return error.InvalidArguments, 10); continue; } + if (std.mem.eql(u8, arg, "--manifest-write-version")) { + cfg.manifest_write_version = try std.fmt.parseInt(u16, args.next() orelse return error.InvalidArguments, 10); + continue; + } if (std.mem.eql(u8, arg, "--embedding-indexes-json")) { cfg.embedding_indexes_json = args.next() orelse return error.InvalidArguments; continue; @@ -343,6 +362,10 @@ fn parseCli(args: *std.process.Args.Iterator) !CliConfig { cfg.max_connection_threads = try std.fmt.parseInt(u32, args.next() orelse return error.InvalidArguments, 10); continue; } + if (std.mem.eql(u8, arg, "--graph-metric-max-parallelism")) { + cfg.graph_metric_max_parallelism = try std.fmt.parseInt(u8, args.next() orelse return error.InvalidArguments, 10); + continue; + } if (std.mem.eql(u8, arg, "--role")) { cfg.role = args.next() orelse return error.InvalidArguments; continue; @@ -652,12 +675,14 @@ fn printUsage(argv0: []const u8) void { \\ --catalog-uri \\ --query-cache-dir \\ --query-cache-max-bytes + \\ --manifest-write-version <18> \\ --query-cache-payload-max-bytes \\ --host \\ --port \\ --health-port \\ --max-request-bytes \\ --max-connection-threads + \\ --graph-metric-max-parallelism <1..16> \\ --role \\ --tick-ms \\ --publish-enabled @@ -682,11 +707,13 @@ fn printUsage(argv0: []const u8) void { \\ ANTFLY_SERVERLESS_QUERY_CACHE_DIR \\ ANTFLY_SERVERLESS_QUERY_CACHE_MAX_BYTES default: 4294967296 \\ ANTFLY_SERVERLESS_QUERY_CACHE_PAYLOAD_MAX_BYTES default: 67108864 + \\ ANTFLY_SERVERLESS_MANIFEST_WRITE_VERSION default: 18 (only the current version is supported) \\ ANTFLY_SERVERLESS_BIND_HOST default: 127.0.0.1 \\ ANTFLY_SERVERLESS_BIND_PORT default: 8080 \\ ANTFLY_SERVERLESS_HEALTH_PORT default: unset (disables dedicated health server) \\ ANTFLY_SERVERLESS_MAX_REQUEST_BYTES default: 33554432 \\ ANTFLY_SERVERLESS_MAX_CONNECTION_THREADS default: 64 (0 unbounded) + \\ ANTFLY_SERVERLESS_GRAPH_METRIC_MAX_PARALLELISM default: 4 \\ ANTFLY_SERVERLESS_ROLE default: combined \\ ANTFLY_SERVERLESS_TICK_INTERVAL_MS default: 25 \\ ANTFLY_SERVERLESS_PUBLISH_ENABLED default: true @@ -715,6 +742,7 @@ fn startupErrorHint(err: anyerror) ?[]const u8 { error.InvalidQueryCacheBudget => "invalid query cache budget; ANTFLY_SERVERLESS_QUERY_CACHE_MAX_BYTES must be greater than zero when the cache is enabled", error.InvalidQueryCachePayloadBudget => "invalid query cache payload budget; ANTFLY_SERVERLESS_QUERY_CACHE_PAYLOAD_MAX_BYTES must be greater than zero when the cache is enabled", error.QueryCachePayloadExceedsBudget => "invalid query cache budgets; the per-payload limit cannot exceed the total cache limit", + error.InvalidGraphMetricBuildOptions => "invalid graph metric build options; ANTFLY_SERVERLESS_GRAPH_METRIC_MAX_PARALLELISM must be between 1 and 16", error.InvalidRuntimeRole => "invalid runtime role; expected combined, api, query, or maintenance", error.InvalidEnvironmentValue => "invalid serverless environment configuration; see the preceding variable-specific error", error.MissingEndpoint => "missing S3-compatible endpoint; configure the storage connection endpoint or AWS_ENDPOINT_URL", diff --git a/zig/pkg/antfly/src/storage/backend_erased.zig b/zig/pkg/antfly/src/storage/backend_erased.zig index be8b744724..f7a35a1d09 100644 --- a/zig/pkg/antfly/src/storage/backend_erased.zig +++ b/zig/pkg/antfly/src/storage/backend_erased.zig @@ -365,6 +365,7 @@ pub const WriteTxn = struct { ptr: *anyopaque, vtable: *const VTable, boundary_dispatch: BoundaryAbi.Dispatch = BoundaryAbi.local_dispatch, + write_gate: ?*std.atomic.Mutex = null, pub const VTable = struct { abort: *const fn (Allocator, *anyopaque) void, @@ -378,12 +379,15 @@ pub const WriteTxn = struct { const BoundaryAbi = runtime_callback_abi.Boundary(VTable); pub fn abort(self: *WriteTxn) void { + const gate = self.write_gate; self.vtable.abort(self.allocator, self.ptr); + if (gate) |mutex| mutex.unlock(); self.* = undefined; } pub fn commit(self: *WriteTxn) !void { try BoundaryAbi.call("commit", self.boundary_dispatch, self.vtable.commit, .{ self.allocator, self.ptr }); + if (self.write_gate) |mutex| mutex.unlock(); self.* = undefined; } @@ -483,6 +487,7 @@ pub const Batch = struct { allocator: Allocator, ptr: *anyopaque, vtable: *const VTable, + write_gate: ?*std.atomic.Mutex = null, pub const VTable = struct { abort: *const fn (Allocator, *anyopaque) void, @@ -494,15 +499,19 @@ pub const Batch = struct { delete: *const fn (*anyopaque, []const u8) anyerror!void, open_cursor: ?*const fn (Allocator, *anyopaque) anyerror!Cursor = null, set_replay_opaque: ?*const fn (*anyopaque, u64, []const u8) anyerror!void = null, + contains_many_sorted: ?*const fn (*anyopaque, []const []const u8, []bool) anyerror!void = null, }; pub fn abort(self: *Batch) void { + const gate = self.write_gate; self.vtable.abort(self.allocator, self.ptr); + if (gate) |mutex| mutex.unlock(); self.* = undefined; } pub fn commit(self: *Batch) !void { try self.vtable.commit(self.allocator, self.ptr); + if (self.write_gate) |mutex| mutex.unlock(); self.* = undefined; } @@ -510,6 +519,19 @@ pub const Batch = struct { return try self.vtable.get(self.ptr, key); } + /// Return presence only. Native backends avoid retained value payloads and + /// share sorted-run/block probes; portable fallbacks preserve get semantics. + pub fn containsManySorted(self: *Batch, keys: []const []const u8, present: []bool) !void { + if (keys.len != present.len) return error.InvalidBatch; + for (keys, 0..) |key, i| if (i != 0 and std.mem.order(u8, keys[i - 1], key) == .gt) return error.InvalidBatch; + @memset(present, false); + if (self.vtable.contains_many_sorted) |contains| return contains(self.ptr, keys, present); + for (keys, present) |key, *exists| exists.* = if (self.get(key)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + } + pub fn getManySorted(self: *Batch, keys: []const []const u8, values: []?[]const u8) !void { if (keys.len != values.len) return error.InvalidBatch; @memset(values, null); @@ -608,6 +630,10 @@ pub const Store = struct { ptr: *anyopaque, vtable: *const VTable, boundary_dispatch: BoundaryAbi.Dispatch = BoundaryAbi.local_dispatch, + /// Opt-in transaction-wide serialization for read/modify/write users. + /// The backend owns this gate and must outlive all stores/transactions. + /// Reads and computation outside a write transaction remain concurrent. + write_gate: ?*std.atomic.Mutex = null, pub const ReplayCallback = *const fn (*anyopaque, u64, []const u8) anyerror!void; @@ -698,16 +724,28 @@ pub const Store = struct { } pub fn beginWrite(self: *Store) !WriteTxn { - return try BoundaryAbi.call("begin_write", self.boundary_dispatch, self.vtable.begin_write, .{ self.allocator, self.ptr }); + if (self.write_gate) |mutex| platform.sync.lockYielding(mutex); + errdefer if (self.write_gate) |mutex| mutex.unlock(); + var txn = try BoundaryAbi.call("begin_write", self.boundary_dispatch, self.vtable.begin_write, .{ self.allocator, self.ptr }); + txn.write_gate = self.write_gate; + return txn; } pub fn beginBatch(self: *Store) !Batch { - return try BoundaryAbi.call("begin_batch", self.boundary_dispatch, self.vtable.begin_batch, .{ self.allocator, self.ptr }); + if (self.write_gate) |mutex| platform.sync.lockYielding(mutex); + errdefer if (self.write_gate) |mutex| mutex.unlock(); + var batch = try BoundaryAbi.call("begin_batch", self.boundary_dispatch, self.vtable.begin_batch, .{ self.allocator, self.ptr }); + batch.write_gate = self.write_gate; + return batch; } pub fn beginBatchWithOptions(self: *Store, options: backend_types.BatchOptions) !Batch { if (self.vtable.begin_batch_with_options) |begin_batch_with_options| { - return try BoundaryAbi.call("begin_batch_with_options", self.boundary_dispatch, begin_batch_with_options, .{ self.allocator, self.ptr, options }); + if (self.write_gate) |mutex| platform.sync.lockYielding(mutex); + errdefer if (self.write_gate) |mutex| mutex.unlock(); + var batch = try BoundaryAbi.call("begin_batch_with_options", self.boundary_dispatch, begin_batch_with_options, .{ self.allocator, self.ptr, options }); + batch.write_gate = self.write_gate; + return batch; } return try self.beginBatch(); } @@ -1369,6 +1407,10 @@ pub fn batchFrom(allocator: Allocator, handle: anytype) !Batch { return try unbox(ptr).handle.get(key); } + fn containsManySorted(ptr: *anyopaque, keys: []const []const u8, present: []bool) anyerror!void { + return unbox(ptr).handle.containsManySorted(keys, present); + } + fn getManySorted(ptr: *anyopaque, keys: []const []const u8, values: []?[]const u8) anyerror!void { if (@hasDecl(Handle, "getManySorted")) { return try unbox(ptr).handle.getManySorted(keys, values); @@ -1423,6 +1465,7 @@ pub fn batchFrom(allocator: Allocator, handle: anytype) !Batch { .delete = vt.delete, .open_cursor = if (@hasDecl(Handle, "openCursor")) vt.openCursor else null, .set_replay_opaque = if (@hasDecl(Handle, "setReplayOpaque")) vt.setReplayOpaque else null, + .contains_many_sorted = if (@hasDecl(Handle, "containsManySorted")) vt.containsManySorted else null, }, }; } @@ -1945,6 +1988,7 @@ test "runtime store erases concrete single-namespace store handles" { }; const MockStore = struct { + fail_open: *bool, pub fn capabilities(_: *@This()) backend_types.Capabilities { return .{ .cursors = true }; } @@ -1953,18 +1997,23 @@ test "runtime store erases concrete single-namespace store handles" { return .{}; } - pub fn beginWrite(_: *@This()) !MockWrite { + pub fn beginWrite(self: *@This()) !MockWrite { + if (self.fail_open.*) return error.OpenFailed; return .{}; } - pub fn beginBatch(_: *@This()) !MockBatch { + pub fn beginBatch(self: *@This()) !MockBatch { + if (self.fail_open.*) return error.OpenFailed; return .{}; } }; - const mock = MockStore{}; + var fail_open = false; + const mock = MockStore{ .fail_open = &fail_open }; var store = try storeFrom(std.testing.allocator, mock); defer store.deinit(); + var gate: std.atomic.Mutex = .unlocked; + store.write_gate = &gate; try std.testing.expect(store.capabilities().cursors); var read = try store.beginRead(); @@ -1985,14 +2034,35 @@ test "runtime store erases concrete single-namespace store handles" { try std.testing.expectEqualStrings("a", (try current_scan_cur.first()).?.key); var write = try store.beginWrite(); + try std.testing.expect(!gate.tryLock()); try write.put("k", "w"); try std.testing.expectEqualStrings("w", try write.get("k")); try write.commit(); + try std.testing.expect(gate.tryLock()); + gate.unlock(); var batch = try store.beginBatch(); + try std.testing.expect(!gate.tryLock()); try batch.put("k", "b"); try std.testing.expectEqualStrings("b", try batch.get("k")); try batch.commit(); + try std.testing.expect(gate.tryLock()); + gate.unlock(); + batch = try store.beginBatchWithOptions(.{}); + try std.testing.expect(!gate.tryLock()); + batch.abort(); + try std.testing.expect(gate.tryLock()); + gate.unlock(); + fail_open = true; + try std.testing.expectError(error.OpenFailed, store.beginWrite()); + try std.testing.expect(gate.tryLock()); + gate.unlock(); + try std.testing.expectError(error.OpenFailed, store.beginBatch()); + try std.testing.expect(gate.tryLock()); + gate.unlock(); + try std.testing.expectError(error.OpenFailed, store.beginBatchWithOptions(.{})); + try std.testing.expect(gate.tryLock()); + gate.unlock(); } test "failed commit keeps erased write handle abortable" { @@ -2045,10 +2115,26 @@ test "failed commit keeps erased write handle abortable" { var shared = Shared{}; var txn = try writeTxnFrom(std.testing.allocator, MockWrite{ .shared = &shared }); + var gate: std.atomic.Mutex = .unlocked; + try std.testing.expect(gate.tryLock()); + txn.write_gate = &gate; try std.testing.expectError(error.CommitFailed, txn.commit()); + try std.testing.expect(!gate.tryLock()); txn.abort(); + try std.testing.expect(gate.tryLock()); + gate.unlock(); try std.testing.expectEqual(@as(usize, 1), shared.commits); try std.testing.expect(shared.aborted); + shared = .{}; + var batch = try batchFrom(std.testing.allocator, MockWrite{ .shared = &shared }); + try std.testing.expect(gate.tryLock()); + batch.write_gate = &gate; + try std.testing.expectError(error.CommitFailed, batch.commit()); + try std.testing.expect(!gate.tryLock()); + batch.abort(); + try std.testing.expect(gate.tryLock()); + gate.unlock(); + try std.testing.expect(shared.aborted); } test "runtime namespace store maps logical namespaces into concrete partitions" { diff --git a/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig b/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig index 208eb858bf..8c227a11c7 100644 --- a/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig +++ b/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig @@ -16,6 +16,7 @@ const std = @import("std"); const builtin = @import("builtin"); const storage_build_options = @import("build_options"); const platform = @import("antfly_platform"); +const platform_clock = platform.clock; const Allocator = std.mem.Allocator; const fs_paths = @import("../../../common/fs_paths.zig"); const CancellationToken = @import("../../../common/cancellation.zig").CancellationToken; @@ -1421,6 +1422,17 @@ pub const IndexManager = struct { retired_lsm_owner_labels_collapsed: [2]u64 = .{ 0, 0 }, sparse_indexes: std.ArrayListUnmanaged(SparseIndex), graph_indexes: std.ArrayListUnmanaged(GraphIndex), + /// Lock-free cursors give bounded scheduler sweeps stable round-robin + /// fairness while the catalog shared lock keeps the indexed slices stable. + graph_metric_coordinator_cursor: std.atomic.Value(usize) = .init(0), + graph_metric_worker_cursor: std.atomic.Value(usize) = .init(0), + /// Worker snapshots pin the inline graph-index array after releasing the + /// catalog lock. Structural graph catalog mutations wait for these bounded + /// page quanta, while unrelated catalog readers and writers no longer wait + /// for an entire maintenance drain. + graph_metric_schedule_pins: std.atomic.Value(usize) = .init(0), + graph_metric_schedule_pin_mutex: std.Io.Mutex = .init, + graph_metric_schedule_pins_drained: std.Io.Condition = .init, algebraic_indexes: std.ArrayListUnmanaged(AlgebraicIndex), enrichments: std.ArrayListUnmanaged(enrichment_catalog.EnrichmentConfig), resolvers: std.ArrayListUnmanaged(resolver_catalog.ResolverConfig) = .empty, @@ -2730,6 +2742,7 @@ pub const IndexManager = struct { apply_mutex: *std.atomic.Mutex, config: types.IndexConfig, edge_type_configs: []graph_mod.EdgeTypeConfig, + metric_configs: []graph_mod.GraphMetricConfig, artifact_sources: []GraphArtifactSource = &.{}, max_edges_per_document: u32 = 0, rebuild_root_path: []u8, @@ -5217,12 +5230,33 @@ pub const IndexManager = struct { if (cfg.field_name) |field_name| self.alloc.free(field_name); } self.alloc.free(entry.edge_type_configs); + graph_mod.freeGraphMetricConfigs(self.alloc, entry.metric_configs); for (entry.artifact_sources) |*source| source.deinit(self.alloc); if (entry.artifact_sources.len > 0) self.alloc.free(entry.artifact_sources); self.alloc.free(entry.rebuild_root_path); entry.config.deinit(self.alloc); } + fn waitForGraphMetricSchedulePins(self: *IndexManager) void { + if (self.io) |io| { + self.graph_metric_schedule_pin_mutex.lockUncancelable(io); + defer self.graph_metric_schedule_pin_mutex.unlock(io); + while (self.graph_metric_schedule_pins.load(.acquire) != 0) { + self.graph_metric_schedule_pins_drained.waitUncancelable(io, &self.graph_metric_schedule_pin_mutex); + } + return; + } + if (comptime builtin.os.tag == .freestanding or builtin.single_threaded) { + if (self.graph_metric_schedule_pins.load(.acquire) != 0) + @panic("cannot drain graph metric schedule pins without an I/O coordinator"); + return; + } + // Manually configured managers may not own an executor. Their worker + // sweeps are normally synchronous; retain the established teardown + // fallback for a structural mutation racing such a sweep. + while (self.graph_metric_schedule_pins.load(.acquire) != 0) std.Thread.yield() catch {}; + } + pub fn deinit(self: *IndexManager) void { self.deinitWithBackendDisposition(false); } @@ -5245,6 +5279,7 @@ pub const IndexManager = struct { } fn deinitWithBackendDisposition(self: *IndexManager, abandon_after_crash: bool) void { + self.waitForGraphMetricSchedulePins(); self.clearVectorBlockGeneration(); self.releaseFullTextPendingBytes(); self.text_merge_scheduler.deinit(self.alloc); @@ -7967,6 +8002,977 @@ pub const IndexManager = struct { return steps; } + pub fn runGraphMetricMaintenance(self: *IndexManager) !usize { + var total_steps: usize = 0; + for (self.graph_indexes.items) |*entry| { + for (entry.metric_configs) |cfg| { + if (cfg.refresh != .background) continue; + var status = try entry.index.graphMetricStatus(cfg.name); + defer status.deinit(entry.index.alloc); + if (status.maintenance_paused) continue; + switch (status.state) { + .not_ready, .stale, .failed => { + var published = entry.index.runGraphMetric(cfg.name) catch |err| switch (err) { + error.GraphMetricDisabled => continue, + else => return err, + }; + published.deinit(entry.index.alloc); + total_steps += 1; + }, + .disabled, .fresh, .building => {}, + } + } + } + return total_steps; + } + + pub fn refreshGraphMetric(self: *IndexManager, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.enableGraphMetric(metric_name); + return try entry.index.runGraphMetric(metric_name); + } + + pub fn rebuildGraphMetric(self: *IndexManager, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.enableGraphMetric(metric_name); + return try entry.index.runGraphMetric(metric_name); + } + + pub fn deleteGraphMetricMaterialization(self: *IndexManager, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.deleteGraphMetricMaterialization(metric_name); + return try entry.index.graphMetricStatus(metric_name); + } + + pub fn pauseGraphMetricMaintenance(self: *IndexManager, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.pauseGraphMetricMaintenance(metric_name); + } + + pub fn resumeGraphMetricMaintenance(self: *IndexManager, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.resumeGraphMetricMaintenance(metric_name); + } + + pub fn ensureGraphMetricPlannedBuild( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + target_generation: u64, + ) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.ensureGraphMetricPlannedBuild(metric_name, target_generation); + } + + pub fn runGraphMetricPlannedWorkerPageStep( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + worker_id: []const u8, + ) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, worker_id); + } + + pub fn runGraphMetricPlannedWorkerPageStepAt( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + worker_id: []const u8, + now_ms: u64, + ) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.runGraphMetricPlannedWorkerPageStepForMetricAt(metric_name, worker_id, now_ms); + } + + pub fn runGraphMetricPlannedCoordinatorStep( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + ) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.runGraphMetricPlannedCoordinatorStepForMetric(metric_name); + } + + pub fn runGraphMetricPlannedCoordinatorStepAt( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + now_ms: u64, + ) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.runGraphMetricPlannedCoordinatorStepForMetricAt(metric_name, now_ms); + } + + pub fn failGraphMetricPlannedBuild( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + err: anyerror, + ) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.failGraphMetricPlannedBuild(metric_name, err); + } + + pub fn runGraphMetricPlannedDrain( + self: *IndexManager, + index_name: []const u8, + metric_name: []const u8, + target_generation: u64, + options: graph_mod.GraphIndex.GraphMetricPlannedDrainOptions, + ) !graph_mod.GraphIndex.GraphMetricStatus { + const entry = self.graphIndex(index_name) orelse return error.IndexNotFound; + return try entry.index.runGraphMetricPlannedDrain(metric_name, target_generation, options); + } + + pub const GraphMetricPlannedSchedulerSweepOptions = struct { + max_metrics: usize = 64, + start_background_builds: bool = true, + now_ms: ?u64 = null, + auto_idle_options: ?GraphMetricPlannedAutoIdleOptions = null, + }; + + pub const GraphMetricPlannedWorkerSweepOptions = struct { + worker_id: []const u8, + max_pages: usize = 64, + now_ms: ?u64 = null, + }; + + const GraphMetricScheduleEntry = struct { + entry: *GraphIndex, + config: *const graph_mod.GraphMetricConfig, + }; + + const GraphMetricWorkerSnapshotEntry = struct { + entry: *GraphIndex, + metric_name: []const u8, + lifecycle_canonical: bool, + }; + + const GraphMetricWorkerSnapshot = struct { + manager: *IndexManager, + entries: []GraphMetricWorkerSnapshotEntry, + names: []u8, + + fn deinit(self: *@This()) void { + self.manager.alloc.free(self.names); + self.manager.alloc.free(self.entries); + if (self.manager.io) |io| { + self.manager.graph_metric_schedule_pin_mutex.lockUncancelable(io); + const prior = self.manager.graph_metric_schedule_pins.fetchSub(1, .release); + std.debug.assert(prior != 0); + if (prior == 1) self.manager.graph_metric_schedule_pins_drained.broadcast(io); + self.manager.graph_metric_schedule_pin_mutex.unlock(io); + } else { + const prior = self.manager.graph_metric_schedule_pins.fetchSub(1, .release); + std.debug.assert(prior != 0); + } + self.* = undefined; + } + }; + + fn graphMetricWorkerSnapshotAlloc(self: *IndexManager) !GraphMetricWorkerSnapshot { + self.catalog_mutex.lockShared(); + defer self.catalog_mutex.unlockShared(); + const metric_count = self.graphMetricScheduleEntryCount(); + const entry_count = metric_count + self.graph_indexes.items.len; + const entries = try self.alloc.alloc(GraphMetricWorkerSnapshotEntry, entry_count); + errdefer self.alloc.free(entries); + var names_len: usize = 0; + for (self.graph_indexes.items) |entry| { + for (entry.metric_configs) |config| names_len = std.math.add(usize, names_len, config.name.len) catch return error.OutOfMemory; + } + const names = try self.alloc.alloc(u8, names_len); + errdefer self.alloc.free(names); + const start = if (entry_count == 0) 0 else self.graph_metric_worker_cursor.fetchAdd(1, .monotonic) % entry_count; + if (metric_count != 0) { + var schedule = GraphMetricScheduleIterator.init(self, 0); + var names_offset: usize = 0; + for (0..metric_count) |i| { + const snapshot_entry = &entries[(i + start) % entry_count]; + const scheduled = schedule.next(); + const metric_name = names[names_offset..][0..scheduled.config.name.len]; + @memcpy(metric_name, scheduled.config.name); + names_offset += metric_name.len; + snapshot_entry.* = .{ + .entry = scheduled.entry, + .metric_name = metric_name, + .lifecycle_canonical = graphMetricLifecycleCanonical(scheduled.entry.metric_configs, scheduled.config.*), + }; + } + std.debug.assert(names_offset == names.len); + } + // Index-scoped topology reclamation must survive removal of the last + // metric. Rotate these entries with numerical work to avoid starvation. + for (self.graph_indexes.items, 0..) |*entry, i| { + entries[(metric_count + i + start) % entry_count] = .{ + .entry = entry, + .metric_name = "", + .lifecycle_canonical = false, + }; + } + _ = self.graph_metric_schedule_pins.fetchAdd(1, .acq_rel); + return .{ .manager = self, .entries = entries, .names = names }; + } + + const GraphMetricScheduleIterator = struct { + manager: *IndexManager, + graph_index: usize, + metric_index: usize, + + fn init(manager: *IndexManager, start: usize) GraphMetricScheduleIterator { + var remaining = start; + for (manager.graph_indexes.items, 0..) |entry, graph_index| { + if (remaining < entry.metric_configs.len) return .{ + .manager = manager, + .graph_index = graph_index, + .metric_index = remaining, + }; + remaining -= entry.metric_configs.len; + } + unreachable; + } + + fn next(self: *GraphMetricScheduleIterator) GraphMetricScheduleEntry { + const entry = &self.manager.graph_indexes.items[self.graph_index]; + const result: GraphMetricScheduleEntry = .{ + .entry = entry, + .config = &entry.metric_configs[self.metric_index], + }; + self.metric_index += 1; + if (self.metric_index == entry.metric_configs.len) { + self.metric_index = 0; + while (true) { + self.graph_index = (self.graph_index + 1) % self.manager.graph_indexes.items.len; + if (self.manager.graph_indexes.items[self.graph_index].metric_configs.len != 0) break; + } + } + return result; + } + }; + + fn graphMetricScheduleEntryCount(self: *IndexManager) usize { + var total: usize = 0; + for (self.graph_indexes.items) |entry| total +|= entry.metric_configs.len; + return total; + } + + pub const GraphMetricPlannedSchedulerSweepResult = struct { + metrics_scanned: usize = 0, + active_builds: usize = 0, + builds_started: usize = 0, + planning_steps: usize = 0, + worker_steps: usize = 0, + coordinator_steps: usize = 0, + publication_checkpoints: usize = 0, + retired_input_records: usize = 0, + pages_claimed: usize = 0, + pages_completed: usize = 0, + phases_advanced: usize = 0, + published: usize = 0, + failed_builds: usize = 0, + rounds_executed: usize = 0, + budget_exhausted: bool = false, + + pub fn progressed(self: @This()) bool { + return self.planning_steps != 0 or self.builds_started != 0 or + self.worker_steps != 0 or + self.coordinator_steps != 0 or + self.retired_input_records != 0 or + self.pages_claimed != 0 or + self.pages_completed != 0 or + self.phases_advanced != 0 or + self.published != 0 or + self.failed_builds != 0; + } + + pub fn durableProgressed(self: @This()) bool { + return self.publication_checkpoints != 0 or self.planning_steps != 0 or self.builds_started != 0 or + self.retired_input_records != 0 or + self.pages_claimed != 0 or + self.pages_completed != 0 or + self.phases_advanced != 0 or + self.published != 0 or + self.failed_builds != 0; + } + + pub fn add(self: *@This(), other: @This()) void { + self.planning_steps += other.planning_steps; + self.metrics_scanned += other.metrics_scanned; + self.active_builds += other.active_builds; + self.builds_started += other.builds_started; + self.worker_steps += other.worker_steps; + self.coordinator_steps += other.coordinator_steps; + self.retired_input_records += other.retired_input_records; + self.publication_checkpoints += other.publication_checkpoints; + self.pages_claimed += other.pages_claimed; + self.pages_completed += other.pages_completed; + self.phases_advanced += other.phases_advanced; + self.published += other.published; + self.failed_builds += other.failed_builds; + self.rounds_executed += other.rounds_executed; + self.budget_exhausted = self.budget_exhausted or other.budget_exhausted; + } + }; + + pub const GraphMetricPlannedMaintenanceOptions = struct { + worker_id: []const u8 = "graph-metric-idle-worker", + worker_ids: []const []const u8 = &.{}, + max_rounds: usize = 1024, + max_metrics_per_round: usize = 64, + max_pages_per_round: usize = 64, + now_ms: ?u64 = null, + }; + + pub const GraphMetricPlannedWorkStats = struct { + metrics_scanned: usize = 0, + queued_builds: usize = 0, + active_builds: usize = 0, + active_pages: usize = 0, + failed_pages: usize = 0, + paused_metrics: usize = 0, + truncated_pages: bool = false, + + pub fn hasWork(self: @This()) bool { + return self.queued_builds != 0 or + self.active_builds != 0 or + self.active_pages != 0 or + self.failed_pages != 0; + } + }; + + pub const GraphMetricPlannedAutoIdleOptions = struct { + max_pagerank_iterations: u32 = std.math.maxInt(u32), + max_eigenvector_iterations: u32 = std.math.maxInt(u32), + max_hits_iterations: u32 = std.math.maxInt(u32), + max_active_builds: usize = 4, + max_active_builds_per_index: usize = 2, + }; + + pub const GraphMetricPlannedAutoIdleDecision = struct { + active_builds: usize = 0, + eligible_queued: usize = 0, + deferred_queued: usize = 0, + ineligible_queued: usize = 0, + + pub fn shouldRunPlanned(self: @This()) bool { + return self.active_builds != 0 or self.eligible_queued != 0; + } + }; + + pub const GraphMetricDegreeCanaryOptions = struct { + max_control_records: usize = 64, + }; + + pub const GraphMetricDegreeCanaryDecision = struct { + active_degree_builds: usize = 0, + eligible_queued_degree: usize = 0, + blocked_active_non_degree: usize = 0, + blocked_queued_non_degree: usize = 0, + control_records: usize = 0, + queued_degree_control_records: usize = 0, + failed_pages: usize = 0, + truncated_pages: bool = false, + max_control_records: usize = 64, + + pub fn shouldRunPlanned(self: @This()) bool { + if (self.blocked_active_non_degree != 0) return false; + if (self.blocked_queued_non_degree != 0) return false; + if (self.failed_pages != 0) return false; + if (self.truncated_pages) return false; + if (self.control_records > self.max_control_records) return false; + if (self.control_records +| self.queued_degree_control_records > self.max_control_records) return false; + if (self.active_degree_builds == 1 and self.eligible_queued_degree == 0) return true; + if (self.active_degree_builds == 0 and self.eligible_queued_degree == 1) return true; + return false; + } + }; + + fn graphMetricLifecycleCanonical(configs: []const graph_mod.GraphMetricConfig, cfg: graph_mod.GraphMetricConfig) bool { + if (cfg.kind != .hits_hub) return true; + for (configs) |candidate| { + if (candidate.kind == .hits_authority and graph_mod.graphMetricHitsPairCompatible(cfg, candidate)) return false; + } + return true; + } + + pub fn graphMetricPlannedWorkStats(self: *IndexManager) !GraphMetricPlannedWorkStats { + var stats = GraphMetricPlannedWorkStats{}; + for (self.graph_indexes.items) |*entry| { + for (entry.metric_configs) |cfg| { + stats.metrics_scanned += 1; + var status = try entry.index.graphMetricStatus(cfg.name); + defer status.deinit(entry.index.alloc); + if (status.maintenance_paused) { + stats.paused_metrics += 1; + continue; + } + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + if (status.build_pages_truncated) stats.truncated_pages = true; + for (status.build_pages) |page| { + switch (page.state) { + .leased => stats.active_pages += 1, + .failed => stats.failed_pages += 1, + .pending, .complete => {}, + } + } + if (status.state == .building or status.phase == .cleanup_old_generations) { + stats.active_builds += 1; + continue; + } + const requested = try entry.index.graphMetricBuildRequested(cfg.name); + if (cfg.refresh != .background and !requested) continue; + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + switch (status.state) { + .not_ready, .stale => stats.queued_builds += 1, + .disabled, .building => {}, + .fresh, .failed => if (requested) { + stats.queued_builds += 1; + }, + } + } + } + return stats; + } + + pub fn shouldRunGraphMetricPlannedAutoIdle( + self: *IndexManager, + options: GraphMetricPlannedAutoIdleOptions, + ) !bool { + const decision = try self.graphMetricPlannedAutoIdleDecision(options); + return decision.shouldRunPlanned(); + } + + pub fn shouldRunGraphMetricDegreeCanary( + self: *IndexManager, + options: GraphMetricDegreeCanaryOptions, + ) !bool { + const decision = try self.graphMetricDegreeCanaryDecision(options); + return decision.shouldRunPlanned(); + } + + pub fn graphMetricDegreeCanaryDecision( + self: *IndexManager, + options: GraphMetricDegreeCanaryOptions, + ) !GraphMetricDegreeCanaryDecision { + var decision = GraphMetricDegreeCanaryDecision{ + .max_control_records = options.max_control_records, + }; + + for (self.graph_indexes.items) |*entry| { + for (entry.metric_configs) |cfg| { + var status = try entry.index.graphMetricStatus(cfg.name); + defer status.deinit(entry.index.alloc); + if (status.maintenance_paused) continue; + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + + const active = status.state == .building or status.phase == .cleanup_old_generations; + if (active) decision.control_records += 1; + decision.control_records += status.build_pages.len; + if (status.build_pages_truncated) decision.truncated_pages = true; + for (status.build_pages) |page| { + if (page.state == .failed) decision.failed_pages += 1; + } + + const queued = cfg.refresh == .background and + graphMetricLifecycleCanonical(entry.metric_configs, cfg) and + (status.state == .not_ready or status.state == .stale); + if (cfg.kind == .degree) { + if (active) decision.active_degree_builds += 1; + if (queued) { + decision.eligible_queued_degree += 1; + decision.queued_degree_control_records +|= entry.index.graphMetricPlannedBuildControlRecordEstimate(cfg); + } + } else { + if (active) decision.blocked_active_non_degree += 1; + if (queued) decision.blocked_queued_non_degree += 1; + } + } + } + + return decision; + } + + pub fn graphMetricPlannedAutoIdleDecision( + self: *IndexManager, + options: GraphMetricPlannedAutoIdleOptions, + ) !GraphMetricPlannedAutoIdleDecision { + self.catalog_mutex.lockShared(); + defer self.catalog_mutex.unlockShared(); + var decision = GraphMetricPlannedAutoIdleDecision{}; + for (self.graph_indexes.items) |*entry| { + const index_active_builds = try graphMetricIndexActiveBuilds(entry); + var index_scheduled_builds: usize = 0; + decision.active_builds += index_active_builds; + for (entry.metric_configs) |cfg| { + const status = try entry.index.graphMetricSchedulerStatus(cfg.name, null); + if (status.maintenance_paused) continue; + + if (status.state == .building or status.phase == .cleanup_old_generations) { + continue; + } + + const requested = try entry.index.graphMetricBuildRequested(cfg.name); + if (cfg.refresh != .background and !requested) continue; + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + switch (status.state) { + .not_ready, .stale, .fresh, .failed => { + if ((status.state == .fresh or status.state == .failed) and !requested) continue; + if (!graphMetricQueuedPlannedAutoEligible(entry.metric_configs, cfg, options)) { + decision.ineligible_queued += 1; + } else if (graphMetricPlannedAutoCanStart( + options, + decision.active_builds, + decision.eligible_queued, + index_active_builds, + index_scheduled_builds, + )) { + decision.eligible_queued += 1; + index_scheduled_builds += 1; + } else { + decision.deferred_queued += 1; + } + }, + .disabled, .building => {}, + } + } + } + + return decision; + } + + fn graphMetricIndexActiveBuilds(entry: *GraphIndex) !usize { + return graphMetricIndexActiveBuildsAt(entry, null); + } + + fn graphMetricIndexActiveBuildsAt(entry: *GraphIndex, now_ms: ?u64) !usize { + var active_builds: usize = 0; + for (entry.metric_configs) |cfg| { + const status = try entry.index.graphMetricSchedulerStatus(cfg.name, now_ms); + if (status.maintenance_paused) continue; + if (status.state == .building or status.phase == .cleanup_old_generations) { + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + active_builds += 1; + } + } + return active_builds; + } + + fn graphMetricPlannedAutoCanStart( + options: GraphMetricPlannedAutoIdleOptions, + active_builds: usize, + eligible_queued: usize, + index_active_builds: usize, + index_scheduled_builds: usize, + ) bool { + if (options.max_active_builds == 0 or options.max_active_builds_per_index == 0) return false; + if (active_builds + eligible_queued >= options.max_active_builds) return false; + if (index_active_builds + index_scheduled_builds >= options.max_active_builds_per_index) return false; + return true; + } + + fn graphMetricQueuedPlannedAutoEligible( + configs: []const graph_mod.GraphMetricConfig, + cfg: graph_mod.GraphMetricConfig, + options: GraphMetricPlannedAutoIdleOptions, + ) bool { + _ = configs; + return switch (cfg.kind) { + .degree => true, + .pagerank => cfg.max_iterations <= options.max_pagerank_iterations, + .eigenvector => cfg.max_iterations <= options.max_eigenvector_iterations, + // Pairing shares a lifecycle; it is not an execution eligibility + // constraint. Standalone lanes use the same bounded worker pages. + .hits_authority, .hits_hub => options.max_hits_iterations != 0 and + cfg.max_iterations <= options.max_hits_iterations, + }; + } + + fn graphMetricShouldAutoStartQueuedBuild( + configs: []const graph_mod.GraphMetricConfig, + cfg: graph_mod.GraphMetricConfig, + options: GraphMetricPlannedAutoIdleOptions, + active_builds: usize, + scheduled_builds: usize, + index_active_builds: usize, + index_scheduled_builds: usize, + ) bool { + if (!graphMetricLifecycleCanonical(configs, cfg)) return false; + if (!graphMetricQueuedPlannedAutoEligible(configs, cfg, options)) return false; + return graphMetricPlannedAutoCanStart( + options, + active_builds, + scheduled_builds, + index_active_builds, + index_scheduled_builds, + ); + } + + pub fn runGraphMetricPlannedCoordinatorSweep( + self: *IndexManager, + options: GraphMetricPlannedSchedulerSweepOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + self.catalog_mutex.lockShared(); + defer self.catalog_mutex.unlockShared(); + return try self.runGraphMetricPlannedCoordinatorSweepUnlocked(options); + } + + fn runGraphMetricPlannedCoordinatorSweepUnlocked( + self: *IndexManager, + options: GraphMetricPlannedSchedulerSweepOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + var result = GraphMetricPlannedSchedulerSweepResult{}; + if (options.max_metrics == 0) return result; + const active_builds_before_start: usize = if (options.auto_idle_options != null) + try self.graphMetricActiveBuildCount() + else + 0; + var scheduled_builds: usize = 0; + + const entry_count = self.graphMetricScheduleEntryCount(); + if (entry_count == 0) return result; + const start = self.graph_metric_coordinator_cursor.fetchAdd(1, .monotonic) % entry_count; + var schedule = GraphMetricScheduleIterator.init(self, start); + var counted_entry: ?*GraphIndex = null; + var index_active_builds: usize = 0; + var index_scheduled_builds: usize = 0; + for (0..entry_count) |_| { + const scheduled = schedule.next(); + const entry = scheduled.entry; + const cfg = scheduled.config.*; + if (options.auto_idle_options != null and counted_entry != entry) { + counted_entry = entry; + index_active_builds = try graphMetricIndexActiveBuildsAt(entry, options.now_ms); + index_scheduled_builds = 0; + } + if (result.metrics_scanned >= options.max_metrics) { + result.budget_exhausted = true; + return result; + } + result.metrics_scanned += 1; + + const status = try entry.index.graphMetricSchedulerStatus(cfg.name, options.now_ms); + if (status.maintenance_paused) continue; + if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + + var active = status.state == .building; + const requested = try entry.index.graphMetricBuildRequested(cfg.name); + if (!active and (requested or (options.start_background_builds and cfg.refresh == .background))) { + switch (status.state) { + .not_ready, .stale, .failed, .fresh => { + if (status.state == .fresh and !requested) continue; + if (status.state == .failed and !requested) { + // Preserve terminal failures for the same + // generation, but do not let a superseded + // snapshot permanently block newer graph + // traffic. A newer dirty generation is a new + // unit of work, not a blind retry. + const failed_generation = status.failed_target_generation; + if (failed_generation >= status.target_edge_generation) continue; + } + if (options.auto_idle_options) |auto_options| { + if (!graphMetricQueuedPlannedAutoEligible(entry.metric_configs, cfg, auto_options)) continue; + } else if (!graphMetricLifecycleCanonical(entry.metric_configs, cfg)) continue; + if (!try entry.index.prepareGraphMetricPartitionForConfigStep(cfg, 4096)) { + result.planning_steps += 1; + continue; + } + const preparation = entry.index.prepareGraphMetricTopologyDetailed(cfg, status.target_edge_generation) catch |err| switch (err) { + error.GraphMetricBuildSnapshotChanged => continue, + else => return err, + }; + switch (preparation) { + .ready => {}, + .queued => { + result.planning_steps += 1; + continue; + }, + .waiting => continue, + } + if (options.auto_idle_options) |auto_options| { + if (!graphMetricShouldAutoStartQueuedBuild( + entry.metric_configs, + cfg, + auto_options, + active_builds_before_start, + scheduled_builds, + index_active_builds, + index_scheduled_builds, + )) continue; + } + var started = entry.index.ensureGraphMetricPlannedBuildFromCachedPlan(cfg.name, status.target_edge_generation) catch |err| switch (err) { + error.GraphMetricDisabled => continue, + error.GraphMetricBuildSnapshotChanged => continue, + else => return err, + }; + defer started.deinit(entry.index.alloc); + result.builds_started += 1; + scheduled_builds += 1; + index_scheduled_builds += 1; + active = true; + }, + .disabled, .building => {}, + } + } + if (!active) continue; + result.active_builds += 1; + + const step = (if (options.now_ms) |now_ms| + entry.index.runGraphMetricPlannedCoordinatorStepForMetricAt(cfg.name, now_ms) + else + entry.index.runGraphMetricPlannedCoordinatorStepForMetric(cfg.name)) catch |err| switch (err) { + error.GraphMetricBuildJobNotFound, error.GraphMetricBuildNotActive, error.GraphMetricDisabled => continue, + else => return err, + }; + result.coordinator_steps += 1; + if (step.checkpointed_publication) result.publication_checkpoints += 1; + result.retired_input_records += step.retired_input_records; + if (step.advanced_phase) result.phases_advanced += 1; + if (step.published or (step.advanced_phase and step.phase == .publish_generation)) { + result.published += 1; + } + if (step.failed_build) result.failed_builds += 1; + } + return result; + } + + fn graphMetricActiveBuildCount(self: *IndexManager) !usize { + var active_builds: usize = 0; + for (self.graph_indexes.items) |*entry| { + active_builds += try graphMetricIndexActiveBuilds(entry); + } + return active_builds; + } + + pub fn runGraphMetricPlannedWorkerSweep( + self: *IndexManager, + options: GraphMetricPlannedWorkerSweepOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + var snapshot = try self.graphMetricWorkerSnapshotAlloc(); + defer snapshot.deinit(); + return try self.runGraphMetricPlannedWorkerSweepSnapshot(options, snapshot.entries); + } + + fn runGraphMetricPlannedWorkerSweepSnapshot( + _: *IndexManager, + options: GraphMetricPlannedWorkerSweepOptions, + scheduled_entries: []const GraphMetricWorkerSnapshotEntry, + ) !GraphMetricPlannedSchedulerSweepResult { + if (options.worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + var result = GraphMetricPlannedSchedulerSweepResult{}; + if (options.max_pages == 0) return result; + + const entry_count = scheduled_entries.len; + if (entry_count == 0) return result; + var topology_census_steps: usize = 0; + for (scheduled_entries) |scheduled| { + const entry = scheduled.entry; + const metric_name = scheduled.metric_name; + if (result.worker_steps >= options.max_pages) { + result.budget_exhausted = true; + return result; + } + + if (metric_name.len == 0) { + // Idle reads are bounded too, but do not report eligible work + // and keep runUntilIdle spinning through a retained catalog. + if (topology_census_steps >= options.max_pages) continue; + topology_census_steps += 1; + const prepared = entry.index.runGraphMetricTopologyPreparationStep(options.worker_id) catch |err| switch (err) { + error.GraphMetricBuildSuperseded, error.GraphMetricBuildSnapshotChanged => continue, + else => return err, + }; + if (prepared) { + result.worker_steps += 1; + // This may be only a packing checkpoint or retirement + // slice, not a completed numerical page. + result.planning_steps += 1; + continue; + } + const topology = try entry.index.cleanupGraphMetricTopologyPageDetailed(); + if (topology.removed != 0) { + result.worker_steps += 1; + result.pages_completed += 1; + result.retired_input_records += topology.removed; + } + continue; + } + if (try entry.index.cleanupDeletedGraphMetricMaterializationPage(metric_name)) { + result.metrics_scanned += 1; + result.worker_steps += 1; + result.pages_completed += 1; + continue; + } + if (try entry.index.cleanupFailedGraphMetricBuildJobPage(metric_name)) { + result.metrics_scanned += 1; + result.worker_steps += 1; + result.pages_completed += 1; + continue; + } + const status = try entry.index.graphMetricSchedulerStatus(metric_name, options.now_ms); + if (status.maintenance_paused) continue; + if (try entry.index.cleanupRetiredGraphMetricScoreGenerationPage(metric_name)) { + result.metrics_scanned += 1; + result.worker_steps += 1; + result.pages_completed += 1; + continue; + } + if (!scheduled.lifecycle_canonical) continue; + const active = status.state == .building or status.phase == .cleanup_old_generations; + if (!active) continue; + result.metrics_scanned += 1; + result.active_builds += 1; + + const step = (if (options.now_ms) |now_ms| + entry.index.runGraphMetricPlannedWorkerPageStepForMetricAt(metric_name, options.worker_id, now_ms) + else + entry.index.runGraphMetricPlannedWorkerPageStepForMetric(metric_name, options.worker_id)) catch |err| switch (err) { + error.GraphMetricBuildJobNotFound, error.GraphMetricBuildNotActive, error.GraphMetricDisabled => continue, + // Graph mutations may supersede a snapshot after the + // coordinator sweep but before this worker sweep. Treat + // that as expected scheduler churn: the coordinator owns + // durable failure/cancellation and will retire the exact + // stale job on its next tick. A worker must neither turn + // normal write traffic into a runtime error nor race a + // replacement lease by failing a job it no longer owns. + error.GraphMetricBuildSuperseded => continue, + else => return err, + }; + result.worker_steps += 1; + if (step.claimed_page) result.pages_claimed += 1; + if (step.completed_page) result.pages_completed += 1; + if (step.advanced_phase) result.phases_advanced += 1; + if (result.worker_steps >= options.max_pages) { + const after_status = try entry.index.graphMetricSchedulerStatus(metric_name, options.now_ms); + if (after_status.state == .building or after_status.phase == .cleanup_old_generations) { + result.budget_exhausted = true; + } + return result; + } + } + return result; + } + + pub fn runGraphMetricPlannedMaintenance( + self: *IndexManager, + options: GraphMetricPlannedMaintenanceOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + return try self.runGraphMetricPlannedMaintenanceWithAuto(options, null); + } + + pub fn runGraphMetricPlannedAutoMaintenance( + self: *IndexManager, + options: GraphMetricPlannedMaintenanceOptions, + auto_options: GraphMetricPlannedAutoIdleOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + return try self.runGraphMetricPlannedMaintenanceWithAuto(options, auto_options); + } + + fn runGraphMetricPlannedMaintenanceWithAuto( + self: *IndexManager, + options: GraphMetricPlannedMaintenanceOptions, + auto_options: ?GraphMetricPlannedAutoIdleOptions, + ) !GraphMetricPlannedSchedulerSweepResult { + try validateGraphMetricPlannedMaintenanceWorkers(options); + var total = GraphMetricPlannedSchedulerSweepResult{}; + if (options.max_rounds == 0) { + total.budget_exhausted = true; + return total; + } + + var rounds: usize = 0; + while (rounds < options.max_rounds) : (rounds += 1) { + var round = GraphMetricPlannedSchedulerSweepResult{}; + const coordinator_before = try self.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = options.max_metrics_per_round, + .start_background_builds = true, + .now_ms = options.now_ms, + .auto_idle_options = auto_options, + }); + round.add(coordinator_before); + + if (options.worker_ids.len == 0) { + const worker = try self.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = options.worker_id, + .max_pages = options.max_pages_per_round, + .now_ms = options.now_ms, + }); + round.add(worker); + } else { + var pages_remaining = options.max_pages_per_round; + while (pages_remaining > 0) { + var worker_progressed = false; + for (options.worker_ids) |worker_id| { + if (pages_remaining == 0) break; + const worker = try self.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = worker_id, + .max_pages = 1, + .now_ms = options.now_ms, + }); + round.add(worker); + pages_remaining -= @min(pages_remaining, worker.worker_steps); + worker_progressed = worker_progressed or worker.durableProgressed(); + } + if (!worker_progressed) break; + } + } + + const coordinator_after = try self.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = options.max_metrics_per_round, + .start_background_builds = true, + .now_ms = options.now_ms, + .auto_idle_options = auto_options, + }); + round.add(coordinator_after); + + const round_budget_exhausted = round.budget_exhausted; + round.budget_exhausted = false; + round.rounds_executed = 1; + total.add(round); + if (!round.durableProgressed()) { + total.budget_exhausted = round_budget_exhausted; + return total; + } + } + total.budget_exhausted = true; + return total; + } + + fn validateGraphMetricPlannedMaintenanceWorkers(options: GraphMetricPlannedMaintenanceOptions) !void { + if (options.worker_ids.len == 0) { + if (options.worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + return; + } + + for (options.worker_ids, 0..) |worker_id, i| { + if (worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + for (options.worker_ids[0..i]) |prior_worker_id| { + if (std.mem.eql(u8, worker_id, prior_worker_id)) return error.InvalidGraphMetricBuildWorker; + } + } + } + + fn graphMetricStatusHasRunnableWorkerPage( + status: graph_mod.GraphIndex.GraphMetricStatus, + worker_id: []const u8, + now_ms: ?u64, + ) bool { + if (status.state == .building or status.phase == .cleanup_old_generations) return true; + if (status.build_pages_truncated) return true; + const now: u64 = now_ms orelse platform_clock.Clock.real().nowRealtimeMs(); + for (status.build_pages) |page| { + switch (page.state) { + .pending, .failed => return true, + .leased => { + if (std.mem.eql(u8, page.worker_id, worker_id)) return true; + if (page.lease_expires_at_ms <= now) return true; + }, + .complete => {}, + } + } + return false; + } + pub const DensePostingMaintenanceOptions = struct { max_postings_per_index: usize = 64, validate_payloads: bool = false, @@ -9881,6 +10887,7 @@ pub const IndexManager = struct { name, try combineRemovalCatalogMutation(atomic_mutation, &cleanup), ); + self.waitForGraphMetricSchedulePins(); self.freeGraphIndexEntry(entry); _ = self.graph_indexes.orderedRemove(i); defer self.dropIndexLoadStateNoLock(name); @@ -16779,7 +17786,10 @@ pub const IndexManager = struct { }, .dense_vector => |entry| try self.dense_indexes.append(self.alloc, entry), .sparse_vector => |entry| try self.sparse_indexes.append(self.alloc, entry), - .graph => |entry| try self.graph_indexes.append(self.alloc, entry), + .graph => |entry| { + self.waitForGraphMetricSchedulePins(); + try self.graph_indexes.append(self.alloc, entry); + }, .algebraic => |entry| try self.algebraic_indexes.append(self.alloc, entry), } } @@ -17496,6 +18506,7 @@ pub const IndexManager = struct { .reverse_lsm_options = self.graph_reverse_lsm_options, .reverse_lsm_root_generation = self.lsm_root_generation, .edge_type_configs = graph_cfg.edge_type_configs, + .metric_configs = graph_cfg.metric_configs, .rebuild_root_path = path, .rebuild_owner_generation = coverageGenerationForConfig(cfg), .algebraic_semiring_traversal = graph_cfg.algebraic_semiring_traversal, @@ -17510,6 +18521,7 @@ pub const IndexManager = struct { .apply_mutex = apply_mutex, .config = cloned_cfg, .edge_type_configs = graph_cfg.edge_type_configs, + .metric_configs = graph_cfg.metric_configs, .artifact_sources = graph_cfg.artifact_sources, .max_edges_per_document = graph_cfg.max_edges_per_document, .rebuild_root_path = try self.alloc.dupe(u8, path), @@ -18921,6 +19933,7 @@ pub const IndexManager = struct { } for (self.graph_indexes.items, 0..) |*entry, i| { if (std.mem.eql(u8, entry.config.name, name)) { + self.waitForGraphMetricSchedulePins(); self.freeGraphIndexEntry(entry); _ = self.graph_indexes.orderedRemove(i); self.dropIndexLoadStateNoLock(name); @@ -18948,7 +19961,10 @@ pub const IndexManager = struct { .full_text => try self.text_indexes.ensureUnusedCapacity(self.alloc, 1), .dense_vector => try self.dense_indexes.ensureUnusedCapacity(self.alloc, 1), .sparse_vector => try self.sparse_indexes.ensureUnusedCapacity(self.alloc, 1), - .graph => try self.graph_indexes.ensureUnusedCapacity(self.alloc, 1), + .graph => { + self.waitForGraphMetricSchedulePins(); + try self.graph_indexes.ensureUnusedCapacity(self.alloc, 1); + }, .algebraic => try self.algebraic_indexes.ensureUnusedCapacity(self.alloc, 1), } } @@ -18992,6 +20008,7 @@ pub const IndexManager = struct { } for (self.graph_indexes.items, 0..) |entry, i| { if (!std.mem.eql(u8, entry.config.name, name)) continue; + self.waitForGraphMetricSchedulePins(); const detached = self.graph_indexes.orderedRemove(i); self.dropIndexLoadStateNoLock(name); return .{ .graph = detached }; @@ -26088,6 +27105,7 @@ pub const GraphArtifactSource = struct { const GraphConfig = struct { edge_type_configs: []graph_mod.EdgeTypeConfig, + metric_configs: []graph_mod.GraphMetricConfig, artifact_sources: []GraphArtifactSource = &.{}, shorthand_asset: ?enrichment_catalog.EnrichmentConfig = null, max_edges_per_document: u32 = 0, @@ -26099,6 +27117,7 @@ const GraphConfig = struct { if (cfg.field_name) |field_name| alloc.free(field_name); } alloc.free(self.edge_type_configs); + graph_mod.freeGraphMetricConfigs(alloc, self.metric_configs); for (self.artifact_sources) |*source| source.deinit(alloc); if (self.artifact_sources.len > 0) alloc.free(self.artifact_sources); if (self.shorthand_asset) |*asset| asset.deinit(alloc); @@ -27289,6 +28308,8 @@ fn parseGraphConfig(alloc: Allocator, raw: []const u8) !GraphConfig { errdefer if (shorthand_asset) |*asset| { asset.deinit(alloc); }; + const metric_configs = try parseGraphMetricConfigs(alloc, root); + errdefer graph_mod.freeGraphMetricConfigs(alloc, metric_configs); if (shorthand_asset) |asset| { if (artifact_sources.len != 1 or !std.mem.eql(u8, asset.name, artifact_sources[0].artifact_name)) @@ -27300,6 +28321,7 @@ fn parseGraphConfig(alloc: Allocator, raw: []const u8) !GraphConfig { const edge_types = root.object.get("edge_types") orelse { return .{ .edge_type_configs = try alloc.alloc(graph_mod.EdgeTypeConfig, 0), + .metric_configs = metric_configs, .artifact_sources = artifact_sources, .shorthand_asset = shorthand_asset, .max_edges_per_document = max_edges_per_document, @@ -27353,6 +28375,7 @@ fn parseGraphConfig(alloc: Allocator, raw: []const u8) !GraphConfig { return .{ .edge_type_configs = configs, + .metric_configs = metric_configs, .artifact_sources = artifact_sources, .shorthand_asset = shorthand_asset, .max_edges_per_document = max_edges_per_document, @@ -27360,6 +28383,137 @@ fn parseGraphConfig(alloc: Allocator, raw: []const u8) !GraphConfig { }; } +fn parseGraphMetricConfigs(alloc: Allocator, root: std.json.Value) ![]graph_mod.GraphMetricConfig { + const metrics_value = root.object.get("metrics") orelse return try alloc.alloc(graph_mod.GraphMetricConfig, 0); + if (metrics_value != .object) return error.InvalidIndexConfig; + + var configs = std.ArrayListUnmanaged(graph_mod.GraphMetricConfig).empty; + errdefer { + for (configs.items) |*cfg| { + alloc.free(cfg.name); + cfg.edge_filter.deinit(alloc); + } + configs.deinit(alloc); + } + + var it = metrics_value.object.iterator(); + while (it.next()) |entry| { + const name = entry.key_ptr.*; + if (entry.value_ptr.* != .object) return error.InvalidIndexConfig; + const metric_obj = entry.value_ptr.*.object; + const enabled = if (metric_obj.get("enabled")) |value| blk: { + if (value != .bool) return error.InvalidIndexConfig; + break :blk value.bool; + } else true; + if (!enabled) continue; + + const kind = if (metric_obj.get("kind")) |value| blk: { + if (value != .string) return error.InvalidIndexConfig; + if (std.mem.eql(u8, value.string, "pagerank")) break :blk graph_mod.GraphMetricKind.pagerank; + if (std.mem.eql(u8, value.string, "degree")) break :blk graph_mod.GraphMetricKind.degree; + if (std.mem.eql(u8, value.string, "eigenvector")) break :blk graph_mod.GraphMetricKind.eigenvector; + if (std.mem.eql(u8, value.string, "hits_authority")) break :blk graph_mod.GraphMetricKind.hits_authority; + if (std.mem.eql(u8, value.string, "hits_hub")) break :blk graph_mod.GraphMetricKind.hits_hub; + return error.InvalidIndexConfig; + } else if (std.mem.eql(u8, name, "pagerank")) + graph_mod.GraphMetricKind.pagerank + else if (std.mem.eql(u8, name, "degree")) + graph_mod.GraphMetricKind.degree + else if (std.mem.eql(u8, name, "eigenvector")) + graph_mod.GraphMetricKind.eigenvector + else if (std.mem.eql(u8, name, "hits_authority")) + graph_mod.GraphMetricKind.hits_authority + else if (std.mem.eql(u8, name, "hits_hub")) + graph_mod.GraphMetricKind.hits_hub + else + return error.InvalidIndexConfig; + + const refresh = if (metric_obj.get("refresh")) |value| blk: { + if (value != .string) return error.InvalidIndexConfig; + if (std.mem.eql(u8, value.string, "background")) break :blk graph_mod.GraphMetricRefreshMode.background; + if (std.mem.eql(u8, value.string, "manual")) break :blk graph_mod.GraphMetricRefreshMode.manual; + return error.InvalidIndexConfig; + } else graph_mod.GraphMetricRefreshMode.background; + + const damping = if (metric_obj.get("damping")) |value| try jsonNumberAsF64(value) else 0.85; + if (damping <= 0.0 or damping >= 1.0) return error.InvalidIndexConfig; + const tolerance = if (metric_obj.get("tolerance")) |value| try jsonNumberAsF64(value) else 0.000001; + if (tolerance <= 0.0) return error.InvalidIndexConfig; + const max_iterations = if (metric_obj.get("max_iterations")) |value| blk: { + const raw = try jsonNumberAsU32(value); + if (raw == 0 or raw > graph_mod.graph_metric_max_iterations) return error.InvalidIndexConfig; + break :blk raw; + } else 50; + + const owned_name = try alloc.dupe(u8, name); + var owned_name_moved = false; + errdefer if (!owned_name_moved) alloc.free(owned_name); + + var cfg = graph_mod.GraphMetricConfig{ + .name = owned_name, + .kind = kind, + .damping = damping, + .tolerance = tolerance, + .max_iterations = max_iterations, + .refresh = refresh, + .edge_filter = try parseGraphMetricEdgeFilter(alloc, metric_obj.get("edge_filter")), + }; + owned_name_moved = true; + var cfg_moved = false; + errdefer if (!cfg_moved) { + alloc.free(cfg.name); + cfg.edge_filter.deinit(alloc); + }; + try configs.append(alloc, cfg); + cfg_moved = true; + } + + const owned = try configs.toOwnedSlice(alloc); + return owned; +} + +fn parseGraphMetricEdgeFilter(alloc: Allocator, maybe_value: ?std.json.Value) !graph_mod.GraphMetricEdgeFilter { + const value = maybe_value orelse return .{}; + if (value != .object) return error.InvalidIndexConfig; + if (value.object.get("types")) |types_value| { + if (types_value != .array or types_value.array.items.len == 0) return error.InvalidIndexConfig; + const edge_types = try alloc.alloc([]const u8, types_value.array.items.len); + var initialized: usize = 0; + errdefer { + for (edge_types[0..initialized]) |edge_type| alloc.free(edge_type); + alloc.free(edge_types); + } + for (types_value.array.items, 0..) |item, i| { + if (item != .string or item.string.len == 0) return error.InvalidIndexConfig; + edge_types[i] = try alloc.dupe(u8, item.string); + initialized += 1; + } + return .{ .mode = .types, .types = edge_types }; + } + if (value.object.get("mode")) |mode_value| { + if (mode_value != .string) return error.InvalidIndexConfig; + if (std.mem.eql(u8, mode_value.string, "all")) return .{}; + return error.InvalidIndexConfig; + } + return .{}; +} + +fn jsonNumberAsF64(value: std.json.Value) !f64 { + return switch (value) { + .integer => |v| @floatFromInt(v), + .float => |v| v, + else => error.InvalidIndexConfig, + }; +} + +fn jsonNumberAsU32(value: std.json.Value) !u32 { + return switch (value) { + .integer => |v| if (v > 0 and v <= std.math.maxInt(u32)) @intCast(v) else error.InvalidIndexConfig, + .float => |v| if (v > 0 and v <= std.math.maxInt(u32) and @floor(v) == v) @intFromFloat(v) else error.InvalidIndexConfig, + else => error.InvalidIndexConfig, + }; +} + fn singleGraphArtifactSourceSliceAlloc(alloc: Allocator, source_value: GraphArtifactSource) ![]GraphArtifactSource { var source = source_value; errdefer source.deinit(alloc); @@ -27656,6 +28810,35 @@ test "graph config declares algebraic provenance semiring traversal law" { )); } +test "graph config bounds iterative metric work" { + const alloc = std.testing.allocator; + var bounded = try parseGraphConfig(alloc, + \\{"metrics":{"pagerank":{"max_iterations":1000}}} + ); + defer bounded.deinit(alloc); + try std.testing.expectEqual(graph_mod.graph_metric_max_iterations, bounded.metric_configs[0].max_iterations); + + try std.testing.expectError(error.InvalidIndexConfig, parseGraphConfig(alloc, + \\{"metrics":{"pagerank":{"max_iterations":1001}}} + )); +} + +test "HITS lifecycle scheduling suppresses only the compatible hub" { + const cites = [_][]const u8{"cites"}; + const mentions = [_][]const u8{"mentions"}; + const compatible = [_]graph_mod.GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority, .edge_filter = .{ .mode = .types, .types = &cites } }, + .{ .name = "hub", .kind = .hits_hub, .edge_filter = .{ .mode = .types, .types = &cites } }, + }; + try std.testing.expect(!IndexManager.graphMetricLifecycleCanonical(&compatible, compatible[1])); + + const unrelated = [_]graph_mod.GraphMetricConfig{ + .{ .name = "authority", .kind = .hits_authority, .edge_filter = .{ .mode = .types, .types = &mentions } }, + .{ .name = "hub", .kind = .hits_hub, .edge_filter = .{ .mode = .types, .types = &cites } }, + }; + try std.testing.expect(IndexManager.graphMetricLifecycleCanonical(&unrelated, unrelated[1])); +} + test "graph config validates and retains edge materialization limits" { const alloc = std.testing.allocator; var cfg = try parseGraphConfig(alloc, diff --git a/zig/pkg/antfly/src/storage/db/core.zig b/zig/pkg/antfly/src/storage/db/core.zig index 9f32d8dc0f..6c77cc045f 100644 --- a/zig/pkg/antfly/src/storage/db/core.zig +++ b/zig/pkg/antfly/src/storage/db/core.zig @@ -74,6 +74,7 @@ pub const PendingWorkStats = struct { promotion: types.ReplayStageStats = .{}, text_merge: types.TextMergeStats = .{}, repair_metadata_rebuild_pending: bool = false, + graph_metric: index_manager_mod.IndexManager.GraphMetricPlannedWorkStats = .{}, }; pub const MaintenanceDriver = struct { diff --git a/zig/pkg/antfly/src/storage/db/db.zig b/zig/pkg/antfly/src/storage/db/db.zig index a6089b2772..3d8ea1383d 100644 --- a/zig/pkg/antfly/src/storage/db/db.zig +++ b/zig/pkg/antfly/src/storage/db/db.zig @@ -237,6 +237,7 @@ const scraping = if (builtin.os.tag == .freestanding or build_options.bench_mini else @import("antfly_scraping"); const graph_mod = @import("../../graph/graph.zig"); +const graph_metric_rerank = @import("../../graph/metric_rerank.zig"); const NodeAdmission = @import("../../graph/node_admission.zig").NodeAdmission; const GraphNodeRef = @import("../../graph/node_admission.zig").NodeRef; const traversal_mod = @import("../../graph/traversal.zig"); @@ -277,6 +278,7 @@ const ttl_runtime_mod = @import("maintenance/ttl_runtime.zig"); const transaction_runtime_mod = @import("maintenance/transaction_runtime.zig"); const text_merge_runtime_mod = @import("maintenance/text_merge_runtime.zig"); const sparse_compaction_runtime_mod = @import("maintenance/sparse_compaction_runtime.zig"); +const graph_metric_runtime_mod = @import("maintenance/graph_metric_runtime.zig"); const transform_mod = @import("transform.zig"); const sim_fixture = @import("../sim_fixture.zig"); const storage_sim = @import("../sim_runtime.zig"); @@ -428,6 +430,13 @@ pub const OpenOptions = struct { } }; + pub const GraphMetricIdleMaintenanceMode = enum { + legacy, + planned, + auto, + degree_canary, + }; + table_storage: ?table_storage_mod.Settings = null, open_mode: OpenOptions.OpenMode = .writer, map_size: usize = 256 * 1024 * 1024, @@ -490,6 +499,11 @@ pub const OpenOptions = struct { transaction_recovery: transaction_runtime_mod.Config = .{}, text_merge: text_merge_runtime_mod.Config = .{}, sparse_compaction: sparse_compaction_runtime_mod.Config = .{}, + graph_metric_maintenance: graph_metric_runtime_mod.Config = .{}, + graph_metric_idle_maintenance: GraphMetricIdleMaintenanceMode = .auto, + graph_metric_idle_planned_options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions = .{}, + graph_metric_idle_auto_options: index_manager_mod.IndexManager.GraphMetricPlannedAutoIdleOptions = .{}, + graph_metric_idle_degree_canary_options: index_manager_mod.IndexManager.GraphMetricDegreeCanaryOptions = .{}, /// Optional cross-shard candidate source for entity resolution blocking, /// injected by the serving layer (see `api/distributed_candidate_source.zig`). /// Null means local-only blocking against the worker's own store. Must @@ -4155,6 +4169,10 @@ pub const DB = struct { executor: *derived_executor_mod.Executor, start_index_workers: bool, optional_runtime_workers_enabled: bool, + graph_metric_idle_maintenance: OpenOptions.GraphMetricIdleMaintenanceMode, + graph_metric_idle_planned_options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, + graph_metric_idle_auto_options: index_manager_mod.IndexManager.GraphMetricPlannedAutoIdleOptions, + graph_metric_idle_degree_canary_options: index_manager_mod.IndexManager.GraphMetricDegreeCanaryOptions, resolver_workers_enabled: bool, secret_store: ?*common_secrets.FileStore, remote_content: ?*const scraping.RemoteContentConfig, @@ -4179,6 +4197,7 @@ pub const DB = struct { transaction_runtime: ?*transaction_runtime_mod.Runtime, text_merge_runtime: ?*text_merge_runtime_mod.TextMergeRuntime, sparse_compaction_runtime: ?*sparse_compaction_runtime_mod.SparseCompactionRuntime, + graph_metric_runtime: ?*graph_metric_runtime_mod.GraphMetricRuntime, // Background retry of quarantined index loads (see retryQuarantinedIndexLoads). // Started after the DB reaches its final address; exits once all // quarantined indexes recover or the DB closes. @@ -4639,6 +4658,10 @@ pub const DB = struct { .executor = executor, .start_index_workers = start_index_workers, .optional_runtime_workers_enabled = false, + .graph_metric_idle_maintenance = opts.graph_metric_idle_maintenance, + .graph_metric_idle_planned_options = opts.graph_metric_idle_planned_options, + .graph_metric_idle_auto_options = opts.graph_metric_idle_auto_options, + .graph_metric_idle_degree_canary_options = opts.graph_metric_idle_degree_canary_options, .resolver_workers_enabled = opts.start_resolver_workers, .secret_store = opts.secret_store, .remote_content = opts.remote_content, @@ -4660,6 +4683,7 @@ pub const DB = struct { .transaction_runtime = null, .text_merge_runtime = null, .sparse_compaction_runtime = null, + .graph_metric_runtime = null, .shadow = null, }; core_owner_transferred = true; @@ -5824,6 +5848,23 @@ pub const DB = struct { self.async_context.sparse_compaction_runtime = runtime; } + fn initOptionalGraphMetricRuntime(self: *DB, cfg: graph_metric_runtime_mod.Config) !void { + if (!self.start_index_workers or !cfg.enabled) return; + const resources = self.core.asyncResources(); + const runtime = try self.runtime_alloc.create(graph_metric_runtime_mod.GraphMetricRuntime); + errdefer self.runtime_alloc.destroy(runtime); + runtime.* = try graph_metric_runtime_mod.GraphMetricRuntime.init( + self.runtime_alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + self.backend_runtime, + cfg, + ); + errdefer runtime.deinit(); + self.graph_metric_runtime = runtime; + } + fn initOptionalRuntimes(self: *DB, opts: *OpenOptions) !void { // Created before enrichment so the enrichment append context can notify // it when extraction artifacts land. @@ -5850,6 +5891,7 @@ pub const DB = struct { } try self.initOptionalTextMergeRuntime(opts.text_merge); try self.initOptionalSparseCompactionRuntime(opts.sparse_compaction); + try self.initOptionalGraphMetricRuntime(opts.graph_metric_maintenance); } fn startOptionalRuntimes(self: *DB) !void { @@ -5865,6 +5907,7 @@ pub const DB = struct { } if (self.text_merge_runtime) |runtime| try runtime.start(); if (self.sparse_compaction_runtime) |runtime| try runtime.start(); + if (self.graph_metric_runtime) |runtime| try runtime.start(); } /// Publish non-joining shutdown to optional workers before a borrowed @@ -6044,6 +6087,10 @@ pub const DB = struct { runtime.deinit(); self.runtime_alloc.destroy(runtime); } + if (self.graph_metric_runtime) |runtime| { + runtime.deinit(); + self.runtime_alloc.destroy(runtime); + } if (executor_ready and self.open_mode.allowsIndexWorkers()) { // The executor has published its final HBC coverage. Mirror the // human-facing lifecycle/status view once at graceful shutdown; @@ -24327,6 +24374,7 @@ pub const DB = struct { .promotion = self.promotionStageStats(), .text_merge = if (self.text_merge_runtime) |runtime| runtime.statsAssumeApplyLockHeld() else self.core.index_manager.textMergeStats(), .repair_metadata_rebuild_pending = self.artifactRepairMetadataRebuildPending(), + .graph_metric = self.core.index_manager.graphMetricPlannedWorkStats() catch .{}, }; } @@ -24633,12 +24681,20 @@ pub const DB = struct { const next_target = self.core.nextDerivedSequence(); if (next_target <= stable_target) { try waitForManagedIndexesApplied(self, sequence, index_names); + if (self.syncTargetsIncludeGraph(index_names)) _ = try self.runGraphMetricMaintenanceForIdle(); return; } stable_target = next_target; } } + fn syncTargetsIncludeGraph(self: *DB, index_names: []const []const u8) bool { + for (index_names) |index_name| { + if (self.core.graphIndex(index_name) != null) return true; + } + return false; + } + pub fn waitForCurrentSyncLevel(self: *DB, sync_level: types.SyncLevel) !void { try self.waitForCurrentSyncLevelWithCancellation(sync_level, .none); } @@ -24734,6 +24790,13 @@ pub const DB = struct { try replayPendingDerivedBatches(self, progress_ctx, progress_hook, .{}); } + /// Appends internal derived work without exposing the DB's batch execution + /// context. Runtime partitions use this boundary while retaining the normal + /// write gate, locking, backlog accounting, and HA mirroring semantics. + pub fn derivedAsyncAppendDerivedBatchRecord(self: *DB, derived_batch: derived_types.DerivedBatch) !u64 { + return try appendDerivedBatchRecord(self, derived_batch); + } + const run_until_idle_max_replay_rounds: usize = 16; fn currentMaintenanceTargetSequence(self: *DB) u64 { @@ -25059,6 +25122,7 @@ pub const DB = struct { // boundary: besides posting repair it advances tree-link repair, // posting checkpoints, and quiescent vector-block publication. _ = try self.runDensePostingMaintenanceForIdle(); + _ = try self.runGraphMetricMaintenanceForIdle(); _ = try self.drainDensePostingMaintenanceForIdle(); // This is a caller-proven stable writer boundary. Publish the native // exact-vector generation here rather than depending on a later live @@ -25077,6 +25141,327 @@ pub const DB = struct { _ = try self.runLsmMaintenanceUntilIdle(); } + pub fn runGraphMetricMaintenanceForIdle(self: *DB) !usize { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + // Planned maintenance uses the same catalog pins and transaction + // fences as background workers. Never hold the ingest lock while + // draining graph computation; graph writes may supersede a build. + switch (self.graph_metric_idle_maintenance) { + .auto => return self.runGraphMetricPlannedAutoMaintenanceForIdle(), + .planned => return self.drainGraphMetricPlannedIdle(), + else => {}, + } + lockApply(self); + defer self.core.unlockApply(); + return switch (self.graph_metric_idle_maintenance) { + .legacy => try self.core.index_manager.runGraphMetricMaintenance(), + .planned, .auto => unreachable, + .degree_canary => try self.runGraphMetricDegreeCanaryMaintenanceForIdleLocked(), + }; + } + + fn graphMetricPlannedProgress(result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult) usize { + return result.planning_steps + result.builds_started + result.pages_completed + result.phases_advanced + result.published; + } + + fn drainGraphMetricPlannedIdle(self: *DB) !usize { + const result = try self.core.index_manager.runGraphMetricPlannedMaintenance(self.graph_metric_idle_planned_options); + if (result.budget_exhausted) return error.RunUntilIdleDidNotConverge; + return graphMetricPlannedProgress(result); + } + + fn runGraphMetricPlannedAutoMaintenanceForIdle(self: *DB) !usize { + const result = try self.core.index_manager.runGraphMetricPlannedAutoMaintenance( + self.graph_metric_idle_planned_options, + self.graph_metric_idle_auto_options, + ); + if (result.budget_exhausted) return error.RunUntilIdleDidNotConverge; + const progressed = graphMetricPlannedProgress(result); + const after = try self.core.index_manager.graphMetricPlannedAutoIdleDecision(self.graph_metric_idle_auto_options); + if (!after.shouldRunPlanned() and after.ineligible_queued != 0) { + // Admission caps must not silently select unlimited local compute. + return error.RunUntilIdleDidNotConverge; + } + return progressed; + } + + fn runGraphMetricDegreeCanaryMaintenanceForIdleLocked(self: *DB) !usize { + const decision = try self.core.index_manager.graphMetricDegreeCanaryDecision(self.graph_metric_idle_degree_canary_options); + if (decision.shouldRunPlanned()) return try self.drainGraphMetricPlannedIdle(); + if (decision.active_degree_builds != 0 or decision.blocked_active_non_degree != 0) { + return error.RunUntilIdleDidNotConverge; + } + return try self.core.index_manager.runGraphMetricMaintenance(); + } + + pub fn runGraphMetricPlannedMaintenanceForIdle( + self: *DB, + options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + return try self.core.index_manager.runGraphMetricPlannedMaintenance(options); + } + + const GraphMetricServiceMaintenanceAction = enum { tick, status, release }; + + const GraphMetricServiceMaintenanceRequest = struct { + action: GraphMetricServiceMaintenanceAction = .tick, + role: graph_metric_runtime_mod.Role, + runtime_id: []const u8, + owner_id: []const u8, + lease_owned: bool = false, + lease_ttl_ms: u64 = 30_000, + worker_id: ?[]const u8 = null, + worker_ids: ?[]const []const u8 = null, + start_background_builds: bool = true, + max_rounds: usize = 1, + max_metrics_per_round: usize = 8, + max_pages_per_round: usize = 1, + preserve_lease_after_tick: bool = false, + now_ms: ?u64 = null, + }; + + pub fn runGraphMetricServiceMaintenanceJsonAlloc(self: *DB, alloc: Allocator, body: []const u8) ![]u8 { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + var parsed = std.json.parseFromSlice(GraphMetricServiceMaintenanceRequest, alloc, if (body.len == 0) "{}" else body, .{ + .allocate = .alloc_always, + .ignore_unknown_fields = true, + }) catch return error.InvalidGraphMetricRuntimeConfig; + defer parsed.deinit(); + + var manual_clock = platform_clock.ManualClock{}; + if (parsed.value.now_ms) |now_ms| manual_clock.setRealtimeNs(now_ms *| std.time.ns_per_ms); + const resources = self.core.asyncResources(); + var runtime = try graph_metric_runtime_mod.GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + self.backend_runtime, + .{ + .enabled = true, + .start_background_loop = false, + .role = parsed.value.role, + .runtime_id = parsed.value.runtime_id, + .lease_owned = parsed.value.lease_owned, + .owner_id = parsed.value.owner_id, + .lease_ttl_ms = parsed.value.lease_ttl_ms, + .coordinator_start_background_builds = parsed.value.start_background_builds, + .planned_options = .{ + .worker_id = parsed.value.worker_id orelse "", + .worker_ids = parsed.value.worker_ids orelse &.{}, + .max_rounds = parsed.value.max_rounds, + .max_metrics_per_round = parsed.value.max_metrics_per_round, + .max_pages_per_round = parsed.value.max_pages_per_round, + }, + .clock = if (parsed.value.now_ms != null) manual_clock.clock() else platform_clock.Clock.real(), + }, + ); + var preserve_lease = false; + defer if (preserve_lease) runtime.deinitPreserveLease() else runtime.deinit(); + + if (parsed.value.action == .release) { + const released = try runtime.ownership.releaseHeldLease(); + var current_lease = try runtime.ownership.loadLease(alloc); + defer if (current_lease) |*lease| lease_mod.deinitRecord(alloc, lease); + var runtime_stats = runtime.stats(); + runtime_stats.shutdown = true; + return try std.json.Stringify.valueAlloc(alloc, .{ + .released = released, + .lease_owner_id_hash = if (current_lease) |lease| graph_metric_runtime_mod.identityHash(lease.owner_id) else 0, + .lease_expires_at_ms = if (current_lease) |lease| lease.expires_at_ms else 0, + .stats = runtime_stats, + }, .{ .emit_null_optional_fields = false }); + } + + const result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult = if (parsed.value.action == .tick) try runtime.runOnceDetailed() else .{}; + const runtime_stats = runtime.stats(); + preserve_lease = parsed.value.preserve_lease_after_tick and parsed.value.lease_owned and parsed.value.action == .tick and runtime_stats.has_lease; + return try std.json.Stringify.valueAlloc(alloc, .{ + .result = result, + .stats = runtime_stats, + }, .{ .emit_null_optional_fields = false }); + } + + pub fn refreshGraphMetric(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.enableGraphMetric(metric_name); + var status = try entry.index.runGraphMetric(metric_name); + defer status.deinit(entry.index.alloc); + const cloned = try cloneGraphMetricStatusFromGraph(alloc, status); + if (self.graph_metric_runtime) |runtime| runtime.notify(); + return cloned; + } + + pub fn rebuildGraphMetric(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.enableGraphMetric(metric_name); + var status = try entry.index.runGraphMetric(metric_name); + defer status.deinit(entry.index.alloc); + const cloned = try cloneGraphMetricStatusFromGraph(alloc, status); + if (self.graph_metric_runtime) |runtime| runtime.notify(); + return cloned; + } + + /// Durably enqueue metric work and return immediately. Public control-plane + /// actions use this path so request latency is independent of graph size; + /// the bounded maintenance runtime performs and checkpoints the build. + pub fn scheduleGraphMetricBuild( + self: *DB, + alloc: Allocator, + index_name: []const u8, + metric_name: []const u8, + force: bool, + ) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + const owned_status = blk: { + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + var current = try entry.index.graphMetricStatus(metric_name); + defer current.deinit(entry.index.alloc); + if (current.state == .disabled) { + try entry.index.enableGraphMetric(metric_name); + current.deinit(entry.index.alloc); + current = try entry.index.graphMetricStatus(metric_name); + } + if (!force and current.state == .fresh) { + break :blk try cloneGraphMetricStatusFromGraph(alloc, current); + } + const target_generation = @max(current.edge_generation, current.target_edge_generation); + // Force means "build another immutable score epoch", not + // "unpublish first". Repeated requests remain idempotent while a + // build is active, and readers keep the verified prior epoch until + // the new pointer is atomically published. + var scheduled = entry.index.queueGraphMetricBuild(metric_name, target_generation) catch |err| switch (err) { + // A newer edge snapshot may be queued while the prior bounded + // build is still active. Treat repeated control-plane actions + // as accepted and expose the active/queued generations in the + // returned status instead of turning a safe retry into a 500. + error.GraphMetricBuildAlreadyRunning => break :blk try cloneGraphMetricStatusFromGraph(alloc, current), + else => return err, + }; + defer scheduled.deinit(self.core.index_manager.alloc); + break :blk try cloneGraphMetricStatusFromGraph(alloc, scheduled); + }; + if (self.graph_metric_runtime) |runtime| runtime.notify(); + return owned_status; + } + + pub fn deleteGraphMetricMaterialization(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + try entry.index.deleteGraphMetricMaterialization(metric_name); + var status = try entry.index.graphMetricStatus(metric_name); + defer status.deinit(entry.index.alloc); + const cloned = try cloneGraphMetricStatusFromGraph(alloc, status); + if (self.graph_metric_runtime) |runtime| runtime.notify(); + return cloned; + } + + pub fn pauseGraphMetricMaintenance(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + var status = try entry.index.pauseGraphMetricMaintenance(metric_name); + defer status.deinit(entry.index.alloc); + return try cloneGraphMetricStatusFromGraph(alloc, status); + } + + pub fn resumeGraphMetricMaintenance(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + const entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; + var status = try entry.index.resumeGraphMetricMaintenance(metric_name); + defer status.deinit(entry.index.alloc); + return try cloneGraphMetricStatusFromGraph(alloc, status); + } + + pub fn ensureGraphMetricPlannedBuild( + self: *DB, + alloc: Allocator, + index_name: []const u8, + metric_name: []const u8, + target_generation: u64, + ) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + var status = try self.core.index_manager.ensureGraphMetricPlannedBuild(index_name, metric_name, target_generation); + defer status.deinit(self.core.index_manager.alloc); + return try cloneGraphMetricStatusFromGraph(alloc, status); + } + + pub fn runGraphMetricPlannedWorkerPageStep(self: *DB, index_name: []const u8, metric_name: []const u8, worker_id: []const u8) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + return try self.core.index_manager.runGraphMetricPlannedWorkerPageStep(index_name, metric_name, worker_id); + } + + pub fn runGraphMetricPlannedWorkerPageStepAt(self: *DB, index_name: []const u8, metric_name: []const u8, worker_id: []const u8, now_ms: u64) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + return try self.core.index_manager.runGraphMetricPlannedWorkerPageStepAt(index_name, metric_name, worker_id, now_ms); + } + + pub fn runGraphMetricPlannedCoordinatorStep(self: *DB, index_name: []const u8, metric_name: []const u8) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + return try self.core.index_manager.runGraphMetricPlannedCoordinatorStep(index_name, metric_name); + } + + pub fn runGraphMetricPlannedCoordinatorStepAt(self: *DB, index_name: []const u8, metric_name: []const u8, now_ms: u64) !graph_mod.GraphIndex.GraphMetricBuildWorkerStepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + return try self.core.index_manager.runGraphMetricPlannedCoordinatorStepAt(index_name, metric_name, now_ms); + } + + pub fn failGraphMetricPlannedBuild(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8, err: anyerror) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + var status = try self.core.index_manager.failGraphMetricPlannedBuild(index_name, metric_name, err); + defer status.deinit(self.core.index_manager.alloc); + return try cloneGraphMetricStatusFromGraph(alloc, status); + } + + pub fn runGraphMetricPlannedDrain(self: *DB, alloc: Allocator, index_name: []const u8, metric_name: []const u8, target_generation: u64, options: graph_mod.GraphIndex.GraphMetricPlannedDrainOptions) !types.GraphMetricStatus { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + lockApply(self); + defer self.core.unlockApply(); + var status = try self.core.index_manager.runGraphMetricPlannedDrain(index_name, metric_name, target_generation, options); + defer status.deinit(self.core.index_manager.alloc); + return try cloneGraphMetricStatusFromGraph(alloc, status); + } + + pub fn runGraphMetricPlannedCoordinatorSweep(self: *DB, options: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + // Planned graph work is generation-fenced and storage-transactional. + // IndexManager pins catalog lifetime without blocking foreground apply. + return try self.core.index_manager.runGraphMetricPlannedCoordinatorSweep(options); + } + + pub fn runGraphMetricPlannedWorkerSweep(self: *DB, options: index_manager_mod.IndexManager.GraphMetricPlannedWorkerSweepOptions) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + return try self.core.index_manager.runGraphMetricPlannedWorkerSweep(options); + } + pub fn runUntilIdle(self: *DB) !void { try self.runUntilIdleWithReplayDrainOptions(.{ .wait_for_enrichment_retries = true }); } @@ -28000,6 +28385,7 @@ pub const DB = struct { if (item.algebraic_planner_lifecycle_blocking_reason) |value| alloc.free(value); if (item.algebraic_last_observed_query_shape) |value| alloc.free(value); if (item.algebraic_last_recommended_materialization) |value| alloc.free(value); + types.freeGraphMetricStatuses(alloc, @constCast(item.graph_metric_status)); if (item.algebraic_top_candidate) |candidate| { alloc.free(candidate.recommendation); alloc.free(candidate.materialization_id); @@ -28422,6 +28808,105 @@ pub const DB = struct { item.algebraic_graph_traversal_result_node_count = algebraic_graph.result_node_count; } + fn cloneGraphMetricBuildPageStatusesFromGraph( + alloc: Allocator, + source: []const graph_mod.GraphIndex.GraphMetricBuildPageStatus, + ) ![]types.GraphMetricBuildPageStatus { + if (source.len == 0) return &.{}; + const out = try alloc.alloc(types.GraphMetricBuildPageStatus, source.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*page| page.deinit(alloc); + alloc.free(out); + } + for (source, 0..) |page, i| { + const worker_id = if (page.worker_id.len > 0) try alloc.dupe(u8, page.worker_id) else ""; + errdefer if (worker_id.len > 0) alloc.free(worker_id); + const cursor = if (page.cursor.len > 0) try alloc.dupe(u8, page.cursor) else ""; + errdefer if (cursor.len > 0) alloc.free(cursor); + const last_error = if (page.last_error.len > 0) try alloc.dupe(u8, page.last_error) else ""; + errdefer if (last_error.len > 0) alloc.free(last_error); + out[i] = .{ + .phase = page.phase, + .iteration = page.iteration, + .page_id = page.page_id, + .state = page.state, + .range_kind = page.range_kind, + .worker_id = worker_id, + .lease_expires_at_ms = page.lease_expires_at_ms, + .attempt = page.attempt, + .cursor = cursor, + .completed_units = page.completed_units, + .total_units = page.total_units, + .last_error = last_error, + }; + initialized += 1; + } + return out; + } + + fn cloneGraphMetricStatusFromGraph( + alloc: Allocator, + source: graph_mod.GraphIndex.GraphMetricStatus, + ) !types.GraphMetricStatus { + var out = types.GraphMetricStatus{ + .name = try alloc.dupe(u8, source.name), + .state = source.state, + .phase = source.phase, + .metadata_version = source.metadata_version, + .config_fingerprint = source.config_fingerprint, + .maintenance_paused = source.maintenance_paused, + .build_queued = source.build_queued, + .published_generation = source.published_edge_generation, + .edge_generation = source.edge_generation, + .target_edge_generation = source.target_edge_generation, + .queued_generation = source.queued_generation, + .building_generation = source.building_generation, + .build_job_id = source.build_job_id, + .build_started_at_ms = source.build_started_at_ms, + .build_iteration = source.build_iteration, + .build_lease_expires_at_ms = source.build_lease_expires_at_ms, + .build_completed_units = source.build_completed_units, + .build_total_units = source.build_total_units, + .build_pages_truncated = source.build_pages_truncated, + .retry_count = source.retry_count, + .progress = source.progress, + .converged = source.converged, + .iterations_completed = source.iterations_completed, + .delta = source.delta, + .computed_at_ms = source.computed_at_ms, + .last_event = source.last_event, + }; + errdefer out.deinit(alloc); + out.edge_filter = try source.edge_filter.cloneAlloc(alloc); + out.build_worker_id = if (source.build_worker_id.len > 0) try alloc.dupe(u8, source.build_worker_id) else ""; + out.build_cursor = if (source.build_cursor.len > 0) try alloc.dupe(u8, source.build_cursor) else ""; + out.last_error = if (source.last_error.len > 0) try alloc.dupe(u8, source.last_error) else ""; + out.recent_events = if (source.recent_events.len > 0) + try alloc.dupe(graph_mod.GraphIndex.GraphMetricEvent, source.recent_events) + else + &.{}; + out.build_pages = try cloneGraphMetricBuildPageStatusesFromGraph(alloc, source.build_pages); + return out; + } + + fn populateGraphMetricStatusStats(alloc: Allocator, item: *types.DBIndexStats, graph_index: *graph_mod.GraphIndex) !void { + if (graph_index.metric_configs.len == 0) return; + const statuses = try alloc.alloc(types.GraphMetricStatus, graph_index.metric_configs.len); + var initialized: usize = 0; + errdefer { + for (statuses[0..initialized]) |*status| status.deinit(alloc); + alloc.free(statuses); + } + for (graph_index.metric_configs, 0..) |cfg, i| { + var status = try graph_index.graphMetricStatus(cfg.name); + defer status.deinit(graph_index.alloc); + statuses[i] = try cloneGraphMetricStatusFromGraph(alloc, status); + initialized += 1; + } + item.graph_metric_status = statuses; + } + fn managedIndexAppliedSequence(self: *DB, alloc: Allocator, index_name: []const u8) !u64 { var applied_sequence = try self.core.loadAppliedSequence(alloc, index_name); if (self.executor.appliedSequence(index_name)) |live_applied| { @@ -28584,6 +29069,7 @@ pub const DB = struct { runtime_stats.promotion = self.promotionStageStats(); runtime_stats.ttl_cleanup = if (self.ttl_runtime) |runtime| runtime.stats() else runtime_stats.ttl_cleanup; runtime_stats.transaction_recovery = if (self.transaction_runtime) |runtime| runtime.stats() else runtime_stats.transaction_recovery; + runtime_stats.graph_metric_runtime = self.graphMetricRuntimeStats(); // Runtime-only diagnostics are optional status-plane detail and may run // under apply. A contended sample is not evidence that the owner went @@ -28901,6 +29387,69 @@ pub const DB = struct { try self.overlayRuntimeStatusIndexesLocked(stats_alloc, runtime_stats); } + pub fn graphMetricRuntimeStats(self: *DB) types.GraphMetricRuntimeStats { + const runtime = self.graph_metric_runtime orelse return .{}; + const runtime_snapshot = runtime.stats(); + const total = runtime_snapshot.total_result; + const last = runtime_snapshot.last_result; + return .{ + .enabled = runtime_snapshot.enabled, + .role = switch (runtime_snapshot.role) { + .combined => .combined, + .coordinator => .coordinator, + .worker => .worker, + .worker_pool => .worker_pool, + }, + .runtime_id_hash = runtime_snapshot.runtime_id_hash, + .owner_id_hash = runtime_snapshot.owner_id_hash, + .lease_key_hash = runtime_snapshot.lease_key_hash, + .worker_id_hash = runtime_snapshot.worker_id_hash, + .worker_count = @intCast(runtime_snapshot.worker_count), + .lease_owned = runtime_snapshot.lease_owned, + .has_lease = runtime_snapshot.has_lease, + .acquisition_count = runtime_snapshot.acquisition_count, + .takeover_count = runtime_snapshot.takeover_count, + .lease_acquire_failures = runtime_snapshot.lease_acquire_failures, + .lost_leases = runtime_snapshot.lost_leases, + .last_acquired_ms = runtime_snapshot.last_acquired_ms, + .lease_expires_at_ms = runtime_snapshot.lease_expires_at_ms, + .lease_renew_after_ms = runtime_snapshot.lease_renew_after_ms, + .renewal_count = runtime_snapshot.renewal_count, + .started = runtime_snapshot.started, + .shutdown = runtime_snapshot.shutdown, + .notified = runtime_snapshot.notified, + .ticks_started = runtime_snapshot.ticks_started, + .ticks_completed = runtime_snapshot.ticks_completed, + .durable_progress_ticks = runtime_snapshot.durable_progress_ticks, + .idle_ticks = runtime_snapshot.idle_ticks, + .error_ticks = runtime_snapshot.error_ticks, + .last_error_name = runtime_snapshot.last_error_name, + .total_metrics_scanned = @intCast(total.metrics_scanned), + .total_active_builds = @intCast(total.active_builds), + .total_builds_started = @intCast(total.builds_started), + .total_worker_steps = @intCast(total.worker_steps), + .total_coordinator_steps = @intCast(total.coordinator_steps), + .total_retired_input_records = @intCast(total.retired_input_records), + .total_pages_claimed = @intCast(total.pages_claimed), + .total_pages_completed = @intCast(total.pages_completed), + .total_phases_advanced = @intCast(total.phases_advanced), + .total_published = @intCast(total.published), + .total_failed_builds = @intCast(total.failed_builds), + .last_metrics_scanned = @intCast(last.metrics_scanned), + .last_active_builds = @intCast(last.active_builds), + .last_builds_started = @intCast(last.builds_started), + .last_worker_steps = @intCast(last.worker_steps), + .last_coordinator_steps = @intCast(last.coordinator_steps), + .last_retired_input_records = @intCast(last.retired_input_records), + .last_pages_claimed = @intCast(last.pages_claimed), + .last_pages_completed = @intCast(last.pages_completed), + .last_phases_advanced = @intCast(last.phases_advanced), + .last_published = @intCast(last.published), + .last_failed_builds = @intCast(last.failed_builds), + .last_budget_exhausted = last.budget_exhausted, + }; + } + pub fn stats(self: *DB, alloc: Allocator) !types.DBStats { if (self.open_mode == .status_only) { return try self.statusOnlyStats(alloc); @@ -29249,6 +29798,7 @@ pub const DB = struct { visible_doc_count = @max(visible_doc_count, item.doc_count); } applyGraphAlgebraicRuntimeStats(&item, &entry.index); + try populateGraphMetricStatusStats(alloc, &item, &entry.index); } }, .algebraic => { @@ -29295,6 +29845,7 @@ pub const DB = struct { .ttl_cleanup = if (self.ttl_runtime) |runtime| runtime.stats() else .{}, .transaction_recovery = if (self.transaction_runtime) |runtime| runtime.stats() else .{}, .text_merge = if (self.text_merge_runtime) |runtime| runtime.statsAssumeApplyLockHeld() else self.core.index_manager.textMergeStatsSnapshot(), + .graph_metric_runtime = self.graphMetricRuntimeStats(), .term_doc_freq_cache_hits = term_doc_freq_cache_hits, .term_doc_freq_cache_misses = term_doc_freq_cache_misses, .async_indexing = async_indexing, @@ -29537,6 +30088,7 @@ pub const DB = struct { .ttl_cleanup = if (self.ttl_runtime) |runtime| runtime.stats() else .{}, .transaction_recovery = if (self.transaction_runtime) |runtime| runtime.stats() else .{}, .text_merge = if (self.text_merge_runtime) |runtime| runtime.statsAssumeApplyLockHeld() else self.core.index_manager.textMergeStats(), + .graph_metric_runtime = self.graphMetricRuntimeStats(), .term_doc_freq_cache_hits = blk: { var total: u64 = 0; for (configs) |cfg| { @@ -30548,18 +31100,24 @@ pub const DB = struct { if (externalize_artifact_ids) try externalizeSearchResultArtifactIds(alloc, &children); return children; } - const selection_req = types.canonicalGroupedMatchSelectionRequest(execution_req); + var selection_req = types.canonicalGroupedMatchSelectionRequest(execution_req); + if (execution_req.graph_metric_rerank) |rerank| { + try types.validateGraphMetricRerankWindow(rerank, execution_req.offset, execution_req.limit); + selection_req.offset = 0; + selection_req.limit = types.graphMetricRerankCandidateCount(rerank, execution_req.offset, execution_req.limit); + } if (searchRequestRequiresComposedSearch(selection_req)) { var composed = try self.searchComposed(alloc, selection_req, exec_ctx, dense_profile_sink); errdefer composed.deinit(); try self.populateCanonicalGroupedMatches(alloc, execution_req, exec_ctx, &composed); + try self.applyGraphMetricRerank(&composed, execution_req); if (externalize_artifact_ids) try externalizeSearchResultArtifactIds(alloc, &composed); return composed; } - const has_primary = selection_req.full_text != null or selection_req.dense != null or selection_req.sparse != null or !db_query_search.isDefaultMatchAll(selection_req.query) or selection_req.graph_queries.len == 0; + const has_primary = selection_req.full_text != null or selection_req.dense != null or selection_req.sparse != null or !db_query_search.isDefaultMatchAll(selection_req.query) or (selection_req.graph_queries.len == 0 and selection_req.graph_metric_queries.len == 0); - var base = if (!has_primary and selection_req.graph_queries.len > 0) + var base = if (!has_primary and (selection_req.graph_queries.len > 0 or selection_req.graph_metric_queries.len > 0)) try db_query_search.emptySearchResult(alloc) else if (selection_req.full_text) |text| try self.searchTextQuery(alloc, selection_req, text) @@ -30596,6 +31154,11 @@ pub const DB = struct { errdefer base.deinit(); try self.populateCanonicalGroupedMatches(alloc, execution_req, exec_ctx, &base); + if (execution_req.graph_metric_queries.len > 0) { + base.graph_metric_results = try self.executeGraphMetricQueries(alloc, execution_req.graph_metric_queries); + } + try self.applyGraphMetricRerank(&base, execution_req); + if (execution_req.graph_queries.len == 0) { if (externalize_artifact_ids) try externalizeSearchResultArtifactIds(alloc, &base); return base; @@ -30607,6 +31170,133 @@ pub const DB = struct { return base; } + fn executeGraphMetricQueries( + self: *DB, + alloc: Allocator, + queries: []const types.NamedGraphMetricQuery, + ) ![]types.GraphMetricResult { + if (queries.len == 0) return &.{}; + const results = try alloc.alloc(types.GraphMetricResult, queries.len); + var initialized: usize = 0; + errdefer { + for (results[0..initialized]) |*result| result.deinit(alloc); + alloc.free(results); + } + for (queries, 0..) |named, i| { + results[i] = try self.executeGraphMetricQuery(alloc, named); + initialized += 1; + } + return results; + } + + fn applyGraphMetricRerank(self: *DB, result: *types.SearchResult, req: types.SearchRequest) !void { + const rerank = req.graph_metric_rerank orelse return; + if (req.count_only) return error.UnsupportedQueryRequest; + const entry = self.core.graphIndex(rerank.index_name) orelse return error.IndexNotFound; + const node_ids = try result.alloc.alloc([]const u8, result.hits.len); + defer result.alloc.free(node_ids); + for (result.hits, 0..) |hit, i| node_ids[i] = hit.id; + var score_snapshot = try entry.index.graphMetricScoreSnapshotWithPolicyAlloc(rerank.metric_name, node_ids, .{ + .require_published = true, + .require_fresh = rerank.freshness == .fresh, + }); + defer score_snapshot.deinit(entry.index.alloc); + if (score_snapshot.status.published_generation == 0) return error.MetricNotReady; + if (rerank.freshness == .fresh and score_snapshot.status.state != .fresh) return error.MetricStale; + + var result_status = try cloneGraphMetricStatusFromGraph(result.alloc, score_snapshot.status); + errdefer result_status.deinit(result.alloc); + const selected = try graph_metric_rerank.selectPageAlloc( + result.alloc, + result.hits, + score_snapshot.scores, + .{ + .base_weight = rerank.base_weight, + .metric_weight = rerank.weight, + .missing_score = rerank.missing_score, + }, + req.offset, + req.limit, + ); + defer result.alloc.free(selected); + for (selected) |selection| { + const hit = &result.hits[selection.original_index]; + var details = types.GraphMetricRerankScoreDetails{ + .index_name = try result.alloc.dupe(u8, rerank.index_name), + .metric_name = undefined, + .base_score = selection.base_score, + .base_weight = rerank.base_weight, + .metric_score = selection.metric_score, + .metric_score_used = selection.metric_score_used, + .metric_weight = rerank.weight, + .missing_score_used = selection.metric_score == null, + .final_score = selection.final_score, + .published_generation = score_snapshot.status.published_generation, + }; + errdefer result.alloc.free(details.index_name); + details.metric_name = try result.alloc.dupe(u8, rerank.metric_name); + if (hit.score_details) |*old| old.deinit(result.alloc); + hit.score_details = details; + hit.score = selection.final_score; + } + const old_hits = result.hits; + const retained = try result.alloc.alloc(bool, old_hits.len); + defer result.alloc.free(retained); + const kept = try result.alloc.alloc(types.SearchHit, selected.len); + @memset(retained, false); + for (selected, 0..) |selection, i| { + retained[selection.original_index] = true; + kept[i] = old_hits[selection.original_index]; + old_hits[selection.original_index] = undefined; + } + for (old_hits, retained) |*hit, keep| if (!keep) hit.deinit(result.alloc); + if (old_hits.len > 0) result.alloc.free(old_hits); + result.hits = kept; + if (result.graph_metric_rerank_status) |*old| old.deinit(result.alloc); + result.graph_metric_rerank_status = result_status; + } + + fn executeGraphMetricQuery( + self: *DB, + alloc: Allocator, + named: types.NamedGraphMetricQuery, + ) !types.GraphMetricResult { + const entry = self.core.graphIndex(named.query.index_name) orelse return error.IndexNotFound; + var metric_snapshot = try entry.index.graphMetricTopKSnapshotAlloc( + named.query.metric_name, + named.query.top_k, + ); + defer metric_snapshot.deinit(entry.index.alloc); + if (named.query.freshness == .fresh and metric_snapshot.status.state != .fresh) return error.MetricStale; + + const raw_scores = metric_snapshot.scores; + const scores = try alloc.alloc(types.GraphMetricScore, raw_scores.len); + var initialized_scores: usize = 0; + errdefer { + for (scores[0..initialized_scores]) |*score| score.deinit(alloc); + alloc.free(scores); + } + for (raw_scores, 0..) |score, i| { + scores[i] = .{ .node = try alloc.dupe(u8, score.node), .score = score.score }; + initialized_scores += 1; + } + const name = try alloc.dupe(u8, named.name); + errdefer alloc.free(name); + const index_name = try alloc.dupe(u8, named.query.index_name); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, named.query.metric_name); + errdefer alloc.free(metric_name); + var owned_status = try cloneGraphMetricStatusFromGraph(alloc, metric_snapshot.status); + errdefer owned_status.deinit(alloc); + return .{ + .name = name, + .index_name = index_name, + .metric_name = metric_name, + .scores = scores, + .status = owned_status, + }; + } + fn hierarchyChildrenInaccessibleParentResult( alloc: Allocator, req: types.SearchRequest, @@ -31066,7 +31756,7 @@ pub const DB = struct { else => return null, }; if (!db_query_search.isDefaultMatchAll(req.query)) return null; - if (req.graph_queries.len != 0 or req.expand_strategy != null) return null; + if (req.graph_queries.len != 0 or req.graph_metric_queries.len != 0 or req.expand_strategy != null) return null; if (req.dense != null or req.sparse != null) return null; if (req.dense_queries.len == 1 and req.sparse_queries.len == 0) { var next = req; @@ -33213,9 +33903,23 @@ pub const DB = struct { alloc: Allocator, index_name: []const u8, keys: []const []const u8, + expected: index_manager_mod.IndexManager.CoverageIdentity, + identity_read_generation: ?u64, ) ![]bool { lockApplyShared(self); defer self.core.unlockApplyShared(); + // Validate under the same apply lease as the reverse snapshot. A + // out-of-lease check can race index replacement and certify an + // old index's negative answers under the new incarnation's cache key. + if (expected.generation == 0 or expected.config_fingerprint == null) + return error.InvalidArgument; + const actual = self.core.index_manager.coverageIdentityForIndex(index_name) orelse + return error.IndexGenerationMismatch; + if (actual.generation != expected.generation or actual.config_fingerprint != expected.config_fingerprint) + return error.IndexGenerationMismatch; + // Reverse-only probes skip document hydration, but their routing + // cache keys still bind the source shard's document/read generation. + _ = try self.currentIdentityReadGenerationForRequest(identity_read_generation); const graph_entry = self.core.graphIndex(index_name) orelse return error.IndexNotFound; return try graph_entry.index.hasIncomingEdgesManyAlloc(alloc, keys); } @@ -99263,6 +99967,32 @@ test "db unfiltered graph search retains algebraic execution" { return error.TestExpectedEqual; } +test "db reverse graph probe rejects a deleted or replaced index incarnation" { + const alloc = std.testing.allocator; + var directory = try TestDirectory.init("graph-incarnation"); + defer directory.cleanup(); + const path = directory.path().ptr; + var db = try DB.open(alloc, std.mem.span(path), .{}); + defer db.close(); + const cfg = types.IndexConfig{ .name = "graph_idx", .kind = .graph, .config_json = "{}" }; + try db.addIndex(cfg); + const previous = db.core.index_manager.coverageIdentityForIndex("graph_idx").?; + try std.testing.expect(try db.deleteIndex("graph_idx")); + // Even an empty probe must not certify a missing or replacement index. + try std.testing.expectError(error.IndexGenerationMismatch, db.graphHasIncomingEdgesForInternalRead(alloc, "graph_idx", &.{}, previous, null)); + // Same-name admission waits for the retired incarnation's asynchronous + // artifact cleanup. Join its owner instead of racing it or sleeping. + db.backend_runtime.durable_jobs.drainOwner(db.repair_cleanup_owner_id); + try db.addIndex(cfg); + const current = db.core.index_manager.coverageIdentityForIndex("graph_idx").?; + try std.testing.expect(previous.generation != current.generation); + try std.testing.expectEqual(previous.config_fingerprint, current.config_fingerprint); + try std.testing.expectError(error.IndexGenerationMismatch, db.graphHasIncomingEdgesForInternalRead(alloc, "graph_idx", &.{"absent"}, previous, null)); + const incoming = try db.graphHasIncomingEdgesForInternalRead(alloc, "graph_idx", &.{"absent"}, current, null); + defer alloc.free(incoming); + try std.testing.expectEqualSlices(bool, &.{false}, incoming); +} + test "db graph search filters result nodes and hidden traversal intermediates" { const alloc = std.testing.allocator; @@ -99346,10 +100076,23 @@ test "db graph search filters result nodes and hidden traversal intermediates" { try std.testing.expect(std.mem.indexOf(u8, hit.stored_data.?, "\"tenant\":\"visible\"") != null); } + const graph_identity = db.core.index_manager.coverageIdentityForIndex("gr_v1").?; + var stale_graph_identity = graph_identity; + stale_graph_identity.generation ^= 1; + try std.testing.expectError(error.IndexGenerationMismatch, db.graphHasIncomingEdgesForInternalRead(alloc, "gr_v1", &.{"n:b"}, stale_graph_identity, null)); + stale_graph_identity = graph_identity; + stale_graph_identity.config_fingerprint = graph_identity.config_fingerprint.? ^ 1; + try std.testing.expectError(error.IndexGenerationMismatch, db.graphHasIncomingEdgesForInternalRead(alloc, "gr_v1", &.{"n:b"}, stale_graph_identity, null)); + try std.testing.expectError(error.IndexGenerationMismatch, db.graphHasIncomingEdgesForInternalRead(alloc, "missing", &.{"n:b"}, graph_identity, null)); + try std.testing.expectError(error.InvalidArgument, db.graphHasIncomingEdgesForInternalRead(alloc, "gr_v1", &.{"n:b"}, .{ .generation = 0, .config_fingerprint = null }, null)); + const read_generation = try db.currentIdentityReadGenerationForRequest(null); + try std.testing.expectError(error.IdentityReadGenerationChanged, db.graphHasIncomingEdgesForInternalRead(alloc, "gr_v1", &.{"n:b"}, graph_identity, read_generation ^ 1)); const incoming = try db.graphHasIncomingEdgesForInternalRead( alloc, "gr_v1", &.{ "n:a", "n:b", "n:c", "n:d", "n:missing" }, + graph_identity, + read_generation, ); defer alloc.free(incoming); try std.testing.expectEqualSlices( diff --git a/zig/pkg/antfly/src/storage/db/derived/derived_executor.zig b/zig/pkg/antfly/src/storage/db/derived/derived_executor.zig index 9433f03984..157d4478e4 100644 --- a/zig/pkg/antfly/src/storage/db/derived/derived_executor.zig +++ b/zig/pkg/antfly/src/storage/db/derived/derived_executor.zig @@ -23,25 +23,9 @@ const runtime_backend = @import("../../runtime_backend.zig"); const background_runtime_mod = @import("../../background_runtime.zig"); const index_manager_mod = @import("../catalog/index_manager.zig"); const types = @import("../types.zig"); -const platform_clock = @import("antfly_platform").clock; const platform_time = @import("antfly_platform").time; -pub const VisibilityWait = struct { - cancellation: types.CancellationToken = .none, - deadline_ns: ?u64 = null, - clock: ?platform_clock.Clock = null, - - pub fn check(self: @This()) !void { - if (self.cancellation.isCancelled()) return error.EnrichmentWaitCanceled; - if (self.deadline_ns) |deadline_ns| { - const now_ns = if (self.clock) |clock| - clock.nowRealtimeNs() - else - platform_time.monotonicNs(); - if (now_ns >= deadline_ns) return error.EnrichmentWaitTimeout; - } - } -}; +pub const VisibilityWait = runtime_types.VisibilityWait; const runtime_types = @import("runtime_types.zig"); const derived_worker = @import("derived_worker.zig"); @@ -762,10 +746,10 @@ fn ioThreadedReleaseBacklogThrough(ptr: *anyopaque, sequence: u64) void { fn ioThreadedWaitForAll(ptr: *anyopaque, sequence: u64, wait: VisibilityWait) !void { const runtime: *io_threaded_runtime_mod.DerivedRuntime = @ptrCast(@alignCast(ptr)); - return try runtime.waitForAllWithVisibilityWait(sequence, wait.cancellation, wait.deadline_ns); + return try runtime.waitForAllWithVisibilityWait(sequence, wait); } fn ioThreadedWaitForIndexes(ptr: *anyopaque, sequence: u64, index_names: []const []const u8, wait: VisibilityWait) !void { const runtime: *io_threaded_runtime_mod.DerivedRuntime = @ptrCast(@alignCast(ptr)); - return try runtime.waitForIndexesWithVisibilityWait(sequence, index_names, wait.cancellation, wait.deadline_ns); + return try runtime.waitForIndexesWithVisibilityWait(sequence, index_names, wait); } diff --git a/zig/pkg/antfly/src/storage/db/derived/derived_types.zig b/zig/pkg/antfly/src/storage/db/derived/derived_types.zig index 53a286a9ab..a3a98e3206 100644 --- a/zig/pkg/antfly/src/storage/db/derived/derived_types.zig +++ b/zig/pkg/antfly/src/storage/db/derived/derived_types.zig @@ -14,8 +14,8 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const types = @import("../types.zig"); const enrichment_types = @import("../enrichment/enrichment_types.zig"); +const graph_edge_types = @import("../graph_edge_types.zig"); pub const DerivedAction = enum { upsert, @@ -74,8 +74,8 @@ pub const DerivedBatch = struct { dense_embeddings: []const DerivedDenseEmbeddingWrite = &.{}, sparse_embeddings: []const DerivedSparseEmbeddingWrite = &.{}, generated_enrichment_refs: []const enrichment_types.GeneratedEnrichmentRef = &.{}, - graph_writes: []const types.GraphEdgeWrite = &.{}, - graph_deletes: []const types.GraphEdgeDelete = &.{}, + graph_writes: []const graph_edge_types.GraphEdgeWrite = &.{}, + graph_deletes: []const graph_edge_types.GraphEdgeDelete = &.{}, }; pub const DerivedLogRecord = struct { @@ -128,7 +128,7 @@ pub fn deinitDerivedSparseEmbedding(alloc: Allocator, embedding: DerivedSparseEm if (embedding.values.len > 0) alloc.free(embedding.values); } -pub fn deinitDerivedGraphWrite(alloc: Allocator, write: types.GraphEdgeWrite) void { +pub fn deinitDerivedGraphWrite(alloc: Allocator, write: graph_edge_types.GraphEdgeWrite) void { alloc.free(@constCast(write.index_name)); alloc.free(@constCast(write.source)); alloc.free(@constCast(write.target)); @@ -136,7 +136,7 @@ pub fn deinitDerivedGraphWrite(alloc: Allocator, write: types.GraphEdgeWrite) vo if (write.metadata_json.len > 0) alloc.free(@constCast(write.metadata_json)); } -pub fn deinitDerivedGraphDelete(alloc: Allocator, delete: types.GraphEdgeDelete) void { +pub fn deinitDerivedGraphDelete(alloc: Allocator, delete: graph_edge_types.GraphEdgeDelete) void { alloc.free(@constCast(delete.index_name)); alloc.free(@constCast(delete.source)); alloc.free(@constCast(delete.target)); @@ -234,7 +234,7 @@ pub fn cloneDerivedSparseEmbedding( }; } -pub fn cloneDerivedGraphWrite(alloc: Allocator, write: types.GraphEdgeWrite) !types.GraphEdgeWrite { +pub fn cloneDerivedGraphWrite(alloc: Allocator, write: graph_edge_types.GraphEdgeWrite) !graph_edge_types.GraphEdgeWrite { const index_name = try alloc.dupe(u8, write.index_name); errdefer alloc.free(index_name); const source = try alloc.dupe(u8, write.source); @@ -259,7 +259,7 @@ pub fn cloneDerivedGraphWrite(alloc: Allocator, write: types.GraphEdgeWrite) !ty }; } -pub fn cloneDerivedGraphDelete(alloc: Allocator, delete: types.GraphEdgeDelete) !types.GraphEdgeDelete { +pub fn cloneDerivedGraphDelete(alloc: Allocator, delete: graph_edge_types.GraphEdgeDelete) !graph_edge_types.GraphEdgeDelete { const index_name = try alloc.dupe(u8, delete.index_name); errdefer alloc.free(index_name); const source = try alloc.dupe(u8, delete.source); @@ -393,7 +393,7 @@ pub fn cloneBatch(alloc: Allocator, batch: DerivedBatch) !DerivedBatch { const generated_enrichment_refs = try enrichment_types.cloneGeneratedRefs(alloc, batch.generated_enrichment_refs); errdefer enrichment_types.deinitGeneratedRefs(alloc, generated_enrichment_refs); - var graph_writes = try alloc.alloc(types.GraphEdgeWrite, batch.graph_writes.len); + var graph_writes = try alloc.alloc(graph_edge_types.GraphEdgeWrite, batch.graph_writes.len); var initialized_graph_writes: usize = 0; errdefer { for (graph_writes[0..initialized_graph_writes]) |write| @@ -405,7 +405,7 @@ pub fn cloneBatch(alloc: Allocator, batch: DerivedBatch) !DerivedBatch { initialized_graph_writes += 1; } - var graph_deletes = try alloc.alloc(types.GraphEdgeDelete, batch.graph_deletes.len); + var graph_deletes = try alloc.alloc(graph_edge_types.GraphEdgeDelete, batch.graph_deletes.len); var initialized_graph_deletes: usize = 0; errdefer { for (graph_deletes[0..initialized_graph_deletes]) |delete| @@ -806,7 +806,7 @@ fn decodeBinaryLogRecord(alloc: Allocator, payload: []const u8) !DecodedLogRecor batch.generated_enrichment_refs = generated_enrichment_refs; const graph_write_count = try reader.readInt(u32); - const graph_writes = try alloc.alloc(types.GraphEdgeWrite, graph_write_count); + const graph_writes = try alloc.alloc(graph_edge_types.GraphEdgeWrite, graph_write_count); errdefer alloc.free(graph_writes); var initialized_graph_writes: usize = 0; errdefer { @@ -834,7 +834,7 @@ fn decodeBinaryLogRecord(alloc: Allocator, payload: []const u8) !DecodedLogRecor batch.graph_writes = graph_writes; const graph_delete_count = try reader.readInt(u32); - const graph_deletes = try alloc.alloc(types.GraphEdgeDelete, graph_delete_count); + const graph_deletes = try alloc.alloc(graph_edge_types.GraphEdgeDelete, graph_delete_count); errdefer alloc.free(graph_deletes); var initialized_graph_deletes: usize = 0; errdefer { diff --git a/zig/pkg/antfly/src/storage/db/derived/io_threaded_runtime.zig b/zig/pkg/antfly/src/storage/db/derived/io_threaded_runtime.zig index 10bd275134..37691699da 100644 --- a/zig/pkg/antfly/src/storage/db/derived/io_threaded_runtime.zig +++ b/zig/pkg/antfly/src/storage/db/derived/io_threaded_runtime.zig @@ -204,11 +204,9 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { pub fn waitForAllWithVisibilityWait( self: *@This(), sequence: u64, - cancellation: types.CancellationToken, - deadline_ns: ?u64, + wait: runtime_types.VisibilityWait, ) !void { - _ = cancellation; - _ = deadline_ns; + _ = wait; return try self.waitForAll(sequence); } @@ -223,11 +221,9 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { self: *@This(), sequence: u64, index_names: []const []const u8, - cancellation: types.CancellationToken, - deadline_ns: ?u64, + wait: runtime_types.VisibilityWait, ) !void { - _ = cancellation; - _ = deadline_ns; + _ = wait; return try self.waitForIndexes(sequence, index_names); } } else struct { @@ -623,14 +619,13 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { } pub fn waitForAll(self: *DerivedRuntime, sequence: u64) !void { - return try self.waitForAllWithVisibilityWait(sequence, .none, null); + return try self.waitForAllWithVisibilityWait(sequence, .{}); } pub fn waitForAllWithVisibilityWait( self: *DerivedRuntime, sequence: u64, - cancellation: types.CancellationToken, - deadline_ns: ?u64, + wait: runtime_types.VisibilityWait, ) !void { const io = self.ioContext(); self.mutex.lockUncancelable(io); @@ -691,7 +686,7 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { self.mutex.unlock(io); io.sleep(Io.Duration.zero, .awake) catch {}; self.mutex.lockUncancelable(io); - try checkVisibilityWait(cancellation, deadline_ns); + try wait.check(); continue; } const truncate_sequence = truncate: { @@ -713,7 +708,7 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { } return; } - try checkVisibilityWait(cancellation, deadline_ns); + try wait.check(); self.mutex.unlock(io); io.sleep(Io.Duration.fromNanoseconds(std.time.ns_per_ms), .awake) catch {}; self.mutex.lockUncancelable(io); @@ -721,15 +716,14 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { } pub fn waitForIndexes(self: *DerivedRuntime, sequence: u64, index_names: []const []const u8) !void { - return try self.waitForIndexesWithVisibilityWait(sequence, index_names, .none, null); + return try self.waitForIndexesWithVisibilityWait(sequence, index_names, .{}); } pub fn waitForIndexesWithVisibilityWait( self: *DerivedRuntime, sequence: u64, index_names: []const []const u8, - cancellation: types.CancellationToken, - deadline_ns: ?u64, + wait: runtime_types.VisibilityWait, ) !void { if (index_names.len == 0) return; const io = self.ioContext(); @@ -796,7 +790,7 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { self.mutex.unlock(io); io.sleep(Io.Duration.zero, .awake) catch {}; self.mutex.lockUncancelable(io); - try checkVisibilityWait(cancellation, deadline_ns); + try wait.check(); continue; } const truncate_sequence = truncate: { @@ -818,7 +812,7 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { } return; } - try checkVisibilityWait(cancellation, deadline_ns); + try wait.check(); self.mutex.unlock(io); io.sleep(Io.Duration.fromNanoseconds(std.time.ns_per_ms), .awake) catch {}; self.mutex.lockUncancelable(io); @@ -843,24 +837,23 @@ pub const DerivedRuntime = if (builtin.os.tag == .freestanding) struct { } }; -fn checkVisibilityWait(cancellation: types.CancellationToken, deadline_ns: ?u64) !void { - if (cancellation.isCancelled()) return error.EnrichmentWaitCanceled; - if (deadline_ns) |deadline| { - if (platform_time.monotonicNs() >= deadline) return error.EnrichmentWaitTimeout; - } -} - test "derived enrichment visibility guard observes cancellation and deadline" { var cancelled = std.atomic.Value(bool).init(true); try std.testing.expectError( error.EnrichmentWaitCanceled, - checkVisibilityWait(types.CancellationToken.fromAtomic(&cancelled), null), + (runtime_types.VisibilityWait{ .cancellation = types.CancellationToken.fromAtomic(&cancelled) }).check(), ); cancelled.store(false, .release); try std.testing.expectError( error.EnrichmentWaitTimeout, - checkVisibilityWait(.none, platform_time.monotonicNs()), + (runtime_types.VisibilityWait{ .deadline_ns = platform_time.monotonicNs() }).check(), ); + var clock = @import("antfly_platform").clock.ManualClock{}; + clock.setRealtimeNs(100); + const wait = runtime_types.VisibilityWait{ .clock = clock.clock(), .deadline_ns = 200 }; + try wait.check(); + clock.setRealtimeNs(200); + try std.testing.expectError(error.EnrichmentWaitTimeout, wait.check()); } fn workerStep(worker: *Worker) ?u64 { diff --git a/zig/pkg/antfly/src/storage/db/derived/runtime_types.zig b/zig/pkg/antfly/src/storage/db/derived/runtime_types.zig index 88019f6d4d..5071e304d2 100644 --- a/zig/pkg/antfly/src/storage/db/derived/runtime_types.zig +++ b/zig/pkg/antfly/src/storage/db/derived/runtime_types.zig @@ -16,6 +16,24 @@ const derived_types = @import("derived_types.zig"); const index_manager_mod = @import("../catalog/index_manager.zig"); +const types = @import("../types.zig"); +const platform = @import("antfly_platform"); + +/// Deadline and clock are one contract across manual and borrowed-Io workers. +/// Never forward the absolute timestamp while discarding its clock domain. +pub const VisibilityWait = struct { + cancellation: types.CancellationToken = .none, + deadline_ns: ?u64 = null, + clock: ?platform.clock.Clock = null, + + pub fn check(self: @This()) !void { + if (self.cancellation.isCancelled()) return error.EnrichmentWaitCanceled; + if (self.deadline_ns) |deadline_ns| { + const now_ns = if (self.clock) |clock| clock.nowRealtimeNs() else platform.time.monotonicNs(); + if (now_ns >= deadline_ns) return error.EnrichmentWaitTimeout; + } + } +}; pub const RuntimeError = error{AsyncWorkerFailed}; diff --git a/zig/pkg/antfly/src/storage/db/graph_edge_types.zig b/zig/pkg/antfly/src/storage/db/graph_edge_types.zig new file mode 100644 index 0000000000..bb6337949a --- /dev/null +++ b/zig/pkg/antfly/src/storage/db/graph_edge_types.zig @@ -0,0 +1,31 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +pub const GraphEdgeWrite = struct { + index_name: []const u8, + source: []const u8, + target: []const u8, + edge_type: []const u8, + weight: f64 = 1.0, + created_at: u64 = 0, + updated_at: u64 = 0, + metadata_json: []const u8 = "", +}; + +pub const GraphEdgeDelete = struct { + index_name: []const u8, + source: []const u8, + target: []const u8, + edge_type: []const u8, +}; diff --git a/zig/pkg/antfly/src/storage/db/graph_runtime.zig b/zig/pkg/antfly/src/storage/db/graph_runtime.zig new file mode 100644 index 0000000000..d02011c5ca --- /dev/null +++ b/zig/pkg/antfly/src/storage/db/graph_runtime.zig @@ -0,0 +1,155 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const builtin = @import("builtin"); + +const graph_mod = @import("../../graph/graph.zig"); +const paths_mod = @import("../../graph/paths.zig"); +const traversal_mod = @import("../../graph/traversal.zig"); +const types = @import("types.zig"); + +const TestHelpers = if (builtin.is_test) @import("test_support.zig") else struct {}; + +test "db graph runtime helpers expose edges neighbors and shortest path" { + const DB = @import("mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{}); + defer db.close(); + + try db.addIndex(.{ + .name = "citations", + .kind = .graph, + .config_json = "{}", + }); + try db.addIndex(.{ + .name = "citations_alg", + .kind = .graph, + .config_json = "{\"algebraic_planning\":{\"bounded_traversal\":{\"law\":\"provenance_semiring\"}}}", + }); + + try db.batch(.{ + .graph_writes = &.{ + .{ .index_name = "citations", .source = "a", .target = "b", .edge_type = "cites", .weight = 1.0 }, + .{ .index_name = "citations", .source = "a", .target = "c", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations", .source = "b", .target = "d", .edge_type = "cites", .weight = 3.0 }, + .{ .index_name = "citations_alg", .source = "a", .target = "b", .edge_type = "cites", .weight = 1.0 }, + .{ .index_name = "citations_alg", .source = "a", .target = "c", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations_alg", .source = "b", .target = "d", .edge_type = "cites", .weight = 3.0 }, + }, + .sync_level = .full_index, + }); + + const edges = try db.getEdges(alloc, "citations", "a", "", .out); + defer graph_mod.GraphIndex.freeEdges(alloc, edges); + try std.testing.expectEqual(@as(usize, 2), edges.len); + + const neighbors = try db.getNeighbors(alloc, "citations", "a", "cites", .out); + defer traversal_mod.freeOwnedResults(alloc, neighbors); + try std.testing.expectEqual(@as(usize, 2), neighbors.len); + + const traversed = try db.traverseEdges(alloc, "citations", "a", .{ + .direction = .out, + .edge_types = &.{"cites"}, + .max_depth = 2, + }); + defer traversal_mod.freeOwnedResults(alloc, traversed); + try std.testing.expectEqual(@as(usize, 3), traversed.len); + + const shortest = (try db.findShortestPath(alloc, "citations", "a", "d", &.{"cites"}, .out, .min_hops, 8, null, null)).?; + defer paths_mod.freePath(alloc, shortest); + try std.testing.expectEqual(@as(u32, 2), shortest.length); + try std.testing.expectEqual(@as(usize, 3), shortest.nodes.len); + try std.testing.expectEqualStrings("a", shortest.nodes[0]); + try std.testing.expectEqualStrings("d", shortest.nodes[2]); + + const algebraic_shortest = (try db.findShortestPath(alloc, "citations_alg", "a", "d", &.{"cites"}, .out, .min_hops, 8, null, null)).?; + defer paths_mod.freePath(alloc, algebraic_shortest); + try std.testing.expectEqual(@as(u32, 2), algebraic_shortest.length); + try std.testing.expectEqual(@as(usize, 3), algebraic_shortest.nodes.len); + try std.testing.expectEqualStrings("a", algebraic_shortest.nodes[0]); + try std.testing.expectEqualStrings("b", algebraic_shortest.nodes[1]); + try std.testing.expectEqualStrings("d", algebraic_shortest.nodes[2]); + try std.testing.expectEqual(@as(usize, 2), algebraic_shortest.edges.len); + try std.testing.expectEqualStrings("cites", algebraic_shortest.edges[0].edge_type); + + const algebraic_k_one = try db.findKShortestPaths(alloc, "citations_alg", "a", "d", 1, &.{"cites"}, .out, .min_hops, 8, null, null); + defer paths_mod.freePaths(alloc, algebraic_k_one); + try std.testing.expectEqual(@as(usize, 1), algebraic_k_one.len); + try std.testing.expectEqual(@as(u32, 2), algebraic_k_one[0].length); + try std.testing.expectEqualStrings("d", algebraic_k_one[0].nodes[2]); + + const stats = try db.stats(alloc); + defer types.freeDBStats(alloc, stats); + var found_alg_stats = false; + for (stats.indexes) |item| { + if (!std.mem.eql(u8, item.name, "citations_alg")) continue; + found_alg_stats = true; + try std.testing.expect(item.algebraic_graph_traversal_attempt_count > 0); + try std.testing.expect(item.algebraic_graph_traversal_proven_count > 0); + try std.testing.expect(item.algebraic_graph_traversal_result_node_count > 0); + } + try std.testing.expect(found_alg_stats); +} + +test "db graph runtime algebraic shortest path applies exact min-hop edge weight filters" { + const DB = @import("mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{}); + defer db.close(); + + try db.addIndex(.{ + .name = "citations_alg", + .kind = .graph, + .config_json = "{\"algebraic_planning\":{\"bounded_traversal\":{\"law\":\"provenance_semiring\"}}}", + }); + + try db.batch(.{ + .graph_writes = &.{ + .{ .index_name = "citations_alg", .source = "a", .target = "b", .edge_type = "cites", .weight = 0.5 }, + .{ .index_name = "citations_alg", .source = "b", .target = "d", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations_alg", .source = "a", .target = "c", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations_alg", .source = "c", .target = "e", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations_alg", .source = "e", .target = "d", .edge_type = "cites", .weight = 2.0 }, + .{ .index_name = "citations_alg", .source = "c", .target = "d", .edge_type = "cites", .weight = 5.0 }, + }, + .sync_level = .full_index, + }); + + const shortest = (try db.findShortestPath(alloc, "citations_alg", "a", "d", &.{"cites"}, .out, .min_hops, 4, 1.0, 3.0)).?; + defer paths_mod.freePath(alloc, shortest); + try std.testing.expectEqual(@as(u32, 3), shortest.length); + try std.testing.expectEqual(@as(usize, 4), shortest.nodes.len); + try std.testing.expectEqualStrings("a", shortest.nodes[0]); + try std.testing.expectEqualStrings("c", shortest.nodes[1]); + try std.testing.expectEqualStrings("e", shortest.nodes[2]); + try std.testing.expectEqualStrings("d", shortest.nodes[3]); + + const algebraic_k_one = try db.findKShortestPaths(alloc, "citations_alg", "a", "d", 1, &.{"cites"}, .out, .min_hops, 4, 1.0, 3.0); + defer paths_mod.freePaths(alloc, algebraic_k_one); + try std.testing.expectEqual(@as(usize, 1), algebraic_k_one.len); + try std.testing.expectEqual(@as(u32, 3), algebraic_k_one[0].length); + try std.testing.expectEqualStrings("c", algebraic_k_one[0].nodes[1]); + try std.testing.expectEqualStrings("d", algebraic_k_one[0].nodes[3]); +} diff --git a/zig/pkg/antfly/src/storage/db/lease.zig b/zig/pkg/antfly/src/storage/db/lease.zig index f003fc60de..bc8943cac3 100644 --- a/zig/pkg/antfly/src/storage/db/lease.zig +++ b/zig/pkg/antfly/src/storage/db/lease.zig @@ -32,6 +32,14 @@ pub const AcquireResult = struct { acquired: bool, epoch: u64 = 0, expires_at_ms: u64 = 0, + kind: AcquireKind = .blocked, +}; + +pub const AcquireKind = enum { + acquired, + renewed, + takeover, + blocked, }; pub const Lease = struct { @@ -66,6 +74,7 @@ pub const Lease = struct { .allocate = .alloc_always, }); defer parsed.deinit(); + if (parsed.value.expires_at_ms == 0) return null; return try cloneRecord(alloc, parsed.value); } @@ -84,6 +93,7 @@ pub const Lease = struct { }; var epoch: u64 = 1; + var kind: AcquireKind = .acquired; if (current_raw) |raw| { const parsed = try std.json.parseFromSlice(LeaseRecord, self.allocator, raw, .{ .allocate = .alloc_always, @@ -98,6 +108,7 @@ pub const Lease = struct { @max(current.epoch, 1) else std.math.add(u64, current.epoch, 1) catch return error.LeaseEpochOverflow; + kind = if (current.expires_at_ms == 0) .acquired else if (current.expires_at_ms > now_ms) .renewed else .takeover; } const expires_at_ms = std.math.add(u64, now_ms, ttl_ms) catch std.math.maxInt(u64); @@ -112,7 +123,7 @@ pub const Lease = struct { try txn.put(self.key, payload); try txn.commit(); committed = true; - return .{ .acquired = true, .epoch = epoch, .expires_at_ms = expires_at_ms }; + return .{ .acquired = true, .epoch = epoch, .expires_at_ms = expires_at_ms, .kind = kind }; } pub fn renew(self: *Lease, owner_id: []const u8, now_ms: u64, ttl_ms: u64) !bool { @@ -172,8 +183,17 @@ pub const Lease = struct { defer parsed.deinit(); if (!std.mem.eql(u8, parsed.value.owner_id, owner_id) or + parsed.value.expires_at_ms == 0 or (epoch != null and parsed.value.epoch != epoch.?)) return false; - try txn.delete(self.key); + // Keep the tenure counter after release. Deleting it would let a + // restarted owner reuse epoch one and revive stale work or releases. + const released = try std.json.Stringify.valueAlloc(self.allocator, LeaseRecord{ + .owner_id = "", + .expires_at_ms = 0, + .epoch = parsed.value.epoch, + }, .{}); + defer self.allocator.free(released); + try txn.put(self.key, released); try txn.commit(); committed = true; return true; @@ -292,6 +312,30 @@ test "lease epochs fence renewal and release after takeover" { try std.testing.expect(try lease.releaseFenced("stable-worker-id", second.epoch)); } +test "lease release preserves tenure fencing across owner ID reuse" { + const alloc = std.testing.allocator; + var backend = mem_backend.Backend.init(alloc, .{}); + defer backend.close(); + var runtime = try backend.runtimeStore(alloc, .{ .name = "lease-tenure" }); + defer runtime.deinit(); + var lease = try Lease.init(alloc, runtime, "\x00\x00__metadata__:lease_tenure"); + defer lease.deinit(); + const first = try lease.tryAcquireFenced("worker", 1000, 250); + try std.testing.expect(first.acquired); + try std.testing.expect(try lease.releaseFenced("worker", first.epoch)); + try std.testing.expect((try lease.load(alloc)) == null); + const second = try lease.tryAcquireFenced("worker", 1100, 250); + try std.testing.expect(second.acquired); + try std.testing.expect(second.epoch > first.epoch); + try std.testing.expect(!(try lease.releaseFenced("worker", first.epoch))); + try std.testing.expect(!(try lease.renewFenced("worker", first.epoch, 1200, 250))); + try std.testing.expect(try lease.renewFenced("worker", second.epoch, 1200, 250)); + const third = try lease.tryAcquireFenced("worker", 1500, 250); + try std.testing.expect(third.epoch > second.epoch); + try std.testing.expectEqual(AcquireKind.takeover, third.kind); + try std.testing.expect(!(try lease.renewFenced("worker", second.epoch, 1501, 250))); +} + test "lease works with memory backend store" { const alloc = std.testing.allocator; var backend = mem_backend.Backend.init(alloc, .{}); diff --git a/zig/pkg/antfly/src/storage/db/maintenance/graph_metric_runtime.zig b/zig/pkg/antfly/src/storage/db/maintenance/graph_metric_runtime.zig new file mode 100644 index 0000000000..fbda495abb --- /dev/null +++ b/zig/pkg/antfly/src/storage/db/maintenance/graph_metric_runtime.zig @@ -0,0 +1,13256 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the Elastic License 2.0 is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See +// the Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const builtin = @import("builtin"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const apply_rw_lock_mod = @import("../apply_rw_lock.zig"); +const index_manager_mod = @import("../catalog/index_manager.zig"); +const lease_mod = @import("../lease.zig"); +const ownership_mod = @import("../ownership.zig"); +const graph_mod = @import("../../../graph/graph.zig"); +const graph_query_mod = @import("../../../graph/query.zig"); +const platform_clock = @import("antfly_platform").clock; +const types = @import("../types.zig"); +const background_runtime_mod = @import("../../background_runtime.zig"); + +fn yieldToBackground(db: anytype) void { + if (db.backend_runtime.io()) |io| { + io.sleep(Io.Duration.fromMilliseconds(10), .awake) catch {}; + return; + } + platform_clock.Clock.real().sleepMs(10); +} + +const TestHelpers = if (builtin.is_test) @import("../test_support.zig") else struct {}; +const max_runtime_workers: usize = 64; + +pub const Role = enum { + combined, + coordinator, + worker, + worker_pool, +}; + +pub const Config = struct { + enabled: bool = false, + start_background_loop: bool = true, + role: Role = .combined, + runtime_id: []const u8 = "", + lease_owned: bool = false, + /// Process-incarnation identity used to fence the runtime lease. This must + /// be unique across concurrent processes and process restarts. + owner_id: []const u8 = "local", + lease_ttl_ms: u64 = 30_000, + coordinator_start_background_builds: bool = true, + /// A short cooperative pause after durable progress prevents a busy graph + /// build from monopolizing storage bandwidth while still sustaining high + /// maintenance throughput. + active_interval_ms: u64 = 2, + idle_interval_ms: u64 = 50, + error_interval_ms: u64 = 250, + // One bounded unit of durable work per runtime tick keeps foreground + // writer latency predictable. Dedicated maintenance commands can opt into + // larger batches explicitly. + planned_options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions = .{ + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + clock: platform_clock.Clock = platform_clock.Clock.real(), +}; + +pub const Stats = struct { + enabled: bool = false, + role: Role = .combined, + runtime_id_hash: u64 = 0, + owner_id_hash: u64 = 0, + lease_key_hash: u64 = 0, + worker_id_hash: u64 = 0, + worker_count: usize = 0, + lease_owned: bool = false, + has_lease: bool = false, + acquisition_count: u64 = 0, + takeover_count: u64 = 0, + lease_acquire_failures: u64 = 0, + lost_leases: u64 = 0, + last_acquired_ms: u64 = 0, + lease_expires_at_ms: u64 = 0, + lease_renew_after_ms: u64 = 0, + renewal_count: u64 = 0, + started: bool = false, + shutdown: bool = false, + notified: bool = false, + ticks_started: u64 = 0, + ticks_completed: u64 = 0, + durable_progress_ticks: u64 = 0, + idle_ticks: u64 = 0, + error_ticks: u64 = 0, + last_error_name: ?[]const u8 = null, + total_result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult = .{}, + last_result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult = .{}, +}; + +fn prepareTopologyForRuntimeTest(db: *@import("../mod.zig").DB, name: []const u8) !void { + const entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const cfg = for (entry.metric_configs) |cfg| { + if (std.mem.eql(u8, cfg.name, name)) break cfg; + } else return error.MetricNotConfigured; + while (!try entry.index.prepareGraphMetricPartitionStep(4096)) {} + for (0..512) |_| { + if (try entry.index.prepareGraphMetricTopology(cfg, entry.index.edge_generation)) break; + _ = try entry.index.runGraphMetricTopologyPreparationStep("runtime-fixture-preparation"); + } else return error.TopologyPreparationDidNotSeal; + for (0..32) |_| if (!try entry.index.runGraphMetricTopologyPreparationStep("runtime-fixture-cleanup")) break; +} + +pub fn expectPlannedAutoIdleDecision( + index_manager: *index_manager_mod.IndexManager, + options: index_manager_mod.IndexManager.GraphMetricPlannedAutoIdleOptions, + should_run_planned: bool, + active_builds: usize, + eligible_queued: usize, + deferred_queued: usize, + ineligible_queued: usize, +) !void { + const decision = try index_manager.graphMetricPlannedAutoIdleDecision(options); + try std.testing.expectEqual(should_run_planned, decision.shouldRunPlanned()); + try std.testing.expectEqual(active_builds, decision.active_builds); + try std.testing.expectEqual(eligible_queued, decision.eligible_queued); + try std.testing.expectEqual(deferred_queued, decision.deferred_queued); + try std.testing.expectEqual(ineligible_queued, decision.ineligible_queued); +} + +pub fn expectDegreeCanaryDecision( + index_manager: *index_manager_mod.IndexManager, + options: index_manager_mod.IndexManager.GraphMetricDegreeCanaryOptions, + should_run_planned: bool, + active_degree_builds: usize, + eligible_queued_degree: usize, + blocked_active_non_degree: usize, + blocked_queued_non_degree: usize, +) !void { + const decision = try index_manager.graphMetricDegreeCanaryDecision(options); + try std.testing.expectEqual(should_run_planned, decision.shouldRunPlanned()); + try std.testing.expectEqual(active_degree_builds, decision.active_degree_builds); + try std.testing.expectEqual(eligible_queued_degree, decision.eligible_queued_degree); + try std.testing.expectEqual(blocked_active_non_degree, decision.blocked_active_non_degree); + try std.testing.expectEqual(blocked_queued_non_degree, decision.blocked_queued_non_degree); + try std.testing.expectEqual(@as(usize, 0), decision.failed_pages); + try std.testing.expect(!decision.truncated_pages); +} + +pub const MaintenanceBoundary = struct { + ptr: *anyopaque, + vtable: *const VTable, + + pub const WorkerPoolSweepOptions = struct { + worker_id: []const u8 = "", + worker_ids: []const []const u8 = &.{}, + max_pages: usize = 64, + now_ms: ?u64 = null, + }; + + pub const VTable = struct { + run_combined: *const fn ( + *anyopaque, + index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, + ) anyerror!index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + run_coordinator: *const fn ( + *anyopaque, + index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions, + ) anyerror!index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + run_worker: *const fn ( + *anyopaque, + index_manager_mod.IndexManager.GraphMetricPlannedWorkerSweepOptions, + ) anyerror!index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + run_worker_pool: *const fn ( + *anyopaque, + WorkerPoolSweepOptions, + ) anyerror!index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + }; + + pub fn direct(index_manager: *index_manager_mod.IndexManager) MaintenanceBoundary { + return .{ + .ptr = index_manager, + .vtable = &direct_vtable, + }; + } + + pub fn init(ptr: *anyopaque, vtable: *const VTable) MaintenanceBoundary { + return .{ + .ptr = ptr, + .vtable = vtable, + }; + } + + pub fn runCombined( + self: MaintenanceBoundary, + options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try self.vtable.run_combined(self.ptr, options); + } + + pub fn runCoordinator( + self: MaintenanceBoundary, + options: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try self.vtable.run_coordinator(self.ptr, options); + } + + pub fn runWorker( + self: MaintenanceBoundary, + options: index_manager_mod.IndexManager.GraphMetricPlannedWorkerSweepOptions, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try self.vtable.run_worker(self.ptr, options); + } + + pub fn runWorkerPool( + self: MaintenanceBoundary, + options: WorkerPoolSweepOptions, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try self.vtable.run_worker_pool(self.ptr, options); + } +}; + +const direct_vtable = MaintenanceBoundary.VTable{ + .run_combined = directRunCombined, + .run_coordinator = directRunCoordinator, + .run_worker = directRunWorker, + .run_worker_pool = directRunWorkerPool, +}; + +fn directIndexManager(ptr: *anyopaque) *index_manager_mod.IndexManager { + return @ptrCast(@alignCast(ptr)); +} + +fn directRunCombined( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try directIndexManager(ptr).runGraphMetricPlannedMaintenance(options); +} + +fn directRunCoordinator( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try directIndexManager(ptr).runGraphMetricPlannedCoordinatorSweep(options); +} + +fn directRunWorker( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedWorkerSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try directIndexManager(ptr).runGraphMetricPlannedWorkerSweep(options); +} + +fn directRunWorkerPool( + ptr: *anyopaque, + options: MaintenanceBoundary.WorkerPoolSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (options.worker_ids.len == 0) { + return try directRunWorker(ptr, .{ + .worker_id = options.worker_id, + .max_pages = options.max_pages, + .now_ms = options.now_ms, + }); + } + + var total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var pages_remaining = options.max_pages; + while (pages_remaining > 0) { + var worker_progressed = false; + for (options.worker_ids) |worker_id| { + if (pages_remaining == 0) break; + const worker = try directRunWorker(ptr, .{ + .worker_id = worker_id, + .max_pages = 1, + .now_ms = options.now_ms, + }); + total.add(worker); + pages_remaining -= @min(pages_remaining, worker.worker_steps); + worker_progressed = worker_progressed or worker.durableProgressed(); + } + if (!worker_progressed) break; + } + return total; +} + +pub const default_lease_key = default_combined_lease_key; +pub const default_combined_lease_key = "\x00\x00__metadata__:graph_metric_runtime_lease:combined"; +pub const default_coordinator_lease_key = "\x00\x00__metadata__:graph_metric_runtime_lease:coordinator"; +pub const default_worker_lease_key = "\x00\x00__metadata__:graph_metric_runtime_lease:worker"; +pub const default_worker_pool_lease_key = "\x00\x00__metadata__:graph_metric_runtime_lease:worker_pool"; + +pub fn defaultLeaseKey(role: Role) []const u8 { + return switch (role) { + .combined => default_combined_lease_key, + .coordinator => default_coordinator_lease_key, + .worker => default_worker_lease_key, + .worker_pool => default_worker_pool_lease_key, + }; +} + +pub const GraphMetricRuntime = if (builtin.os.tag == .freestanding) struct { + config: Config, + + pub fn init( + _: Allocator, + _: anytype, + _: *index_manager_mod.IndexManager, + _: *apply_rw_lock_mod.ApplyRwLock, + _: *background_runtime_mod.BackendRuntime, + config: Config, + ) !@This() { + return .{ .config = config }; + } + + pub fn deinit(self: *@This()) void { + self.* = undefined; + } + + pub fn deinitPreserveLease(self: *@This()) void { + self.* = undefined; + } + + pub fn start(self: *@This()) !void { + if (self.config.enabled and self.config.start_background_loop) return error.UnsupportedPlatform; + } + + pub fn notify(self: *@This()) void { + _ = self; + } + + pub fn stats(self: *@This()) Stats { + return initialStats(self.config); + } + + pub fn runOnce(self: *@This()) !bool { + _ = self; + return false; + } + + pub fn runOnceDetailed(self: *@This()) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + _ = self; + return .{}; + } + + pub fn runCoordinatorOnce(self: *@This(), start_background_builds: bool) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + _ = self; + _ = start_background_builds; + return .{}; + } + + pub fn runWorkerOnce(self: *@This(), worker_id: []const u8) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + _ = self; + _ = worker_id; + return .{}; + } + + pub fn runWorkerPoolOnce(self: *@This()) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + _ = self; + return .{}; + } +} else struct { + alloc: Allocator, + io_impl: ?*Io.Threaded, + maintenance_boundary: MaintenanceBoundary, + config: Config, + lease_key: []u8, + ownership: ownership_mod.State, + mutex: Io.Mutex = .init, + wake_event: Io.Event = .unset, + shutdown: bool = false, + notified: bool = false, + future: ?Io.Future(void) = null, + stats_snapshot: Stats = .{}, + + pub fn init( + alloc: Allocator, + store: anytype, + index_manager: *index_manager_mod.IndexManager, + apply_mutex: *apply_rw_lock_mod.ApplyRwLock, + backend_runtime: *background_runtime_mod.BackendRuntime, + config: Config, + ) !GraphMetricRuntime { + const io_impl = backend_runtime.io_impl; + _ = apply_mutex; + if (config.enabled and io_impl == null) return error.MissingBackendRuntimeIo; + try validateConfig(config); + const lease_key = try runtimeLeaseKeyAlloc(alloc, config); + errdefer alloc.free(lease_key); + return .{ + .alloc = alloc, + .io_impl = io_impl, + .maintenance_boundary = MaintenanceBoundary.direct(index_manager), + .config = config, + .lease_key = lease_key, + .ownership = try ownership_mod.State.init(alloc, store, lease_key, .{ + .lease_owned = config.lease_owned, + .owner_id = if (config.owner_id.len != 0) config.owner_id else config.runtime_id, + .lease_ttl_ms = config.lease_ttl_ms, + }), + .stats_snapshot = initialStatsWithLeaseKey(config, lease_key), + }; + } + + fn stopRuntime(self: *GraphMetricRuntime) void { + if (self.io_impl) |io_impl| { + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + self.shutdown = true; + self.notified = true; + self.mutex.unlock(io); + self.wake_event.set(io); + + if (self.future) |*future| _ = future.await(io); + } + self.future = null; + } + + pub fn deinit(self: *GraphMetricRuntime) void { + self.stopRuntime(); + self.ownership.deinit(self.alloc); + self.alloc.free(self.lease_key); + self.* = undefined; + } + + pub fn deinitPreserveLease(self: *GraphMetricRuntime) void { + self.stopRuntime(); + self.ownership.deinitPreserveLease(self.alloc); + self.alloc.free(self.lease_key); + self.* = undefined; + } + + pub fn start(self: *GraphMetricRuntime) !void { + if (!self.config.enabled) return; + if (!self.config.start_background_loop) return; + const io_impl = self.io_impl orelse return error.MissingBackendRuntimeIo; + self.future = try io_impl.io().concurrent(workerMain, .{self}); + self.recordStarted(); + } + + pub fn notify(self: *GraphMetricRuntime) void { + if (!self.config.enabled) return; + const io_impl = self.io_impl orelse return; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + self.notified = true; + self.mutex.unlock(io); + self.wake_event.set(io); + } + + pub fn stats(self: *GraphMetricRuntime) Stats { + const io_impl = self.io_impl orelse return self.stats_snapshot; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + var snapshot = self.stats_snapshot; + snapshot.started = self.future != null; + snapshot.shutdown = self.shutdown; + snapshot.notified = self.notified; + applyOwnershipStats(&snapshot, self.ownership.stats()); + return snapshot; + } + + pub fn runOnce(self: *GraphMetricRuntime) !bool { + const result = try self.runOnceDetailed(); + return result.durableProgressed(); + } + + pub fn runOnceDetailed(self: *GraphMetricRuntime) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (!self.config.enabled) return .{}; + self.recordTickStarted(); + const now_ms = self.config.clock.nowRealtimeMs(); + if (!self.ensureRuntimeLease(now_ms)) { + self.recordTickSuccess(.{}); + return .{}; + } + + const result = runBoundaryTick(self.maintenance_boundary, self.config, now_ms) catch |err| { + self.recordTickError(err); + return err; + }; + self.recordTickSuccess(result); + return result; + } + + pub fn runCoordinatorOnce( + self: *GraphMetricRuntime, + start_background_builds: bool, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (!self.config.enabled) return .{}; + self.recordTickStarted(); + if (!coordinatorCallAllowed(self.config)) { + const err = error.InvalidGraphMetricRuntimeRole; + self.recordTickError(err); + return err; + } + const now_ms = self.config.clock.nowRealtimeMs(); + if (!self.ensureRuntimeLease(now_ms)) { + self.recordTickSuccess(.{}); + return .{}; + } + + const result = self.maintenance_boundary.runCoordinator(.{ + .max_metrics = self.config.planned_options.max_metrics_per_round, + .start_background_builds = start_background_builds, + .now_ms = now_ms, + }) catch |err| { + self.recordTickError(err); + return err; + }; + self.recordTickSuccess(result); + return result; + } + + pub fn runWorkerOnce( + self: *GraphMetricRuntime, + worker_id: []const u8, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (!self.config.enabled) return .{}; + self.recordTickStarted(); + if (!workerCallAllowed(self.config, worker_id)) { + const err = error.InvalidGraphMetricBuildWorker; + self.recordTickError(err); + return err; + } + const now_ms = self.config.clock.nowRealtimeMs(); + if (!self.ensureRuntimeLease(now_ms)) { + self.recordTickSuccess(.{}); + return .{}; + } + + const result = self.maintenance_boundary.runWorker(.{ + .worker_id = worker_id, + .max_pages = self.config.planned_options.max_pages_per_round, + .now_ms = now_ms, + }) catch |err| { + self.recordTickError(err); + return err; + }; + self.recordTickSuccess(result); + return result; + } + + pub fn runWorkerPoolOnce(self: *GraphMetricRuntime) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + if (!workerPoolCallAllowed(self.config)) { + if (!self.config.enabled) return .{}; + self.recordTickStarted(); + const err = error.InvalidGraphMetricRuntimeRole; + self.recordTickError(err); + return err; + } + if (self.config.planned_options.worker_ids.len == 0) { + return try self.runWorkerOnce(self.config.planned_options.worker_id); + } + if (!self.config.enabled) return .{}; + self.recordTickStarted(); + const now_ms = self.config.clock.nowRealtimeMs(); + if (!self.ensureRuntimeLease(now_ms)) { + self.recordTickSuccess(.{}); + return .{}; + } + + const total = self.runWorkerPoolSweepLockedAt(now_ms) catch |err| { + self.recordTickError(err); + return err; + }; + self.recordTickSuccess(total); + return total; + } + + fn runWorkerPoolSweepLocked(self: *GraphMetricRuntime) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + return try self.runWorkerPoolSweepLockedAt(self.config.clock.nowRealtimeMs()); + } + + fn runWorkerPoolSweepLockedAt( + self: *GraphMetricRuntime, + now_ms: u64, + ) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + const worker_ids = self.config.planned_options.worker_ids; + const page_budget = self.config.planned_options.max_pages_per_round; + if (worker_ids.len > 1 and page_budget > 1) { + const io = (self.io_impl orelse return error.MissingBackendRuntimeIo).io(); + const width = @min(worker_ids.len, page_budget); + var results: [max_runtime_workers]index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult = @splat(.{}); + var failures: [max_runtime_workers]?anyerror = @splat(null); + const Worker = struct { + fn run( + boundary: MaintenanceBoundary, + worker_id: []const u8, + max_pages: usize, + tick_ms: u64, + result: *index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + failure: *?anyerror, + ) void { + result.* = boundary.runWorker(.{ + .worker_id = worker_id, + .max_pages = max_pages, + .now_ms = tick_ms, + }) catch |err| { + failure.* = err; + return; + }; + } + }; + var group: Io.Group = .init; + const pages_per_worker = page_budget / width; + const extra_pages = page_budget % width; + for (worker_ids[0..width], 0..) |worker_id, index| { + group.async(io, Worker.run, .{ + self.maintenance_boundary, + worker_id, + pages_per_worker + @intFromBool(index < extra_pages), + now_ms, + &results[index], + &failures[index], + }); + } + try group.await(io); + var total: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult = .{}; + for (0..width) |index| { + if (failures[index]) |err| return err; + total.add(results[index]); + } + return total; + } + return try self.maintenance_boundary.runWorkerPool(.{ + .worker_id = self.config.planned_options.worker_id, + .worker_ids = self.config.planned_options.worker_ids, + .max_pages = self.config.planned_options.max_pages_per_round, + .now_ms = now_ms, + }); + } + + fn ensureRuntimeLease(self: *GraphMetricRuntime, now_ms: u64) bool { + const io_impl = self.io_impl orelse return false; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + return self.ownership.ensureLease(now_ms) catch { + self.ownership.noteAcquireFailure(); + return false; + }; + } + + fn recordStarted(self: *GraphMetricRuntime) void { + const io_impl = self.io_impl orelse { + self.stats_snapshot.started = true; + return; + }; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + self.stats_snapshot.started = true; + self.mutex.unlock(io); + } + + fn recordTickStarted(self: *GraphMetricRuntime) void { + const io_impl = self.io_impl orelse { + self.stats_snapshot.ticks_started += 1; + return; + }; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + self.stats_snapshot.ticks_started += 1; + self.mutex.unlock(io); + } + + fn recordTickSuccess( + self: *GraphMetricRuntime, + result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, + ) void { + const io_impl = self.io_impl orelse { + updateSuccessStats(&self.stats_snapshot, result); + return; + }; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + updateSuccessStats(&self.stats_snapshot, result); + self.mutex.unlock(io); + } + + fn recordTickError(self: *GraphMetricRuntime, err: anyerror) void { + const io_impl = self.io_impl orelse { + updateErrorStats(&self.stats_snapshot, err); + return; + }; + const io = io_impl.io(); + self.mutex.lockUncancelable(io); + updateErrorStats(&self.stats_snapshot, err); + self.mutex.unlock(io); + } +}; + +fn updateSuccessStats( + stats_snapshot: *Stats, + result: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult, +) void { + stats_snapshot.ticks_completed += 1; + if (result.durableProgressed()) { + stats_snapshot.durable_progress_ticks += 1; + } else { + stats_snapshot.idle_ticks += 1; + } + stats_snapshot.last_error_name = null; + stats_snapshot.total_result.add(result); + stats_snapshot.last_result = result; +} + +test "graph metric runtime retirement remains durable through aggregation and idle accounting" { + const Sweep = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult; + var aggregate = Sweep{}; + aggregate.add(.{ .coordinator_steps = 1, .retired_input_records = 512 }); + try std.testing.expect(aggregate.durableProgressed()); + try std.testing.expect(aggregate.progressed()); + var snapshot = initialStats(.{}); + updateSuccessStats(&snapshot, aggregate); + try std.testing.expectEqual(@as(u64, 1), snapshot.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 0), snapshot.idle_ticks); + try std.testing.expectEqual(@as(usize, 512), snapshot.total_result.retired_input_records); + try std.testing.expectEqual(@as(usize, 512), snapshot.last_result.retired_input_records); + updateSuccessStats(&snapshot, .{ .coordinator_steps = 1 }); + try std.testing.expectEqual(@as(u64, 1), snapshot.idle_ticks); +} + +fn updateErrorStats(stats_snapshot: *Stats, err: anyerror) void { + stats_snapshot.error_ticks += 1; + stats_snapshot.last_error_name = @errorName(err); +} + +pub fn initialStats(config: Config) Stats { + return .{ + .enabled = config.enabled, + .role = config.role, + .runtime_id_hash = identityHash(config.runtime_id), + .owner_id_hash = identityHash(runtimeOwnerId(config)), + .worker_id_hash = workerIdentityHash(config), + .worker_count = configuredWorkerCount(config), + .lease_owned = config.lease_owned, + .has_lease = !config.lease_owned, + }; +} + +pub fn initialStatsWithLeaseKey(config: Config, lease_key: []const u8) Stats { + var stats = initialStats(config); + stats.lease_key_hash = identityHash(lease_key); + return stats; +} + +fn applyOwnershipStats(stats_snapshot: *Stats, ownership_stats: ownership_mod.Stats) void { + stats_snapshot.lease_owned = ownership_stats.lease_owned; + stats_snapshot.has_lease = ownership_stats.has_lease; + stats_snapshot.acquisition_count = ownership_stats.acquisition_count; + stats_snapshot.takeover_count = ownership_stats.takeover_count; + stats_snapshot.lease_acquire_failures = ownership_stats.lease_acquire_failures; + stats_snapshot.lost_leases = ownership_stats.lost_leases; + stats_snapshot.last_acquired_ms = ownership_stats.last_acquired_ms; + stats_snapshot.lease_expires_at_ms = ownership_stats.lease_expires_at_ms; + stats_snapshot.lease_renew_after_ms = ownership_stats.lease_renew_after_ms; + stats_snapshot.renewal_count = ownership_stats.renewal_count; +} + +pub fn identityHash(value: []const u8) u64 { + if (value.len == 0) return 0; + return std.hash.Wyhash.hash(0, value); +} + +pub fn runtimeOwnerId(config: Config) []const u8 { + return if (config.owner_id.len != 0) config.owner_id else config.runtime_id; +} + +pub fn workerSetIdentityHash(worker_ids: []const []const u8) u64 { + if (worker_ids.len == 0) return 0; + var xor_hash: u64 = 0; + var sum_hash: u64 = 0; + for (worker_ids) |worker_id| { + const item_hash = identityHash(worker_id); + xor_hash ^= item_hash; + sum_hash +%= item_hash; + } + const fingerprint_words = [_]u64{ + @intCast(worker_ids.len), + xor_hash, + sum_hash, + }; + return std.hash.Wyhash.hash(0, std.mem.asBytes(&fingerprint_words)); +} + +pub fn workerIdentityHash(config: Config) u64 { + if (config.role == .coordinator) return 0; + if (config.planned_options.worker_ids.len == 0) return identityHash(config.planned_options.worker_id); + return workerSetIdentityHash(config.planned_options.worker_ids); +} + +pub fn runtimeLeaseKeyAlloc(alloc: Allocator, config: Config) ![]u8 { + const base_key = defaultLeaseKey(config.role); + switch (config.role) { + .combined, .coordinator => return try alloc.dupe(u8, base_key), + .worker, .worker_pool => { + if (configuredWorkerCount(config) == 0) return try alloc.dupe(u8, base_key); + return try std.fmt.allocPrint(alloc, "{s}:{x}", .{ + base_key, + workerIdentityHash(config), + }); + }, + } +} + +pub fn runBoundaryTick( + boundary: MaintenanceBoundary, + config: Config, + now_ms: u64, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + var planned_options = config.planned_options; + planned_options.now_ms = now_ms; + return switch (config.role) { + .combined => boundary.runCombined(planned_options), + .coordinator => boundary.runCoordinator(.{ + .max_metrics = config.planned_options.max_metrics_per_round, + .start_background_builds = config.coordinator_start_background_builds, + .now_ms = now_ms, + }), + .worker => boundary.runWorker(.{ + .worker_id = config.planned_options.worker_id, + .max_pages = config.planned_options.max_pages_per_round, + .now_ms = now_ms, + }), + .worker_pool => boundary.runWorkerPool(.{ + .worker_id = config.planned_options.worker_id, + .worker_ids = config.planned_options.worker_ids, + .max_pages = config.planned_options.max_pages_per_round, + .now_ms = now_ms, + }), + }; +} + +fn validateConfig(config: Config) !void { + if (!config.enabled) return; + if (config.lease_ttl_ms == 0) return error.InvalidGraphMetricRuntimeConfig; + if (config.lease_owned and + (config.owner_id.len == 0 or std.mem.eql(u8, config.owner_id, "local"))) + { + return error.InvalidGraphMetricRuntimeConfig; + } + if (config.planned_options.max_rounds == 0) return error.InvalidGraphMetricRuntimeConfig; + if (config.planned_options.max_metrics_per_round == 0) return error.InvalidGraphMetricRuntimeConfig; + if (config.planned_options.max_pages_per_round == 0) return error.InvalidGraphMetricRuntimeConfig; + try validateWorkerIdentities(config); +} + +fn validateWorkerIdentities(config: Config) !void { + if (config.role == .coordinator) { + if (config.planned_options.worker_ids.len != 0) return error.InvalidGraphMetricBuildWorker; + return; + } + if (config.role == .worker and config.planned_options.worker_ids.len != 0) { + return error.InvalidGraphMetricBuildWorker; + } + if (config.planned_options.worker_ids.len == 0) { + if ((config.role == .worker or config.role == .worker_pool) and + config.planned_options.worker_id.len == 0) + { + return error.InvalidGraphMetricBuildWorker; + } + return; + } + if (config.planned_options.worker_ids.len > max_runtime_workers) + return error.InvalidGraphMetricRuntimeConfig; + + for (config.planned_options.worker_ids, 0..) |worker_id, i| { + if (worker_id.len == 0) return error.InvalidGraphMetricBuildWorker; + for (config.planned_options.worker_ids[0..i]) |prior_worker_id| { + if (std.mem.eql(u8, worker_id, prior_worker_id)) return error.InvalidGraphMetricBuildWorker; + } + } +} + +fn workerCallAllowed(config: Config, worker_id: []const u8) bool { + if (worker_id.len == 0) return false; + return switch (config.role) { + .combined => true, + .coordinator => false, + .worker => std.mem.eql(u8, worker_id, config.planned_options.worker_id), + .worker_pool => { + if (config.planned_options.worker_ids.len == 0) { + return std.mem.eql(u8, worker_id, config.planned_options.worker_id); + } + for (config.planned_options.worker_ids) |configured_worker_id| { + if (std.mem.eql(u8, worker_id, configured_worker_id)) return true; + } + return false; + }, + }; +} + +fn coordinatorCallAllowed(config: Config) bool { + return config.role == .combined or config.role == .coordinator; +} + +fn workerPoolCallAllowed(config: Config) bool { + return config.role == .combined or config.role == .worker or config.role == .worker_pool; +} + +fn configuredWorkerCount(config: Config) usize { + if (config.role == .coordinator) return 0; + if (config.planned_options.worker_ids.len != 0) return config.planned_options.worker_ids.len; + return if (config.planned_options.worker_id.len == 0) 0 else 1; +} + +test "graph metric runtime config rejects zero lease and maintenance budgets when enabled" { + try std.testing.expectError(error.InvalidGraphMetricRuntimeConfig, validateConfig(.{ + .enabled = true, + .lease_ttl_ms = 0, + })); + try std.testing.expectError(error.InvalidGraphMetricRuntimeConfig, validateConfig(.{ + .enabled = true, + .planned_options = .{ .max_rounds = 0 }, + })); + try std.testing.expectError(error.InvalidGraphMetricRuntimeConfig, validateConfig(.{ + .enabled = true, + .planned_options = .{ .max_metrics_per_round = 0 }, + })); + try std.testing.expectError(error.InvalidGraphMetricRuntimeConfig, validateConfig(.{ + .enabled = true, + .planned_options = .{ .max_pages_per_round = 0 }, + })); + try validateConfig(.{ + .enabled = false, + .lease_ttl_ms = 0, + .planned_options = .{ + .max_rounds = 0, + .max_metrics_per_round = 0, + .max_pages_per_round = 0, + }, + }); +} + +test "graph metric runtime requires an explicit incarnation owner for leased operation" { + try std.testing.expectError(error.InvalidGraphMetricRuntimeConfig, validateConfig(.{ + .enabled = true, + .lease_owned = true, + })); + try validateConfig(.{ + .enabled = true, + .lease_owned = true, + .owner_id = "runtime-a:pid-42:start-100", + }); +} + +test "graph metric runtime config rejects worker id lists for single-owner roles" { + const workers = [_][]const u8{ "worker-a", "worker-b" }; + + try validateConfig(.{ + .enabled = true, + .role = .coordinator, + .planned_options = .{ .worker_id = "coordinator-unused" }, + }); + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, validateConfig(.{ + .enabled = true, + .role = .coordinator, + .planned_options = .{ .worker_ids = workers[0..] }, + })); + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, validateConfig(.{ + .enabled = true, + .role = .worker, + .planned_options = .{ + .worker_id = "worker-a", + .worker_ids = workers[0..], + }, + })); +} + +test "graph metric runtime role gates apply without durable lease ownership" { + const combined = Config{ + .enabled = true, + .role = .combined, + .lease_owned = false, + .planned_options = .{ .worker_id = "combined-worker" }, + }; + try std.testing.expect(coordinatorCallAllowed(combined)); + try std.testing.expect(workerCallAllowed(combined, "any-worker")); + try std.testing.expect(workerPoolCallAllowed(combined)); + + const coordinator = Config{ + .enabled = true, + .role = .coordinator, + .lease_owned = false, + .planned_options = .{ .worker_id = "coordinator-unused" }, + }; + try std.testing.expect(coordinatorCallAllowed(coordinator)); + try std.testing.expect(!workerCallAllowed(coordinator, "coordinator-unused")); + try std.testing.expect(!workerPoolCallAllowed(coordinator)); + + const worker = Config{ + .enabled = true, + .role = .worker, + .lease_owned = false, + .planned_options = .{ .worker_id = "worker-a" }, + }; + try std.testing.expect(!coordinatorCallAllowed(worker)); + try std.testing.expect(workerCallAllowed(worker, "worker-a")); + try std.testing.expect(!workerCallAllowed(worker, "worker-b")); + try std.testing.expect(workerPoolCallAllowed(worker)); + + const pool_workers = [_][]const u8{ "pool-a", "pool-b" }; + const worker_pool = Config{ + .enabled = true, + .role = .worker_pool, + .lease_owned = false, + .planned_options = .{ .worker_ids = pool_workers[0..] }, + }; + try std.testing.expect(!coordinatorCallAllowed(worker_pool)); + try std.testing.expect(workerCallAllowed(worker_pool, "pool-a")); + try std.testing.expect(workerCallAllowed(worker_pool, "pool-b")); + try std.testing.expect(!workerCallAllowed(worker_pool, "pool-c")); + try std.testing.expect(workerPoolCallAllowed(worker_pool)); +} + +test "graph metric runtime worker pool identity is order independent" { + const alloc = std.testing.allocator; + const workers_ab = [_][]const u8{ "worker-a", "worker-b" }; + const workers_ba = [_][]const u8{ "worker-b", "worker-a" }; + const workers_ac = [_][]const u8{ "worker-a", "worker-c" }; + + const config_ab = Config{ + .enabled = true, + .role = .worker_pool, + .lease_owned = true, + .planned_options = .{ .worker_ids = workers_ab[0..] }, + }; + const config_ba = Config{ + .enabled = true, + .role = .worker_pool, + .lease_owned = true, + .planned_options = .{ .worker_ids = workers_ba[0..] }, + }; + const config_ac = Config{ + .enabled = true, + .role = .worker_pool, + .lease_owned = true, + .planned_options = .{ .worker_ids = workers_ac[0..] }, + }; + + try std.testing.expectEqual(@as(usize, 2), configuredWorkerCount(config_ab)); + try std.testing.expectEqual(workerIdentityHash(config_ab), workerIdentityHash(config_ba)); + try std.testing.expect(workerIdentityHash(config_ab) != workerIdentityHash(config_ac)); + + const lease_ab = try runtimeLeaseKeyAlloc(alloc, config_ab); + defer alloc.free(lease_ab); + const lease_ba = try runtimeLeaseKeyAlloc(alloc, config_ba); + defer alloc.free(lease_ba); + const lease_ac = try runtimeLeaseKeyAlloc(alloc, config_ac); + defer alloc.free(lease_ac); + + try std.testing.expectEqualStrings(lease_ab, lease_ba); + try std.testing.expect(!std.mem.eql(u8, lease_ab, lease_ac)); +} + +const FakeMaintenanceBoundary = struct { + combined_calls: usize = 0, + coordinator_calls: usize = 0, + worker_calls: usize = 0, + worker_pool_calls: usize = 0, + last_worker_id: []const u8 = "", + last_worker_count: usize = 0, + last_max_pages: usize = 0, + last_now_ms: ?u64 = null, + + fn boundary(self: *FakeMaintenanceBoundary) MaintenanceBoundary { + return MaintenanceBoundary.init(self, &fake_boundary_vtable); + } +}; + +const fake_boundary_vtable = MaintenanceBoundary.VTable{ + .run_combined = fakeBoundaryRunCombined, + .run_coordinator = fakeBoundaryRunCoordinator, + .run_worker = fakeBoundaryRunWorker, + .run_worker_pool = fakeBoundaryRunWorkerPool, +}; + +fn fakeBoundaryContext(ptr: *anyopaque) *FakeMaintenanceBoundary { + return @ptrCast(@alignCast(ptr)); +} + +fn fakeBoundaryRunCombined( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedMaintenanceOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + const fake = fakeBoundaryContext(ptr); + fake.combined_calls += 1; + fake.last_worker_id = options.worker_id; + fake.last_worker_count = options.worker_ids.len; + fake.last_max_pages = options.max_pages_per_round; + fake.last_now_ms = options.now_ms; + return .{ .metrics_scanned = 1 }; +} + +fn fakeBoundaryRunCoordinator( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + const fake = fakeBoundaryContext(ptr); + fake.coordinator_calls += 1; + fake.last_max_pages = options.max_metrics; + fake.last_now_ms = options.now_ms; + return .{ .coordinator_steps = 1 }; +} + +fn fakeBoundaryRunWorker( + ptr: *anyopaque, + options: index_manager_mod.IndexManager.GraphMetricPlannedWorkerSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + const fake = fakeBoundaryContext(ptr); + fake.worker_calls += 1; + fake.last_worker_id = options.worker_id; + fake.last_max_pages = options.max_pages; + fake.last_now_ms = options.now_ms; + return .{ .worker_steps = 1 }; +} + +fn fakeBoundaryRunWorkerPool( + ptr: *anyopaque, + options: MaintenanceBoundary.WorkerPoolSweepOptions, +) !index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult { + const fake = fakeBoundaryContext(ptr); + fake.worker_pool_calls += 1; + fake.last_worker_id = options.worker_id; + fake.last_worker_count = options.worker_ids.len; + fake.last_max_pages = options.max_pages; + fake.last_now_ms = options.now_ms; + return .{ .worker_steps = options.worker_ids.len, .pages_completed = options.worker_ids.len }; +} + +test "graph metric runtime boundary tick preserves worker pool operation" { + const workers = [_][]const u8{ "worker-a", "worker-b" }; + var fake = FakeMaintenanceBoundary{}; + const result = try runBoundaryTick(fake.boundary(), .{ + .enabled = true, + .role = .worker_pool, + .planned_options = .{ + .worker_id = "unused-worker", + .worker_ids = workers[0..], + .max_pages_per_round = 5, + }, + }, 1234); + + try std.testing.expectEqual(@as(usize, 0), fake.worker_calls); + try std.testing.expectEqual(@as(usize, 1), fake.worker_pool_calls); + try std.testing.expectEqualStrings("unused-worker", fake.last_worker_id); + try std.testing.expectEqual(@as(usize, 2), fake.last_worker_count); + try std.testing.expectEqual(@as(usize, 5), fake.last_max_pages); + try std.testing.expectEqual(@as(?u64, 1234), fake.last_now_ms); + try std.testing.expectEqual(@as(usize, 2), result.worker_steps); + try std.testing.expectEqual(@as(usize, 2), result.pages_completed); +} + +fn waitForGraphMetricFresh( + alloc: Allocator, + db: anytype, + index_name: []const u8, + metric_name: []const u8, + target_generation: u64, + max_attempts: usize, +) !bool { + for (0..max_attempts) |attempt| { + yieldToBackground(db); + if (attempt % 8 != 0 and attempt + 1 != max_attempts) continue; + const graph_entry = db.core.graphIndex(index_name) orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) return true; + } + return false; +} + +fn waitForGraphMetricPairFresh( + alloc: Allocator, + db: anytype, + index_name: []const u8, + first_metric_name: []const u8, + second_metric_name: []const u8, + target_generation: u64, + max_attempts: usize, +) !bool { + for (0..max_attempts) |attempt| { + yieldToBackground(db); + if (attempt % 8 != 0 and attempt + 1 != max_attempts) continue; + const graph_entry = db.core.graphIndex(index_name) orelse return error.IndexNotFound; + var first_status = try graph_entry.index.graphMetricStatus(first_metric_name); + defer first_status.deinit(alloc); + var second_status = try graph_entry.index.graphMetricStatus(second_metric_name); + defer second_status.deinit(alloc); + if (first_status.state == .fresh and + second_status.state == .fresh and + first_status.published_generation == target_generation and + second_status.published_generation == target_generation) + { + return true; + } + } + return false; +} + +fn workerMain(runtime: *GraphMetricRuntime) void { + while (true) { + if (isShutdown(runtime)) return; + const ran = runtime.runOnce() catch |err| { + if (builtin.os.tag != .freestanding) { + std.log.warn("graph metric maintenance worker failed: {s}", .{@errorName(err)}); + } + sleepMs(runtime, runtime.config.error_interval_ms); + continue; + }; + if (ran) { + sleepMs(runtime, runtime.config.active_interval_ms); + continue; + } + waitForWork(runtime); + } +} + +fn waitForWork(runtime: *GraphMetricRuntime) void { + var remaining_ms = runtime.config.idle_interval_ms; + if (remaining_ms == 0) remaining_ms = 1; + + const io_impl = runtime.io_impl orelse return; + const io = io_impl.io(); + runtime.mutex.lockUncancelable(io); + if (runtime.notified or runtime.shutdown) { + runtime.notified = false; + runtime.wake_event.reset(); + runtime.mutex.unlock(io); + return; + } + runtime.wake_event.reset(); + runtime.mutex.unlock(io); + + if (runtime.config.clock.isReal()) { + runtime.wake_event.waitTimeout(io, .{ .duration = .{ + .raw = std.Io.Duration.fromMilliseconds(@intCast(remaining_ms)), + .clock = .awake, + } }) catch |err| switch (err) { + error.Timeout, error.Canceled => {}, + }; + runtime.mutex.lockUncancelable(io); + runtime.notified = false; + runtime.mutex.unlock(io); + return; + } + + while (remaining_ms > 0) { + if (isShutdown(runtime)) return; + const slice_ms: u64 = @min(remaining_ms, 10); + runtimeSleepSlice(runtime, slice_ms); + remaining_ms -= slice_ms; + runtime.mutex.lockUncancelable(io); + const notified = runtime.notified; + runtime.notified = false; + runtime.mutex.unlock(io); + if (notified) return; + } +} + +fn sleepMs(runtime: *GraphMetricRuntime, ms: u64) void { + var remaining_ms = if (ms == 0) 1 else ms; + while (remaining_ms > 0) { + if (isShutdown(runtime)) return; + const slice_ms: u64 = @min(remaining_ms, 10); + runtimeSleepSlice(runtime, slice_ms); + remaining_ms -= slice_ms; + } +} + +fn runtimeSleepSlice(runtime: *GraphMetricRuntime, ms: u64) void { + if (!runtime.config.clock.isReal()) { + runtime.config.clock.sleepMs(ms); + return; + } + const io_impl = runtime.io_impl orelse { + runtime.config.clock.sleepMs(ms); + return; + }; + std.Io.Clock.Duration.sleep(.{ + .clock = .awake, + .raw = .fromMilliseconds(@intCast(if (ms == 0) @as(u64, 1) else ms)), + }, io_impl.io()) catch {}; +} + +fn isShutdown(runtime: *GraphMetricRuntime) bool { + const io_impl = runtime.io_impl orelse return runtime.shutdown; + const io = io_impl.io(); + runtime.mutex.lockUncancelable(io); + defer runtime.mutex.unlock(io); + return runtime.shutdown; +} + +test "db graph metric runtime lease ownership blocks duplicate owners and allows takeover" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(1_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var owner_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-owned-a", + .lease_owned = true, + .owner_id = "runtime-owner-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-owned-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer owner_a.deinit(); + var owner_b = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-owned-b", + .lease_owned = true, + .owner_id = "runtime-owner-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-owned-worker-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer owner_b.deinit(); + + const owner_a_tick = try owner_a.runOnceDetailed(); + try std.testing.expect(owner_a_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), owner_a_tick.builds_started); + { + const stats = owner_a.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-owner-a"), stats.owner_id_hash); + try std.testing.expectEqual(owner_a.stats().lease_key_hash, owner_b.stats().lease_key_hash); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 0), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1_000), stats.last_acquired_ms); + } + + const blocked_tick = try owner_b.runOnceDetailed(); + try std.testing.expect(!blocked_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), blocked_tick.builds_started); + try std.testing.expectEqual(@as(usize, 0), blocked_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), blocked_tick.coordinator_steps); + { + const stats = owner_b.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-owner-b"), stats.owner_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 1), stats.ticks_started); + try std.testing.expectEqual(@as(u64, 1), stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), stats.idle_ticks); + } + + manual_clock.advanceMs(101); + const takeover_tick = try owner_b.runOnceDetailed(); + try std.testing.expect(takeover_tick.durableProgressed()); + { + const stats = owner_b.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 1_101), stats.last_acquired_ms); + } + + const lost_tick = try owner_a.runOnceDetailed(); + try std.testing.expect(!lost_tick.durableProgressed()); + { + const stats = owner_a.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + db.graph_metric_runtime = &owner_b; + defer db.graph_metric_runtime = null; + { + const mapped_stats = try db.stats(alloc); + defer types.freeDBStats(alloc, mapped_stats); + try std.testing.expect(mapped_stats.graph_metric_runtime.lease_owned); + try std.testing.expect(mapped_stats.graph_metric_runtime.has_lease); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-owner-b"), mapped_stats.graph_metric_runtime.owner_id_hash); + try std.testing.expectEqual(owner_b.stats().lease_key_hash, mapped_stats.graph_metric_runtime.lease_key_hash); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.takeover_count); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 1_101), mapped_stats.graph_metric_runtime.last_acquired_ms); + } +} + +test "db graph metric runtime lease releases durable owner lease on deinit" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(5_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var owner_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-release-a", + .lease_owned = true, + .owner_id = "runtime-release-owner-a", + .lease_ttl_ms = 30_000, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-release-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + var owner_a_active = true; + errdefer if (owner_a_active) owner_a.deinit(); + + const owner_a_tick = try owner_a.runOnceDetailed(); + try std.testing.expect(!owner_a_tick.durableProgressed()); + { + const stats = owner_a.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + } + const owner_a_lease_key_hash = owner_a.stats().lease_key_hash; + { + var lease = try lease_mod.Lease.init(alloc, resources.store, default_combined_lease_key); + defer lease.deinit(); + var record = (try lease.load(alloc)) orelse return error.TestExpectedGraphMetricRuntimeLease; + defer lease_mod.deinitRecord(alloc, &record); + try std.testing.expectEqualStrings("runtime-release-owner-a", record.owner_id); + try std.testing.expectEqual(@as(u64, 35_000), record.expires_at_ms); + } + + owner_a.deinit(); + owner_a_active = false; + { + var lease = try lease_mod.Lease.init(alloc, resources.store, default_combined_lease_key); + defer lease.deinit(); + try std.testing.expect((try lease.load(alloc)) == null); + } + + var owner_b = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-release-b", + .lease_owned = true, + .owner_id = "runtime-release-owner-b", + .lease_ttl_ms = 30_000, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-release-worker-b", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer owner_b.deinit(); + + const owner_b_tick = try owner_b.runOnceDetailed(); + try std.testing.expect(!owner_b_tick.durableProgressed()); + { + const stats = owner_b.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(owner_a_lease_key_hash, stats.lease_key_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-release-owner-b"), stats.owner_id_hash); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 5_000), stats.last_acquired_ms); + } +} + +test "db graph metric runtime lease stale deinit preserves replacement owner lease" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(6_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var owner_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-stale-release-a", + .lease_owned = true, + .owner_id = "runtime-stale-release-owner-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-stale-release-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + var owner_a_active = true; + errdefer if (owner_a_active) owner_a.deinit(); + + const owner_a_tick = try owner_a.runOnceDetailed(); + try std.testing.expect(!owner_a_tick.durableProgressed()); + { + const stats = owner_a.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 6_000), stats.last_acquired_ms); + } + const lease_key_hash = owner_a.stats().lease_key_hash; + { + var lease = try lease_mod.Lease.init(alloc, resources.store, default_combined_lease_key); + defer lease.deinit(); + var record = (try lease.load(alloc)) orelse return error.TestExpectedGraphMetricRuntimeLease; + defer lease_mod.deinitRecord(alloc, &record); + try std.testing.expectEqualStrings("runtime-stale-release-owner-a", record.owner_id); + try std.testing.expectEqual(@as(u64, 6_100), record.expires_at_ms); + } + + manual_clock.advanceMs(101); + var owner_b = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-stale-release-b", + .lease_owned = true, + .owner_id = "runtime-stale-release-owner-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-stale-release-worker-b", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer owner_b.deinit(); + + const owner_b_tick = try owner_b.runOnceDetailed(); + try std.testing.expect(!owner_b_tick.durableProgressed()); + { + const stats = owner_b.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(lease_key_hash, stats.lease_key_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-stale-release-owner-b"), stats.owner_id_hash); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 6_101), stats.last_acquired_ms); + } + { + var lease = try lease_mod.Lease.init(alloc, resources.store, default_combined_lease_key); + defer lease.deinit(); + var record = (try lease.load(alloc)) orelse return error.TestExpectedGraphMetricRuntimeLease; + defer lease_mod.deinitRecord(alloc, &record); + try std.testing.expectEqualStrings("runtime-stale-release-owner-b", record.owner_id); + try std.testing.expectEqual(@as(u64, 6_201), record.expires_at_ms); + } + + owner_a.deinit(); + owner_a_active = false; + { + var lease = try lease_mod.Lease.init(alloc, resources.store, default_combined_lease_key); + defer lease.deinit(); + var record = (try lease.load(alloc)) orelse return error.TestExpectedGraphMetricRuntimeLease; + defer lease_mod.deinitRecord(alloc, &record); + try std.testing.expectEqualStrings("runtime-stale-release-owner-b", record.owner_id); + try std.testing.expectEqual(@as(u64, 6_201), record.expires_at_ms); + } + + var owner_c = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-stale-release-c", + .lease_owned = true, + .owner_id = "runtime-stale-release-owner-c", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-stale-release-worker-c", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer owner_c.deinit(); + + const owner_c_tick = try owner_c.runOnceDetailed(); + try std.testing.expect(!owner_c_tick.durableProgressed()); + { + const stats = owner_c.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(lease_key_hash, stats.lease_key_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-stale-release-owner-c"), stats.owner_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const owner_b_renew_tick = try owner_b.runOnceDetailed(); + try std.testing.expect(!owner_b_renew_tick.durableProgressed()); + { + const stats = owner_b.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 0), stats.lost_leases); + } +} + +test "db graph metric runtime role leases allow split owners and block duplicate coordinators" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(2_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var coordinator = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-role-lease-coordinator-a", + .lease_owned = true, + .owner_id = "role-lease-coordinator-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-role-lease-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator.deinit(); + var duplicate_coordinator = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-role-lease-coordinator-b", + .lease_owned = true, + .owner_id = "role-lease-coordinator-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-role-lease-coordinator-b-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_coordinator.deinit(); + var worker = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-role-lease-worker", + .lease_owned = true, + .owner_id = "role-lease-worker", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-role-lease-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker.deinit(); + var duplicate_worker = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-role-lease-worker-duplicate", + .lease_owned = true, + .owner_id = "role-lease-worker-duplicate", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-role-lease-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_worker.deinit(); + + const early_worker = try worker.runOnceDetailed(); + try std.testing.expect(!early_worker.durableProgressed()); + { + const stats = worker.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + } + + const coordinator_start = try coordinator.runOnceDetailed(); + try std.testing.expect(coordinator_start.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), coordinator_start.builds_started); + { + const stats = coordinator.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + } + + const duplicate_blocked = try duplicate_coordinator.runOnceDetailed(); + try std.testing.expect(!duplicate_blocked.durableProgressed()); + { + const stats = duplicate_coordinator.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const duplicate_worker_blocked = try duplicate_worker.runOnceDetailed(); + try std.testing.expect(!duplicate_worker_blocked.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), duplicate_worker_blocked.worker_steps); + try std.testing.expectEqual(@as(usize, 0), duplicate_worker_blocked.pages_completed); + { + const stats = duplicate_worker.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(worker.stats().lease_key_hash, stats.lease_key_hash); + try std.testing.expectEqual(worker.stats().worker_id_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const worker_prepare = try worker.runOnceDetailed(); + try std.testing.expect(worker_prepare.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.worker_steps); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.pages_completed); + + manual_clock.advanceMs(101); + const duplicate_takeover = try duplicate_coordinator.runOnceDetailed(); + try std.testing.expect(duplicate_takeover.durableProgressed()); + { + const stats = duplicate_coordinator.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 2_101), stats.last_acquired_ms); + } + + const coordinator_lost = try coordinator.runOnceDetailed(); + try std.testing.expect(!coordinator_lost.durableProgressed()); + { + const stats = coordinator.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const worker_after_coordinator_takeover = try worker.runOnceDetailed(); + try std.testing.expect(worker_after_coordinator_takeover.durableProgressed()); + try std.testing.expect(worker_after_coordinator_takeover.worker_steps > 0); + { + const stats = worker.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lost_leases); + } +} + +test "db graph metric runtime role worker leases are scoped by worker identity" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(3_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var worker_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-worker-lease-a", + .lease_owned = true, + .owner_id = "runtime-worker-owner-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_a.deinit(); + var worker_b = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-worker-lease-b", + .lease_owned = true, + .owner_id = "runtime-worker-owner-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-worker-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_b.deinit(); + var duplicate_worker_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-worker-lease-a-duplicate", + .lease_owned = true, + .owner_id = "runtime-worker-owner-a-duplicate", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_worker_a.deinit(); + + const idle_a = try worker_a.runOnceDetailed(); + try std.testing.expect(!idle_a.durableProgressed()); + { + const stats = worker_a.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(usize, 1), stats.worker_count); + try std.testing.expect(stats.worker_id_hash != 0); + try std.testing.expect(stats.lease_key_hash != 0); + try std.testing.expect(stats.lease_key_hash != worker_b.stats().lease_key_hash); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_worker_a.stats().lease_key_hash); + } + + const idle_b = try worker_b.runOnceDetailed(); + try std.testing.expect(!idle_b.durableProgressed()); + { + const stats = worker_b.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expectEqual(@as(usize, 1), stats.worker_count); + try std.testing.expect(stats.worker_id_hash != 0); + } + + const duplicate_blocked = try duplicate_worker_a.runOnceDetailed(); + try std.testing.expect(!duplicate_blocked.durableProgressed()); + { + const stats = duplicate_worker_a.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + manual_clock.advanceMs(101); + const duplicate_takeover = try duplicate_worker_a.runOnceDetailed(); + try std.testing.expect(!duplicate_takeover.durableProgressed()); + { + const stats = duplicate_worker_a.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.takeover_count); + try std.testing.expectEqual(@as(u64, 3_101), stats.last_acquired_ms); + } + + const worker_a_lost = try worker_a.runOnceDetailed(); + try std.testing.expect(!worker_a_lost.durableProgressed()); + { + const stats = worker_a.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const worker_b_renewed = try worker_b.runOnceDetailed(); + try std.testing.expect(!worker_b_renewed.durableProgressed()); + { + const stats = worker_b.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + } +} + +test "db graph metric runtime role worker pool leases are scoped by worker identity set" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(4_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + const pool_workers = [_][]const u8{ "runtime-pool-worker-a", "runtime-pool-worker-b" }; + const reversed_pool_workers = [_][]const u8{ "runtime-pool-worker-b", "runtime-pool-worker-a" }; + const other_pool_workers = [_][]const u8{ "runtime-pool-worker-c", "runtime-pool-worker-d" }; + var pool_a = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-worker-pool-lease-a", + .lease_owned = true, + .owner_id = "runtime-worker-pool-owner-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_ids = pool_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer pool_a.deinit(); + var duplicate_reordered_pool = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-worker-pool-lease-a-reordered", + .lease_owned = true, + .owner_id = "runtime-worker-pool-owner-a-reordered", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_ids = reversed_pool_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer duplicate_reordered_pool.deinit(); + var pool_b = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-worker-pool-lease-b", + .lease_owned = true, + .owner_id = "runtime-worker-pool-owner-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_ids = other_pool_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer pool_b.deinit(); + + const idle_pool_a = try pool_a.runOnceDetailed(); + try std.testing.expect(!idle_pool_a.durableProgressed()); + { + const stats = pool_a.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(usize, 2), stats.worker_count); + try std.testing.expect(stats.worker_id_hash != 0); + try std.testing.expect(stats.lease_key_hash != 0); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_reordered_pool.stats().lease_key_hash); + try std.testing.expect(stats.lease_key_hash != pool_b.stats().lease_key_hash); + } + + const duplicate_blocked = try duplicate_reordered_pool.runOnceDetailed(); + try std.testing.expect(!duplicate_blocked.durableProgressed()); + { + const stats = duplicate_reordered_pool.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 0), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + try std.testing.expectEqual(pool_a.stats().worker_id_hash, stats.worker_id_hash); + } + + const idle_pool_b = try pool_b.runOnceDetailed(); + try std.testing.expect(!idle_pool_b.durableProgressed()); + { + const stats = pool_b.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 0), stats.lease_acquire_failures); + try std.testing.expect(stats.worker_id_hash != pool_a.stats().worker_id_hash); + } + + manual_clock.advanceMs(101); + const duplicate_takeover = try duplicate_reordered_pool.runOnceDetailed(); + try std.testing.expect(!duplicate_takeover.durableProgressed()); + { + const stats = duplicate_reordered_pool.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 4_101), stats.last_acquired_ms); + } +} + +test "db graph metric runtime role planned worker pools reject duplicate worker identities" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + const duplicate_workers = [_][]const u8{ "runtime-pool-duplicate", "runtime-pool-duplicate" }; + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = duplicate_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 2, + })); + + const resources = db.core.asyncResources(); + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-worker-pool-duplicate", + .planned_options = .{ + .worker_ids = duplicate_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 2, + }, + }, + )); +} + +test "db graph metric runtime role owned runtime worker calls are bound to configured identity" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(5_000 * std.time.ns_per_ms); + const resources = db.core.asyncResources(); + var worker_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-owned-bound-worker", + .lease_owned = true, + .owner_id = "runtime-owned-bound-worker-owner", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-owned-bound-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_runtime.deinit(); + + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, worker_runtime.runWorkerOnce("runtime-owned-bound-worker-b")); + { + const stats = worker_runtime.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.ticks_started); + try std.testing.expectEqual(@as(u64, 0), stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), stats.error_ticks); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildWorker", stats.last_error_name.?); + } + + const allowed_worker_tick = try worker_runtime.runWorkerOnce("runtime-owned-bound-worker-a"); + try std.testing.expect(!allowed_worker_tick.durableProgressed()); + { + const stats = worker_runtime.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), stats.idle_ticks); + try std.testing.expectEqual(@as(?[]const u8, null), stats.last_error_name); + } + try std.testing.expectError(error.InvalidGraphMetricRuntimeRole, worker_runtime.runCoordinatorOnce(false)); + { + const stats = worker_runtime.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqualStrings("InvalidGraphMetricRuntimeRole", stats.last_error_name.?); + } + + const pool_workers = [_][]const u8{ "runtime-owned-bound-pool-a", "runtime-owned-bound-pool-b" }; + var pool_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-owned-bound-pool", + .lease_owned = true, + .owner_id = "runtime-owned-bound-pool-owner", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_ids = pool_workers[0..], + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer pool_runtime.deinit(); + + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, pool_runtime.runWorkerOnce("runtime-owned-bound-pool-c")); + { + const stats = pool_runtime.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.error_ticks); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildWorker", stats.last_error_name.?); + } + + const allowed_pool_tick = try pool_runtime.runWorkerOnce("runtime-owned-bound-pool-b"); + try std.testing.expect(!allowed_pool_tick.durableProgressed()); + { + const stats = pool_runtime.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), stats.ticks_completed); + try std.testing.expectEqual(@as(?[]const u8, null), stats.last_error_name); + } + try std.testing.expectError(error.InvalidGraphMetricRuntimeRole, pool_runtime.runCoordinatorOnce(false)); + { + const stats = pool_runtime.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqualStrings("InvalidGraphMetricRuntimeRole", stats.last_error_name.?); + } + + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-owned-bound-coordinator", + .lease_owned = true, + .owner_id = "runtime-owned-bound-coordinator-owner", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-owned-bound-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 1, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, coordinator_runtime.runWorkerOnce("runtime-owned-bound-coordinator-worker")); + { + const stats = coordinator_runtime.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.error_ticks); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildWorker", stats.last_error_name.?); + } + try std.testing.expectError(error.InvalidGraphMetricRuntimeRole, coordinator_runtime.runWorkerPoolOnce()); + { + const stats = coordinator_runtime.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqualStrings("InvalidGraphMetricRuntimeRole", stats.last_error_name.?); + } +} + +test "db graph metric runtime role automatic coordinator and worker loops stay separate" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-role-coordinator-owner", + .planned_options = .{ + .worker_id = "runtime-role-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + var worker_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-role-worker-owner", + .planned_options = .{ + .worker_id = "runtime-role-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_runtime.deinit(); + + const early_worker = try worker_runtime.runOnceDetailed(); + try std.testing.expect(!early_worker.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), early_worker.builds_started); + try std.testing.expectEqual(@as(usize, 0), early_worker.worker_steps); + + const coordinator_start = try coordinator_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), coordinator_start.builds_started); + try std.testing.expectEqual(@as(usize, 0), coordinator_start.worker_steps); + try std.testing.expectEqual(@as(usize, 0), coordinator_start.pages_completed); + + const worker_prepare = try worker_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.worker_steps); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.pages_completed); + try std.testing.expectEqual(@as(usize, 0), worker_prepare.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), worker_prepare.published); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, status.phase); + } + + const coordinator_advance = try coordinator_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 0), coordinator_advance.builds_started); + try std.testing.expect(coordinator_advance.phases_advanced > 0); + + var finished = false; + var steps: usize = 0; + while (steps < 200) : (steps += 1) { + const worker_tick = try worker_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 0), worker_tick.phases_advanced); + + const coordinator_tick = try coordinator_runtime.runOnceDetailed(); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + finished = true; + break; + } + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-role-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_runtime.stats(); + try std.testing.expectEqual(Role.worker, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-role-worker-owner"), worker_stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-role-worker"), worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 1), worker_stats.worker_count); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expect(worker_stats.total_result.worker_steps > 0); + try std.testing.expect(worker_stats.total_result.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:a", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime role distinct worker owners complete separate active pages" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d:0>3}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + // Keep this lease-takeover fixture multi-page without thousands of writes. + db.core.graphIndex("graph_idx").?.index.test_partition_target_units = 64; + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var manual_clock = platform_clock.ManualClock{}; + manual_clock.setRealtimeNs(6_000 * std.time.ns_per_ms); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-distinct-workers-coordinator", + .lease_owned = true, + .owner_id = "runtime-distinct-workers-coordinator", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-distinct-workers-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + var worker_a_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-distinct-worker-a", + .lease_owned = true, + .owner_id = "runtime-distinct-worker-a", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-distinct-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_a_runtime.deinit(); + + var worker_b_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-distinct-worker-b", + .lease_owned = true, + .owner_id = "runtime-distinct-worker-b", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-distinct-worker-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_b_runtime.deinit(); + + var replacement_worker_a_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-distinct-worker-a-replacement", + .lease_owned = true, + .owner_id = "runtime-distinct-worker-a-replacement", + .lease_ttl_ms = 100, + .clock = manual_clock.clock(), + .planned_options = .{ + .worker_id = "runtime-distinct-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer replacement_worker_a_runtime.deinit(); + + const coordinator_start = try coordinator_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), coordinator_start.builds_started); + try std.testing.expectEqual(@as(usize, 0), coordinator_start.worker_steps); + + const worker_a_prepare = try worker_a_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), worker_a_prepare.worker_steps); + try std.testing.expectEqual(@as(usize, 1), worker_a_prepare.pages_completed); + try std.testing.expectEqual(@as(usize, 0), worker_a_prepare.phases_advanced); + + const coordinator_scan = try coordinator_runtime.runOnceDetailed(); + try std.testing.expect(coordinator_scan.phases_advanced > 0); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.phase); + } + + const replacement_blocked = try replacement_worker_a_runtime.runOnceDetailed(); + try std.testing.expect(!replacement_blocked.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), replacement_blocked.worker_steps); + try std.testing.expectEqual(@as(usize, 0), replacement_blocked.pages_completed); + { + const stats = replacement_worker_a_runtime.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(worker_a_runtime.stats().lease_key_hash, stats.lease_key_hash); + try std.testing.expectEqual(worker_a_runtime.stats().worker_id_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + manual_clock.advanceMs(101); + const replacement_scan = try replacement_worker_a_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), replacement_scan.worker_steps); + try std.testing.expectEqual(@as(usize, 1), replacement_scan.pages_completed); + try std.testing.expectEqual(@as(usize, 0), replacement_scan.phases_advanced); + try std.testing.expect(replacement_scan.budget_exhausted); + { + const stats = replacement_worker_a_runtime.stats(); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 6_101), stats.last_acquired_ms); + try std.testing.expect(stats.last_result.budget_exhausted); + } + + const worker_a_lost = try worker_a_runtime.runOnceDetailed(); + try std.testing.expect(!worker_a_lost.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), worker_a_lost.worker_steps); + { + const stats = worker_a_runtime.stats(); + try std.testing.expect(!stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), stats.lost_leases); + try std.testing.expectEqual(@as(u64, 1), stats.lease_acquire_failures); + } + + const worker_b_scan = try worker_b_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), worker_b_scan.worker_steps); + try std.testing.expectEqual(@as(usize, 1), worker_b_scan.pages_completed); + try std.testing.expectEqual(@as(usize, 0), worker_b_scan.phases_advanced); + + { + const worker_a_stats = worker_a_runtime.stats(); + const worker_b_stats = worker_b_runtime.stats(); + try std.testing.expect(worker_a_stats.lease_owned); + try std.testing.expect(worker_b_stats.lease_owned); + try std.testing.expect(!worker_a_stats.has_lease); + try std.testing.expect(worker_b_stats.has_lease); + try std.testing.expect(worker_a_stats.lease_key_hash != 0); + try std.testing.expect(worker_b_stats.lease_key_hash != 0); + try std.testing.expect(worker_a_stats.lease_key_hash != worker_b_stats.lease_key_hash); + try std.testing.expect(worker_a_stats.worker_id_hash != worker_b_stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 1), worker_a_stats.lease_acquire_failures); + try std.testing.expectEqual(@as(u64, 0), worker_b_stats.lease_acquire_failures); + try std.testing.expect(worker_a_stats.total_result.pages_completed >= 1); + try std.testing.expect(worker_b_stats.total_result.pages_completed >= 1); + const replacement_stats = replacement_worker_a_runtime.stats(); + try std.testing.expect(replacement_stats.has_lease); + try std.testing.expectEqual(worker_a_stats.lease_key_hash, replacement_stats.lease_key_hash); + try std.testing.expectEqual(worker_a_stats.worker_id_hash, replacement_stats.worker_id_hash); + try std.testing.expect(replacement_stats.total_result.pages_completed >= 1); + } + + var finished = false; + var steps: usize = 0; + var consecutive_idle_workers: usize = 0; + while (steps < 200) : (steps += 1) { + const worker_tick = if (steps % 2 == 0) + try replacement_worker_a_runtime.runOnceDetailed() + else + try worker_b_runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 0), worker_tick.phases_advanced); + + const coordinator_tick = try coordinator_runtime.runOnceDetailed(); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation and status.phase == .complete) { + finished = true; + break; + } + if (worker_tick.durableProgressed() or coordinator_tick.durableProgressed()) { + consecutive_idle_workers = 0; + } else { + consecutive_idle_workers += 1; + } + // A cursor-resumable page remains bound to its current worker until + // completion or lease expiry. Only declare a stall after every worker + // identity in this pool has had a chance to run. + if (consecutive_idle_workers >= 2) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:hub", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime background skips paused metrics" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var paused = try db.pauseGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer paused.deinit(alloc); + try std.testing.expect(paused.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, paused.state); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expectEqual(@as(usize, 1), pending.paused_metrics); + try std.testing.expect(!pending.hasWork()); + } + + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-paused-degree", + .planned_options = .{ + .worker_id = "runtime-paused-degree-worker", + .max_rounds = 4, + .max_metrics_per_round = 8, + .max_pages_per_round = 4, + }, + }, + ); + defer runtime.deinit(); + + const paused_tick = try runtime.runOnceDetailed(); + try std.testing.expect(!paused_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), paused_tick.builds_started); + try std.testing.expectEqual(@as(usize, 0), paused_tick.pages_claimed); + try std.testing.expectEqual(@as(usize, 0), paused_tick.pages_completed); + try std.testing.expectEqual(@as(usize, 0), paused_tick.published); + { + const runtime_stats = runtime.stats(); + try std.testing.expectEqual(@as(u64, 1), runtime_stats.ticks_started); + try std.testing.expectEqual(@as(u64, 1), runtime_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 0), runtime_stats.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), runtime_stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), runtime_stats.error_ticks); + try std.testing.expect(!runtime_stats.last_result.durableProgressed()); + } + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expect(status.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + try std.testing.expectEqual(@as(u64, 0), status.build_job_id); + try std.testing.expectEqual(@as(u64, 0), status.published_generation); + } + + var resumed = try db.resumeGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer resumed.deinit(alloc); + try std.testing.expect(!resumed.maintenance_paused); + + var steps: usize = 0; + while (try runtime.runOnce()) { + steps += 1; + if (steps > 200) return error.TestUnexpectedResult; + } + try std.testing.expect(steps > 0); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background idles after synchronously cleaning a small failed generation" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-failed-terminal-degree", + .planned_options = .{ + .worker_id = "runtime-failed-terminal-degree-worker", + .max_rounds = 4, + .max_metrics_per_round = 8, + .max_pages_per_round = 4, + }, + }, + ); + defer runtime.deinit(); + db.graph_metric_runtime = &runtime; + defer db.graph_metric_runtime = null; + + const started_tick = try runtime.runCoordinatorOnce(true); + try std.testing.expect(started_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), started_tick.builds_started); + + var failed = try db.failGraphMetricPlannedBuild(alloc, "graph_idx", "degree", error.InvalidGraphMetricBuildManifest); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expect(failed.build_queued); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed.last_error); + const failed_target_generation = failed.target_edge_generation; + + const terminal_tick = try runtime.runOnceDetailed(); + try std.testing.expect(!terminal_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.active_builds); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.builds_started); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.pages_claimed); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.pages_completed); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.published); + try std.testing.expectEqual(@as(usize, 0), terminal_tick.failed_builds); + + { + const runtime_stats = runtime.stats(); + try std.testing.expectEqual(@as(u64, 2), runtime_stats.ticks_started); + try std.testing.expectEqual(@as(u64, 2), runtime_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), runtime_stats.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), runtime_stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), runtime_stats.error_ticks); + try std.testing.expect(!runtime_stats.last_result.durableProgressed()); + } + { + const mapped_stats = try db.stats(alloc); + defer types.freeDBStats(alloc, mapped_stats); + try std.testing.expect(mapped_stats.graph_metric_runtime.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.combined, mapped_stats.graph_metric_runtime.role.?); + try std.testing.expectEqual(@as(u64, 2), mapped_stats.graph_metric_runtime.ticks_started); + try std.testing.expectEqual(mapped_stats.graph_metric_runtime.ticks_started, mapped_stats.graph_metric_runtime.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), mapped_stats.graph_metric_runtime.error_ticks); + try std.testing.expectEqual(@as(u64, 0), mapped_stats.graph_metric_runtime.last_builds_started); + try std.testing.expectEqual(@as(u64, 0), mapped_stats.graph_metric_runtime.last_failed_builds); + try std.testing.expect(!mapped_stats.graph_metric_runtime.last_budget_exhausted); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqual(failed_target_generation, status.target_edge_generation); + try std.testing.expectEqual(@as(u64, 0), status.build_job_id); + var failed_events: usize = 0; + for (status.recent_events) |event| { + if (event.kind == .failed) failed_events += 1; + } + try std.testing.expectEqual(@as(usize, 1), failed_events); + } + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:c", + .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const new_generation_tick = try runtime.runCoordinatorOnce(true); + try std.testing.expect(new_generation_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), new_generation_tick.builds_started); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expect(status.building_generation > failed_target_generation); + } +} + +test "db graph metric runtime background skips paused active planned builds" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-paused-active-degree", + .planned_options = .{ + .worker_id = "runtime-paused-active-degree-worker", + .max_rounds = 4, + .max_metrics_per_round = 8, + .max_pages_per_round = 4, + }, + }, + ); + defer runtime.deinit(); + + const started_tick = try runtime.runCoordinatorOnce(true); + try std.testing.expect(started_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 1), started_tick.builds_started); + try std.testing.expectEqual(@as(usize, 0), started_tick.pages_claimed); + try std.testing.expectEqual(@as(usize, 0), started_tick.pages_completed); + try std.testing.expectEqual(@as(usize, 0), started_tick.published); + + const active_job_id, const active_phase = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(target_generation, status.building_generation); + try std.testing.expect(status.build_job_id != 0); + break :blk .{ status.build_job_id, status.phase }; + }; + + var paused = try db.pauseGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer paused.deinit(alloc); + try std.testing.expect(paused.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, paused.state); + try std.testing.expectEqual(target_generation, paused.building_generation); + try std.testing.expectEqual(active_job_id, paused.build_job_id); + try std.testing.expectEqual(active_phase, paused.phase); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expectEqual(@as(usize, 1), pending.paused_metrics); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + try std.testing.expect(!pending.hasWork()); + } + + const paused_worker_tick = try runtime.runWorkerOnce("runtime-paused-active-worker"); + try std.testing.expect(!paused_worker_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), paused_worker_tick.active_builds); + try std.testing.expectEqual(@as(usize, 0), paused_worker_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), paused_worker_tick.pages_claimed); + try std.testing.expectEqual(@as(usize, 0), paused_worker_tick.pages_completed); + try std.testing.expectEqual(@as(usize, 0), paused_worker_tick.published); + + const paused_coordinator_tick = try runtime.runCoordinatorOnce(true); + try std.testing.expect(!paused_coordinator_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), paused_coordinator_tick.active_builds); + try std.testing.expectEqual(@as(usize, 0), paused_coordinator_tick.builds_started); + try std.testing.expectEqual(@as(usize, 0), paused_coordinator_tick.coordinator_steps); + try std.testing.expectEqual(@as(usize, 0), paused_coordinator_tick.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), paused_coordinator_tick.published); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expect(status.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(target_generation, status.building_generation); + try std.testing.expectEqual(active_job_id, status.build_job_id); + try std.testing.expectEqual(active_phase, status.phase); + try std.testing.expectEqual(@as(usize, 0), status.build_pages.len); + try std.testing.expectEqual(@as(u64, 0), status.published_generation); + } + + var resumed = try db.resumeGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer resumed.deinit(alloc); + try std.testing.expect(!resumed.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, resumed.state); + try std.testing.expectEqual(target_generation, resumed.building_generation); + try std.testing.expectEqual(active_job_id, resumed.build_job_id); + + var finished = false; + var step_index: usize = 0; + while (step_index < 200) : (step_index += 1) { + const worker_tick = try runtime.runWorkerOnce("runtime-resumed-active-worker"); + const coordinator_tick = try runtime.runCoordinatorOnce(false); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + finished = true; + break; + } + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background starts automatically and drains notified degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .runtime_id = "runtime-auto-degree", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-auto-degree-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.executor.waitForAll(db.core.nextDerivedSequence()); + + var target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var fresh = false; + for (0..700) |_| { + yieldToBackground(&db); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + // Derived graph updates and metric maintenance are independent + // background pipelines. Convergence means publishing the latest graph + // generation, which may legitimately be newer than the generation we + // first observed after the document executor drained. + const current_generation = graph_entry.index.edge_generation; + if (status.state == .fresh and status.published_edge_generation == current_generation) { + target_generation = current_generation; + fresh = true; + break; + } + } + try std.testing.expect(fresh); + + { + const runtime_stats = db.graphMetricRuntimeStats(); + try std.testing.expect(runtime_stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.combined, runtime_stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-auto-degree"), runtime_stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-auto-degree-worker"), runtime_stats.worker_id_hash); + try std.testing.expect(runtime_stats.ticks_started > 0); + try std.testing.expect(runtime_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), runtime_stats.error_ticks); + try std.testing.expect(runtime_stats.total_builds_started > 0); + try std.testing.expect(runtime_stats.total_worker_steps > 0); + try std.testing.expect(runtime_stats.total_coordinator_steps > 0); + try std.testing.expect(runtime_stats.total_pages_completed > 0); + try std.testing.expect(runtime_stats.total_published > 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:a", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background open-configured split owners publish degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + + for (0..64) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d:0>3}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + var coordinator_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var worker_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + const workers = [_][]const u8{ "open-runtime-worker-a", "open-runtime-worker-b" }; + var saw_coordinator_role = false; + var saw_worker_pool_role = false; + var fresh = false; + for (0..400) |_| { + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .coordinator, + .runtime_id = "open-configured-coordinator", + .lease_owned = true, + .owner_id = "open-configured-coordinator", + .planned_options = .{ + .worker_id = "open-configured-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer coordinator.close(); + + const tick = try coordinator.graph_metric_runtime.?.runOnceDetailed(); + coordinator_total.add(tick); + const stats = coordinator.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.coordinator, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "open-configured-coordinator"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "open-configured-coordinator"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + try std.testing.expectEqual(@as(u64, 0), stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_worker_steps); + try std.testing.expectEqual(@as(u64, 0), stats.error_ticks); + saw_coordinator_role = true; + } + + { + var worker_pool = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .worker_pool, + .runtime_id = "open-configured-worker-pool", + .lease_owned = true, + .owner_id = "open-configured-worker-pool", + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + }); + defer worker_pool.close(); + + const tick = try worker_pool.graph_metric_runtime.?.runOnceDetailed(); + worker_total.add(tick); + const stats = worker_pool.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.worker_pool, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "open-configured-worker-pool"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "open-configured-worker-pool"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + try std.testing.expectEqual(workerSetIdentityHash(workers[0..]), stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 2), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_coordinator_steps); + try std.testing.expectEqual(@as(u64, 0), stats.total_published); + try std.testing.expectEqual(@as(u64, 0), stats.error_ticks); + saw_worker_pool_role = true; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .coordinator, + .runtime_id = "open-configured-coordinator", + .lease_owned = true, + .owner_id = "open-configured-coordinator", + .planned_options = .{ + .worker_id = "open-configured-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer coordinator.close(); + + const tick = try coordinator.graph_metric_runtime.?.runOnceDetailed(); + coordinator_total.add(tick); + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + fresh = true; + break; + } + } + } + try std.testing.expect(fresh); + try std.testing.expect(saw_coordinator_role); + try std.testing.expect(saw_worker_pool_role); + try std.testing.expect(worker_total.worker_steps > 0); + try std.testing.expect(worker_total.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_total.coordinator_steps); + + try std.testing.expect(coordinator_total.builds_started > 0); + try std.testing.expect(coordinator_total.coordinator_steps > 0); + try std.testing.expect(coordinator_total.phases_advanced > 0); + try std.testing.expect(coordinator_total.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_total.worker_steps); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var metric_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:hub", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 64.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); + } +} + +test "db graph metric runtime background separates coordinator and worker ticks" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .planned_options = .{ + .worker_id = "runtime-split-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + try std.testing.expectError(error.InvalidGraphMetricBuildWorker, runtime.runWorkerOnce("")); + { + const error_stats = runtime.stats(); + try std.testing.expectEqual(@as(u64, 1), error_stats.ticks_started); + try std.testing.expectEqual(@as(u64, 0), error_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), error_stats.error_ticks); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildWorker", error_stats.last_error_name.?); + } + + const coordinator_start = try runtime.runCoordinatorOnce(true); + try std.testing.expectEqual(@as(usize, 1), coordinator_start.builds_started); + try std.testing.expectEqual(@as(usize, 0), coordinator_start.worker_steps); + try std.testing.expectEqual(@as(usize, 0), coordinator_start.pages_completed); + { + const recovered_stats = runtime.stats(); + try std.testing.expectEqual(@as(?[]const u8, null), recovered_stats.last_error_name); + try std.testing.expectEqual(@as(u64, 1), recovered_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 1), recovered_stats.durable_progress_ticks); + try std.testing.expectEqual(@as(usize, 1), recovered_stats.last_result.builds_started); + } + + const worker_prepare = try runtime.runWorkerOnce("runtime-split-worker"); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.worker_steps); + try std.testing.expectEqual(@as(usize, 1), worker_prepare.pages_completed); + try std.testing.expectEqual(@as(usize, 0), worker_prepare.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), worker_prepare.published); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, status.phase); + } + + const coordinator_advance = try runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), coordinator_advance.builds_started); + try std.testing.expect(coordinator_advance.phases_advanced > 0); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, status.phase); + } + + var steps: usize = 0; + while (try runtime.runOnce()) { + steps += 1; + if (steps > 200) return error.TestUnexpectedResult; + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:a", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime background coordinator and worker loops publish degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-bg-coordinator-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + var worker_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-bg-worker-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer worker_runtime.deinit(); + + try coordinator_runtime.start(); + try worker_runtime.start(); + coordinator_runtime.notify(); + worker_runtime.notify(); + + try std.testing.expect(try waitForGraphMetricFresh(alloc, &db, "graph_idx", "degree", target_generation, 300)); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expect(coordinator_stats.started); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.ticks_started > 0); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.error_ticks); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_runtime.stats(); + try std.testing.expect(worker_stats.started); + try std.testing.expectEqual(Role.worker, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-worker-owner"), worker_stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-worker"), worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 1), worker_stats.worker_count); + try std.testing.expect(worker_stats.ticks_started > 0); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), worker_stats.error_ticks); + try std.testing.expect(worker_stats.total_result.worker_steps > 0); + try std.testing.expect(worker_stats.total_result.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:a", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background coordinator and worker pool loops publish degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d:0>3}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-bg-pool-coordinator-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + const workers = [_][]const u8{ "runtime-bg-pool-worker-a", "runtime-bg-pool-worker-b" }; + var worker_pool_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-bg-pool-worker-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer worker_pool_runtime.deinit(); + + try coordinator_runtime.start(); + try worker_pool_runtime.start(); + coordinator_runtime.notify(); + worker_pool_runtime.notify(); + + try std.testing.expect(try waitForGraphMetricFresh(alloc, &db, "graph_idx", "degree", target_generation, 500)); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expect(coordinator_stats.started); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-pool-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.ticks_started > 0); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.error_ticks); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_pool_runtime.stats(); + try std.testing.expect(worker_stats.started); + try std.testing.expectEqual(Role.worker_pool, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-pool-worker-owner"), worker_stats.runtime_id_hash); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), worker_stats.worker_count); + try std.testing.expect(worker_stats.ticks_started > 0); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), worker_stats.error_ticks); + try std.testing.expect(worker_stats.total_result.worker_steps >= 2); + try std.testing.expect(worker_stats.total_result.pages_completed >= 2); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:hub", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 130.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background coordinator and worker pool loops publish pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-bg-pagerank-pool-coordinator-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-pagerank-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + const workers = [_][]const u8{ "runtime-bg-pagerank-pool-worker-a", "runtime-bg-pagerank-pool-worker-b" }; + var worker_pool_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-bg-pagerank-pool-worker-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer worker_pool_runtime.deinit(); + + try coordinator_runtime.start(); + try worker_pool_runtime.start(); + coordinator_runtime.notify(); + worker_pool_runtime.notify(); + + try std.testing.expect(try waitForGraphMetricFresh(alloc, &db, "graph_idx", "pagerank", target_generation, 700)); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expect(coordinator_stats.started); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-pagerank-pool-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.ticks_started > 0); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.error_ticks); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expect(coordinator_stats.total_result.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_pool_runtime.stats(); + try std.testing.expect(worker_stats.started); + try std.testing.expectEqual(Role.worker_pool, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-pagerank-pool-worker-owner"), worker_stats.runtime_id_hash); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), worker_stats.worker_count); + try std.testing.expect(worker_stats.ticks_started > 0); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), worker_stats.error_ticks); + try std.testing.expect(worker_stats.total_result.worker_steps > 0); + try std.testing.expect(worker_stats.total_result.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime background coordinator and worker pool loops publish eigenvector" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-bg-eigenvector-pool-coordinator-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-eigenvector-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + const workers = [_][]const u8{ "runtime-bg-eigenvector-pool-worker-a", "runtime-bg-eigenvector-pool-worker-b" }; + var worker_pool_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-bg-eigenvector-pool-worker-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer worker_pool_runtime.deinit(); + + try coordinator_runtime.start(); + try worker_pool_runtime.start(); + coordinator_runtime.notify(); + worker_pool_runtime.notify(); + + try std.testing.expect(try waitForGraphMetricFresh(alloc, &db, "graph_idx", "eigenvector", target_generation, 700)); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expect(coordinator_stats.started); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-eigenvector-pool-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.ticks_started > 0); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.error_ticks); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expect(coordinator_stats.total_result.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_pool_runtime.stats(); + try std.testing.expect(worker_stats.started); + try std.testing.expectEqual(Role.worker_pool, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-eigenvector-pool-worker-owner"), worker_stats.runtime_id_hash); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), worker_stats.worker_count); + try std.testing.expect(worker_stats.ticks_started > 0); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), worker_stats.error_ticks); + try std.testing.expect(worker_stats.total_result.worker_steps > 0); + try std.testing.expect(worker_stats.total_result.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime background coordinator and worker pool loops publish hits pair" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var coordinator_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-bg-hits-pool-coordinator-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_id = "runtime-bg-hits-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer coordinator_runtime.deinit(); + + const workers = [_][]const u8{ "runtime-bg-hits-pool-worker-a", "runtime-bg-hits-pool-worker-b" }; + var worker_pool_runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-bg-hits-pool-worker-owner", + .idle_interval_ms = 1, + .error_interval_ms = 1, + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer worker_pool_runtime.deinit(); + + try coordinator_runtime.start(); + try worker_pool_runtime.start(); + coordinator_runtime.notify(); + worker_pool_runtime.notify(); + + try std.testing.expect(try waitForGraphMetricPairFresh(alloc, &db, "graph_idx", "hits_authority", "hits_hub", target_generation, 700)); + + { + const coordinator_stats = coordinator_runtime.stats(); + try std.testing.expect(coordinator_stats.started); + try std.testing.expectEqual(Role.coordinator, coordinator_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-hits-pool-coordinator-owner"), coordinator_stats.runtime_id_hash); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.worker_count); + try std.testing.expect(coordinator_stats.ticks_started > 0); + try std.testing.expect(coordinator_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), coordinator_stats.error_ticks); + try std.testing.expect(coordinator_stats.total_result.builds_started > 0); + try std.testing.expect(coordinator_stats.total_result.coordinator_steps > 0); + try std.testing.expect(coordinator_stats.total_result.phases_advanced > 0); + try std.testing.expect(coordinator_stats.total_result.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_stats.total_result.worker_steps); + try std.testing.expect(coordinator_stats.last_result.worker_steps == 0); + } + { + const worker_stats = worker_pool_runtime.stats(); + try std.testing.expect(worker_stats.started); + try std.testing.expectEqual(Role.worker_pool, worker_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-bg-hits-pool-worker-owner"), worker_stats.runtime_id_hash); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, worker_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), worker_stats.worker_count); + try std.testing.expect(worker_stats.ticks_started > 0); + try std.testing.expect(worker_stats.durable_progress_ticks > 0); + try std.testing.expectEqual(@as(u64, 0), worker_stats.error_ticks); + try std.testing.expect(worker_stats.total_result.worker_steps > 0); + try std.testing.expect(worker_stats.total_result.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_stats.total_result.coordinator_steps); + try std.testing.expect(worker_stats.last_result.coordinator_steps == 0); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); + try std.testing.expectEqualStrings("doc:authority", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime background worker pool survives separate reopened handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d:0>3}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "runtime-reopened-pool-worker-a", "runtime-reopened-pool-worker-b" }; + const reversed_workers = [_][]const u8{ "runtime-reopened-pool-worker-b", "runtime-reopened-pool-worker-a" }; + var coordinator_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var worker_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var saw_worker_pool_role = false; + var saw_two_page_worker_pool_tick = false; + var saw_live_duplicate_worker_pool_fenced = false; + var fresh = false; + for (0..400) |_| { + const worker_tick = blk: { + var worker_pool = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_pool.close(); + + const worker_resources = worker_pool.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + worker_resources.store, + worker_resources.index_manager, + worker_resources.apply_mutex, + worker_pool.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-reopened-pool-worker-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-pool-worker-owner", + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer runtime.deinit(); + + const tick = try runtime.runOnceDetailed(); + const stats = runtime.stats(); + try std.testing.expectEqual(Role.worker_pool, stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pool-worker-owner"), stats.runtime_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 2), stats.worker_count); + try std.testing.expectEqual(@as(usize, 0), stats.total_result.coordinator_steps); + saw_worker_pool_role = true; + if (tick.worker_steps >= 2 and tick.pages_completed >= 2) saw_two_page_worker_pool_tick = true; + if (tick.worker_steps > 0) { + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + worker_resources.store, + worker_resources.index_manager, + worker_resources.apply_mutex, + worker_pool.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-reopened-pool-worker-owner-duplicate", + .lease_owned = true, + .owner_id = "runtime-reopened-pool-worker-owner-duplicate", + .planned_options = .{ + .worker_ids = &reversed_workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const duplicate_tick = try duplicate_runtime.runOnceDetailed(); + try std.testing.expect(!duplicate_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.pages_completed); + const duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(!duplicate_stats.has_lease); + try std.testing.expectEqual(stats.worker_id_hash, duplicate_stats.worker_id_hash); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_stats.lease_key_hash); + try std.testing.expectEqual(@as(u64, 0), duplicate_stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), duplicate_stats.lease_acquire_failures); + saw_live_duplicate_worker_pool_fenced = true; + } + break :blk tick; + }; + worker_total.add(worker_tick); + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const coordinator_resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + coordinator_resources.store, + coordinator_resources.index_manager, + coordinator_resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-reopened-pool-coordinator-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-pool-coordinator-owner", + .planned_options = .{ + .worker_id = "runtime-reopened-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const tick = try runtime.runOnceDetailed(); + const stats = runtime.stats(); + try std.testing.expectEqual(Role.coordinator, stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pool-coordinator-owner"), stats.runtime_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(@as(u64, 0), stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 0), stats.worker_count); + try std.testing.expectEqual(@as(usize, 0), stats.total_result.worker_steps); + break :blk tick; + }; + coordinator_total.add(coordinator_tick); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + fresh = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(fresh); + try std.testing.expect(saw_worker_pool_role); + try std.testing.expect(saw_two_page_worker_pool_tick); + try std.testing.expect(saw_live_duplicate_worker_pool_fenced); + try std.testing.expect(coordinator_total.builds_started > 0); + try std.testing.expect(coordinator_total.coordinator_steps > 0); + try std.testing.expect(coordinator_total.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_total.worker_steps); + try std.testing.expect(worker_total.worker_steps >= 2); + try std.testing.expect(worker_total.pages_completed >= 2); + try std.testing.expectEqual(@as(usize, 0), worker_total.coordinator_steps); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var metric_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:hub", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 130.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); + } +} + +test "db graph metric runtime background open-configured pagerank worker pool survives separate reopened handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "runtime-reopened-pagerank-pool-worker-a", "runtime-reopened-pagerank-pool-worker-b" }; + const reversed_workers = [_][]const u8{ "runtime-reopened-pagerank-pool-worker-b", "runtime-reopened-pagerank-pool-worker-a" }; + var coordinator_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var worker_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var saw_worker_pool_role = false; + var saw_live_duplicate_worker_pool_fenced = false; + var fresh = false; + for (0..500) |_| { + const worker_tick = blk: { + var worker_pool = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .worker_pool, + .runtime_id = "runtime-reopened-pagerank-pool-worker-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-pagerank-pool-worker-owner", + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + }); + defer worker_pool.close(); + + const tick = try worker_pool.graph_metric_runtime.?.runOnceDetailed(); + const stats = worker_pool.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.worker_pool, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pagerank-pool-worker-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pagerank-pool-worker-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 2), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_coordinator_steps); + saw_worker_pool_role = true; + if (tick.worker_steps > 0) { + const duplicate_resources = worker_pool.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + worker_pool.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-reopened-pagerank-pool-worker-owner-duplicate", + .lease_owned = true, + .owner_id = "runtime-reopened-pagerank-pool-worker-owner-duplicate", + .planned_options = .{ + .worker_ids = &reversed_workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const duplicate_tick = try duplicate_runtime.runOnceDetailed(); + try std.testing.expect(!duplicate_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.pages_completed); + const duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(duplicate_stats.enabled); + try std.testing.expectEqual(Role.worker_pool, duplicate_stats.role); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(!duplicate_stats.has_lease); + try std.testing.expectEqual(stats.worker_id_hash, duplicate_stats.worker_id_hash); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_stats.lease_key_hash); + try std.testing.expectEqual(@as(u64, 0), duplicate_stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), duplicate_stats.lease_acquire_failures); + saw_live_duplicate_worker_pool_fenced = true; + } + break :blk tick; + }; + worker_total.add(worker_tick); + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .coordinator, + .runtime_id = "runtime-reopened-pagerank-pool-coordinator-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-pagerank-pool-coordinator-owner", + .planned_options = .{ + .worker_id = "runtime-reopened-pagerank-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer coordinator.close(); + + const tick = try coordinator.graph_metric_runtime.?.runOnceDetailed(); + const stats = coordinator.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.coordinator, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pagerank-pool-coordinator-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-pagerank-pool-coordinator-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + try std.testing.expectEqual(@as(u64, 0), stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_worker_steps); + break :blk tick; + }; + coordinator_total.add(coordinator_tick); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + fresh = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(fresh); + try std.testing.expect(saw_worker_pool_role); + try std.testing.expect(saw_live_duplicate_worker_pool_fenced); + try std.testing.expect(coordinator_total.builds_started > 0); + try std.testing.expect(coordinator_total.coordinator_steps > 0); + try std.testing.expect(coordinator_total.phases_advanced > 0); + try std.testing.expect(coordinator_total.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_total.worker_steps); + try std.testing.expect(worker_total.worker_steps > 0); + try std.testing.expect(worker_total.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_total.coordinator_steps); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var metric_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(metric_result.graph_metric_results[0].scores[0].score >= metric_result.graph_metric_results[0].scores[1].score); + } +} + +test "db graph metric runtime background open-configured eigenvector worker pool survives separate reopened handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "runtime-reopened-eigenvector-pool-worker-a", "runtime-reopened-eigenvector-pool-worker-b" }; + const reversed_workers = [_][]const u8{ "runtime-reopened-eigenvector-pool-worker-b", "runtime-reopened-eigenvector-pool-worker-a" }; + var coordinator_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var worker_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var saw_worker_pool_role = false; + var saw_live_duplicate_worker_pool_fenced = false; + var fresh = false; + for (0..500) |_| { + const worker_tick = blk: { + var worker_pool = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .worker_pool, + .runtime_id = "runtime-reopened-eigenvector-pool-worker-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-eigenvector-pool-worker-owner", + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + }); + defer worker_pool.close(); + + const tick = try worker_pool.graph_metric_runtime.?.runOnceDetailed(); + const stats = worker_pool.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.worker_pool, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-eigenvector-pool-worker-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-eigenvector-pool-worker-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 2), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_coordinator_steps); + saw_worker_pool_role = true; + if (tick.worker_steps > 0) { + const duplicate_resources = worker_pool.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + worker_pool.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-reopened-eigenvector-pool-worker-owner-duplicate", + .lease_owned = true, + .owner_id = "runtime-reopened-eigenvector-pool-worker-owner-duplicate", + .planned_options = .{ + .worker_ids = &reversed_workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const duplicate_tick = try duplicate_runtime.runOnceDetailed(); + try std.testing.expect(!duplicate_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.pages_completed); + const duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(duplicate_stats.enabled); + try std.testing.expectEqual(Role.worker_pool, duplicate_stats.role); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(!duplicate_stats.has_lease); + try std.testing.expectEqual(stats.worker_id_hash, duplicate_stats.worker_id_hash); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_stats.lease_key_hash); + try std.testing.expectEqual(@as(u64, 0), duplicate_stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), duplicate_stats.lease_acquire_failures); + saw_live_duplicate_worker_pool_fenced = true; + } + break :blk tick; + }; + worker_total.add(worker_tick); + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .coordinator, + .runtime_id = "runtime-reopened-eigenvector-pool-coordinator-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-eigenvector-pool-coordinator-owner", + .planned_options = .{ + .worker_id = "runtime-reopened-eigenvector-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer coordinator.close(); + + const tick = try coordinator.graph_metric_runtime.?.runOnceDetailed(); + const stats = coordinator.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.coordinator, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-eigenvector-pool-coordinator-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-eigenvector-pool-coordinator-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + try std.testing.expectEqual(@as(u64, 0), stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_worker_steps); + break :blk tick; + }; + coordinator_total.add(coordinator_tick); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation) { + fresh = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(fresh); + try std.testing.expect(saw_worker_pool_role); + try std.testing.expect(saw_live_duplicate_worker_pool_fenced); + try std.testing.expect(coordinator_total.builds_started > 0); + try std.testing.expect(coordinator_total.coordinator_steps > 0); + try std.testing.expect(coordinator_total.phases_advanced > 0); + try std.testing.expect(coordinator_total.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_total.worker_steps); + try std.testing.expect(worker_total.worker_steps > 0); + try std.testing.expect(worker_total.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_total.coordinator_steps); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var metric_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); + } +} + +test "db graph metric runtime background open-configured hits worker pool survives separate reopened handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + const workers = [_][]const u8{ "runtime-reopened-hits-pool-worker-a", "runtime-reopened-hits-pool-worker-b" }; + const reversed_workers = [_][]const u8{ "runtime-reopened-hits-pool-worker-b", "runtime-reopened-hits-pool-worker-a" }; + var coordinator_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var worker_total = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepResult{}; + var saw_worker_pool_role = false; + var saw_live_duplicate_worker_pool_fenced = false; + var fresh = false; + for (0..500) |_| { + const worker_tick = blk: { + var worker_pool = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .worker_pool, + .runtime_id = "runtime-reopened-hits-pool-worker-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-hits-pool-worker-owner", + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + }); + defer worker_pool.close(); + + const tick = try worker_pool.graph_metric_runtime.?.runOnceDetailed(); + const stats = worker_pool.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.worker_pool, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-hits-pool-worker-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-hits-pool-worker-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + const expected_worker_hash = workerSetIdentityHash(workers[0..]); + try std.testing.expectEqual(expected_worker_hash, stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 2), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_coordinator_steps); + saw_worker_pool_role = true; + if (tick.worker_steps > 0) { + const duplicate_resources = worker_pool.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + worker_pool.backend_runtime, + .{ + .enabled = true, + .role = .worker_pool, + .runtime_id = "runtime-reopened-hits-pool-worker-owner-duplicate", + .lease_owned = true, + .owner_id = "runtime-reopened-hits-pool-worker-owner-duplicate", + .planned_options = .{ + .worker_ids = &reversed_workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const duplicate_tick = try duplicate_runtime.runOnceDetailed(); + try std.testing.expect(!duplicate_tick.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 0), duplicate_tick.pages_completed); + const duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(duplicate_stats.enabled); + try std.testing.expectEqual(Role.worker_pool, duplicate_stats.role); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(!duplicate_stats.has_lease); + try std.testing.expectEqual(stats.worker_id_hash, duplicate_stats.worker_id_hash); + try std.testing.expectEqual(stats.lease_key_hash, duplicate_stats.lease_key_hash); + try std.testing.expectEqual(@as(u64, 0), duplicate_stats.acquisition_count); + try std.testing.expectEqual(@as(u64, 1), duplicate_stats.lease_acquire_failures); + saw_live_duplicate_worker_pool_fenced = true; + } + break :blk tick; + }; + worker_total.add(worker_tick); + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .writer_no_replay, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_maintenance = .{ + .enabled = true, + .start_background_loop = false, + .role = .coordinator, + .runtime_id = "runtime-reopened-hits-pool-coordinator-owner", + .lease_owned = true, + .owner_id = "runtime-reopened-hits-pool-coordinator-owner", + .planned_options = .{ + .worker_id = "runtime-reopened-hits-pool-coordinator-unused", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + }); + defer coordinator.close(); + + const tick = try coordinator.graph_metric_runtime.?.runOnceDetailed(); + const stats = coordinator.graphMetricRuntimeStats(); + try std.testing.expect(stats.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.coordinator, stats.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-hits-pool-coordinator-owner"), stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-reopened-hits-pool-coordinator-owner"), stats.owner_id_hash); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expect(!stats.started); + try std.testing.expectEqual(@as(u64, 0), stats.worker_id_hash); + try std.testing.expectEqual(@as(u64, 0), stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), stats.total_worker_steps); + break :blk tick; + }; + coordinator_total.add(coordinator_tick); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + if (authority_status.state == .fresh and + hub_status.state == .fresh and + authority_status.published_generation == target_generation and + hub_status.published_generation == target_generation) + { + fresh = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(fresh); + try std.testing.expect(saw_worker_pool_role); + try std.testing.expect(saw_live_duplicate_worker_pool_fenced); + try std.testing.expect(coordinator_total.builds_started > 0); + try std.testing.expect(coordinator_total.coordinator_steps > 0); + try std.testing.expect(coordinator_total.phases_advanced > 0); + try std.testing.expect(coordinator_total.published > 0); + try std.testing.expectEqual(@as(usize, 0), coordinator_total.worker_steps); + try std.testing.expect(worker_total.worker_steps > 0); + try std.testing.expect(worker_total.pages_completed > 0); + try std.testing.expectEqual(@as(usize, 0), worker_total.coordinator_steps); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .open_mode = .query_readonly, + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var metric_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); + try std.testing.expectEqualStrings("doc:authority", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); + } +} + +test "db graph metric runtime background split ticks survive reopened pagerank handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + // This fixture exercises numerical coordinator/page recovery; cold + // preparation recovery has its own independent-task regression. + try prepareTopologyForRuntimeTest(&db, "pagerank"); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-reopened-coordinator", + .lease_owned = true, + .owner_id = "runtime-reopened-coordinator", + .planned_options = .{ + .worker_id = "runtime-reopened-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const started = try runtime.runCoordinatorOnce(true); + try std.testing.expectEqual(@as(usize, 1), started.builds_started); + try std.testing.expectEqual(@as(usize, 0), started.worker_steps); + try std.testing.expectEqual(@as(usize, 0), started.pages_completed); + { + const stats = runtime.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(Role.coordinator, stats.role); + } + } + + { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = "runtime-reopened-worker-a", + .lease_owned = true, + .owner_id = "runtime-reopened-worker-a", + .planned_options = .{ + .worker_id = "runtime-reopened-worker-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const prepared = try runtime.runWorkerOnce("runtime-reopened-worker-a"); + try std.testing.expectEqual(@as(usize, 1), prepared.worker_steps); + try std.testing.expectEqual(@as(usize, 1), prepared.pages_completed); + try std.testing.expectEqual(@as(usize, 0), prepared.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), prepared.published); + { + const stats = runtime.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(Role.worker, stats.role); + } + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, status.phase); + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-reopened-coordinator", + .lease_owned = true, + .owner_id = "runtime-reopened-coordinator", + .planned_options = .{ + .worker_id = "runtime-reopened-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const advanced = try runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), advanced.builds_started); + try std.testing.expect(advanced.phases_advanced > 0); + { + const stats = runtime.stats(); + try std.testing.expect(stats.lease_owned); + try std.testing.expect(stats.has_lease); + try std.testing.expectEqual(Role.coordinator, stats.role); + } + } + + const workers = [_][]const u8{ "runtime-reopened-worker-a", "runtime-reopened-worker-b" }; + var finished = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker_tick = blk: { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .runtime_id = workers[step_index % workers.len], + .lease_owned = true, + .owner_id = workers[step_index % workers.len], + .planned_options = .{ + .worker_id = workers[step_index % workers.len], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runWorkerOnce(workers[step_index % workers.len]); + }; + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-reopened-coordinator", + .lease_owned = true, + .owner_id = "runtime-reopened-coordinator", + .planned_options = .{ + .worker_id = "runtime-reopened-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + const tick = try runtime.runCoordinatorOnce(false); + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation == target_generation and status.phase == .complete) { + try std.testing.expect(status.iterations_completed > 0); + finished = true; + } + break :blk tick; + }; + + if (finished) break; + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var published_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[0].scores[0].score >= published_result.graph_metric_results[0].scores[1].score); + } +} + +test "db graph metric runtime background reopened coordinators do not duplicate pagerank publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "runtime-publish-race-worker-a", "runtime-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker_tick = blk: { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = workers[step_index % workers.len], + .planned_options = .{ + .worker_id = workers[step_index % workers.len], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runWorkerOnce(workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .lease_owned = true, + .owner_id = "runtime-publish-race-coordinator", + .planned_options = .{ + .worker_id = "runtime-publish-race-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runCoordinatorOnce(false); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const materialize_graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const materialized = try materialize_graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("pagerank", "runtime-publish-race-materializer"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, materialized.phase); + try std.testing.expect(materialized.completed_page); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-publish-race-coordinator-a", + .lease_owned = true, + .owner_id = "runtime-publish-race-coordinator-a", + .planned_options = .{ + .worker_id = "runtime-publish-race-coordinator-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const publish = try runtime.runCoordinatorOnce(false); + try std.testing.expect(publish.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), publish.worker_steps); + const publish_stats = runtime.stats(); + try std.testing.expect(publish_stats.lease_owned); + try std.testing.expect(publish_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, publish_stats.role); + + { + const duplicate_resources = coordinator.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const live_duplicate = try duplicate_runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.published); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.worker_steps); + const live_duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(live_duplicate_stats.lease_owned); + try std.testing.expect(!live_duplicate_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), live_duplicate_stats.lease_acquire_failures); + try std.testing.expectEqual(Role.coordinator, live_duplicate_stats.role); + } + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const resources = duplicate_coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + duplicate_coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const duplicate = try runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), duplicate.published); + try std.testing.expectEqual(@as(usize, 0), duplicate.worker_steps); + const duplicate_stats = runtime.stats(); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(duplicate_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, duplicate_stats.role); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = "runtime-publish-race-cleaner", + .planned_options = .{ + .worker_id = "runtime-publish-race-cleaner", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const cleanup = try runtime.runWorkerOnce("runtime-publish-race-cleaner"); + try std.testing.expectEqual(@as(usize, 0), cleanup.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), cleanup.published); + const graph_entry = worker.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } +} + +test "db graph metric runtime background reopened coordinators do not duplicate eigenvector publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "runtime-eigenvector-publish-race-worker-a", "runtime-eigenvector-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker_tick = blk: { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = workers[step_index % workers.len], + .planned_options = .{ + .worker_id = workers[step_index % workers.len], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runWorkerOnce(workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .lease_owned = true, + .owner_id = "runtime-eigenvector-publish-race-coordinator", + .planned_options = .{ + .worker_id = "runtime-eigenvector-publish-race-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runCoordinatorOnce(false); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-eigenvector-publish-race-coordinator-a", + .lease_owned = true, + .owner_id = "runtime-eigenvector-publish-race-coordinator-a", + .planned_options = .{ + .worker_id = "runtime-eigenvector-publish-race-coordinator-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const materialize_graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const materialized = try materialize_graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("eigenvector", "runtime-eigenvector-publish-race-materializer"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, materialized.phase); + try std.testing.expect(materialized.completed_page); + + const publish = try runtime.runCoordinatorOnce(false); + try std.testing.expect(publish.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), publish.worker_steps); + const publish_stats = runtime.stats(); + try std.testing.expect(publish_stats.lease_owned); + try std.testing.expect(publish_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, publish_stats.role); + + { + const duplicate_resources = coordinator.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-eigenvector-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-eigenvector-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-eigenvector-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const live_duplicate = try duplicate_runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.published); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.worker_steps); + const live_duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(live_duplicate_stats.lease_owned); + try std.testing.expect(!live_duplicate_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), live_duplicate_stats.lease_acquire_failures); + try std.testing.expectEqual(Role.coordinator, live_duplicate_stats.role); + } + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const resources = duplicate_coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + duplicate_coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-eigenvector-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-eigenvector-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-eigenvector-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const duplicate = try runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), duplicate.published); + try std.testing.expectEqual(@as(usize, 0), duplicate.worker_steps); + const duplicate_stats = runtime.stats(); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(duplicate_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, duplicate_stats.role); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = "runtime-eigenvector-publish-race-cleaner", + .planned_options = .{ + .worker_id = "runtime-eigenvector-publish-race-cleaner", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const cleanup = try runtime.runWorkerOnce("runtime-eigenvector-publish-race-cleaner"); + try std.testing.expectEqual(@as(usize, 0), cleanup.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), cleanup.published); + const graph_entry = worker.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } +} + +test "db graph metric runtime background reopened coordinators do not duplicate hits publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub_a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub_b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "runtime-hits-publish-race-worker-a", "runtime-hits-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker_tick = blk: { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = workers[step_index % workers.len], + .planned_options = .{ + .worker_id = workers[step_index % workers.len], + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runWorkerOnce(workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator_tick = blk: { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .lease_owned = true, + .owner_id = "runtime-hits-publish-race-coordinator", + .planned_options = .{ + .worker_id = "runtime-hits-publish-race-coordinator", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + break :blk try runtime.runCoordinatorOnce(false); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker_tick.durableProgressed() and !coordinator_tick.durableProgressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const resources = coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-hits-publish-race-coordinator-a", + .lease_owned = true, + .owner_id = "runtime-hits-publish-race-coordinator-a", + .planned_options = .{ + .worker_id = "runtime-hits-publish-race-coordinator-a", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const materialize_graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const materialized = try materialize_graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("hits_authority", "runtime-hits-publish-race-materializer"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, materialized.phase); + try std.testing.expect(materialized.completed_page); + + const publish = try runtime.runCoordinatorOnce(false); + try std.testing.expect(publish.phases_advanced > 0); + try std.testing.expectEqual(@as(usize, 0), publish.worker_steps); + const publish_stats = runtime.stats(); + try std.testing.expect(publish_stats.lease_owned); + try std.testing.expect(publish_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, publish_stats.role); + + { + const duplicate_resources = coordinator.core.asyncResources(); + var duplicate_runtime = try GraphMetricRuntime.init( + alloc, + duplicate_resources.store, + duplicate_resources.index_manager, + duplicate_resources.apply_mutex, + coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-hits-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-hits-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-hits-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer duplicate_runtime.deinit(); + + const live_duplicate = try duplicate_runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.published); + try std.testing.expectEqual(@as(usize, 0), live_duplicate.worker_steps); + const live_duplicate_stats = duplicate_runtime.stats(); + try std.testing.expect(live_duplicate_stats.lease_owned); + try std.testing.expect(!live_duplicate_stats.has_lease); + try std.testing.expectEqual(@as(u64, 1), live_duplicate_stats.lease_acquire_failures); + try std.testing.expectEqual(Role.coordinator, live_duplicate_stats.role); + } + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const resources = duplicate_coordinator.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + duplicate_coordinator.backend_runtime, + .{ + .enabled = true, + .role = .coordinator, + .runtime_id = "runtime-hits-publish-race-coordinator-b", + .lease_owned = true, + .owner_id = "runtime-hits-publish-race-coordinator-b", + .planned_options = .{ + .worker_id = "runtime-hits-publish-race-coordinator-b", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const duplicate = try runtime.runCoordinatorOnce(false); + try std.testing.expectEqual(@as(usize, 0), duplicate.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), duplicate.published); + try std.testing.expectEqual(@as(usize, 0), duplicate.worker_steps); + const duplicate_stats = runtime.stats(); + try std.testing.expect(duplicate_stats.lease_owned); + try std.testing.expect(duplicate_stats.has_lease); + try std.testing.expectEqual(Role.coordinator, duplicate_stats.role); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } + + { + var worker = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const resources = worker.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + worker.backend_runtime, + .{ + .enabled = true, + .role = .worker, + .lease_owned = true, + .owner_id = "runtime-hits-publish-race-cleaner", + .planned_options = .{ + .worker_id = "runtime-hits-publish-race-cleaner", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + const cleanup = try runtime.runWorkerOnce("runtime-hits-publish-race-cleaner"); + try std.testing.expectEqual(@as(usize, 0), cleanup.phases_advanced); + try std.testing.expectEqual(@as(usize, 0), cleanup.published); + const graph_entry = worker.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + if (authority.state == .fresh and authority.phase == .complete and hub.state == .fresh and hub.phase == .complete) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } +} + +test "db graph metric runtime background cycles multiple worker ids across planned pages" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{.{ .key = "doc:hub", .value = "{\"title\":\"hub\"}" }}, + .sync_level = .write, + }); + + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:{d:0>3}", .{i}); + defer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + defer alloc.free(value); + try db.batch(.{ + .writes = &.{.{ .key = key, .value = value }}, + .sync_level = .write, + }); + } + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + db.core.graphIndex("graph_idx").?.index.test_partition_target_units = 64; + + const workers = [_][]const u8{ "runtime-worker-a", "runtime-worker-b" }; + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .planned_options = .{ + .worker_ids = &workers, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 2, + }, + }, + ); + defer runtime.deinit(); + + const prepare_tick = try runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 1), prepare_tick.builds_started); + try std.testing.expectEqual(@as(usize, 2), prepare_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 1), prepare_tick.pages_completed); + try std.testing.expect(prepare_tick.phases_advanced > 0); + + const scan_tick = try runtime.runOnceDetailed(); + try std.testing.expectEqual(@as(usize, 0), scan_tick.builds_started); + try std.testing.expectEqual(@as(usize, 2), scan_tick.worker_steps); + try std.testing.expectEqual(@as(usize, 2), scan_tick.pages_completed); + try std.testing.expectEqual(@as(usize, 0), scan_tick.phases_advanced); + + var steps: usize = 0; + while (try runtime.runOnce()) { + steps += 1; + if (steps > 200) return error.TestUnexpectedResult; + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:hub", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 130.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime planned scheduler does not auto retry failed graph metric generation" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .planned, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + try prepareTopologyForRuntimeTest(&db, "pagerank"); + const start = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = true, + }); + try std.testing.expectEqual(@as(usize, 1), start.builds_started); + try std.testing.expectEqual(@as(usize, 1), start.active_builds); + + var failed = try db.failGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", error.InvalidGraphMetricBuildManifest); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expect(failed.build_queued); + try std.testing.expectEqualStrings("InvalidGraphMetricBuildManifest", failed.last_error); + const failed_target_generation = failed.target_edge_generation; + + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, false, 0, 0, 0, 0); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + const duplicate = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = true, + }); + try std.testing.expectEqual(@as(usize, 0), duplicate.builds_started); + try std.testing.expectEqual(@as(usize, 0), duplicate.active_builds); + try std.testing.expectEqual(@as(usize, 0), duplicate.coordinator_steps); + try std.testing.expect(!duplicate.durableProgressed()); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqual(failed_target_generation, status.target_edge_generation); + try std.testing.expectEqual(@as(u64, 0), status.build_job_id); + var failed_events: usize = 0; + for (status.recent_events) |event| { + if (event.kind == .failed) failed_events += 1; + } + try std.testing.expectEqual(@as(usize, 1), failed_events); + } + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:e", + .value = "{\"title\":\"epsilon\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + const retry_new_generation = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = true, + }); + try std.testing.expectEqual(@as(usize, 0), retry_new_generation.builds_started); + try std.testing.expect(retry_new_generation.planning_steps > 0); + try prepareTopologyForRuntimeTest(&db, "pagerank"); + const admitted = try db.runGraphMetricPlannedCoordinatorSweep(.{ .max_metrics = 8, .start_background_builds = true }); + try std.testing.expectEqual(@as(usize, 1), admitted.builds_started); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expect(status.building_generation > failed_target_generation); + } +} + +test "db graph metric runtime planned scheduler boundary completes degree by name" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "manual_degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const workers = [_][]const u8{ "worker-a", "worker-b" }; + var step_index: usize = 0; + var published = false; + while (step_index < 1000) : (step_index += 1) { + const now_ms: u64 = 1000 + @as(u64, @intCast(step_index)) * 2; + const worker_step = try db.runGraphMetricPlannedWorkerPageStepAt("graph_idx", "manual_degree", workers[step_index % workers.len], now_ms); + try std.testing.expect(!worker_step.advanced_phase); + if (worker_step.completed_build) { + try std.testing.expect(worker_step.phase == .complete or worker_step.phase == .cleanup_old_generations); + published = true; + break; + } + + const coordinator_step = try db.runGraphMetricPlannedCoordinatorStepAt("graph_idx", "manual_degree", now_ms + 1); + if (coordinator_step.completed_build) { + published = true; + break; + } + + const progressed = + worker_step.claimed_page or + worker_step.completed_page or + coordinator_step.advanced_phase; + if (!progressed and worker_step.phase != .cleanup_old_generations) return error.GraphMetricBuildNoEligiblePage; + } + try std.testing.expect(published); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 3, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:b", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), published_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime planned scheduler sweeps active degree work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const start = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + try std.testing.expectEqual(@as(usize, 1), start.metrics_scanned); + try std.testing.expectEqual(@as(usize, 0), start.builds_started); + try std.testing.expect(start.active_builds > 0); + + const workers = [_][]const u8{ "sweep-worker-a", "sweep-worker-b" }; + var finished = false; + var saw_coordinator_publish = false; + var step_index: usize = 0; + while (step_index < 1000) : (step_index += 1) { + const worker = try db.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = workers[step_index % workers.len], + .max_pages = 1, + }); + try std.testing.expectEqual(@as(usize, 0), worker.published); + const coordinator = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + if (coordinator.published > 0) saw_coordinator_publish = true; + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation != 0 and status.phase == .complete) { + finished = true; + break; + } + } + if (!worker.progressed() and !coordinator.progressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + try std.testing.expect(saw_coordinator_publish); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 3, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:b", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), published_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime planned scheduler sweeps active pagerank work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":20,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const workers = [_][]const u8{ "pagerank-sweep-a", "pagerank-sweep-b", "pagerank-sweep-c" }; + var finished = false; + var step_index: usize = 0; + while (step_index < 2000) : (step_index += 1) { + const worker = try db.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = workers[step_index % workers.len], + .max_pages = 1, + }); + const coordinator = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation != 0 and status.phase == .complete) { + try std.testing.expect(status.iterations_completed > 0); + finished = true; + break; + } + } + if (!worker.progressed() and !coordinator.progressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[0].scores[0].score >= published_result.graph_metric_results[0].scores[1].score); +} + +test "db graph metric runtime planned single-vector failed planned rebuild preserves published public reads" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + try TestHelpers.verifyDbSingleVectorFailedPlannedRebuildPreservesPublishedPublicReads(DB, alloc, "pagerank", "pagerank"); + try TestHelpers.verifyDbSingleVectorFailedPlannedRebuildPreservesPublishedPublicReads(DB, alloc, "eigenvector", "eigenvector"); +} + +test "db graph metric runtime planned paired hits failed planned rebuild preserves published public reads" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "hits_authority"); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + const published_generation = refreshed.published_generation; + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expectEqual(published_generation, hub_status.published_generation); + } + + var initial = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer initial.deinit(); + try std.testing.expectEqual(@as(usize, 2), initial.graph_metric_results.len); + try std.testing.expectEqualStrings("doc:authority", initial.graph_metric_results[0].scores[0].node); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[1].status.state); + try std.testing.expectEqual(published_generation, initial.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(published_generation, initial.graph_metric_results[1].status.published_generation); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:new-hub", .value = "{\"title\":\"new hub\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const rebuilding_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + try std.testing.expect(target_generation > published_generation); + var building = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + break :blk target_generation; + }; + + var failed = try db.failGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, authority_status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, hub_status.state); + try std.testing.expectEqual(published_generation, authority_status.published_generation); + try std.testing.expectEqual(published_generation, hub_status.published_generation); + try std.testing.expectEqual(rebuilding_generation, authority_status.target_edge_generation); + try std.testing.expectEqual(rebuilding_generation, hub_status.target_edge_generation); + } + + var published_after_failure = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 4, + .freshness = .published, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 4, + .freshness = .published, + }, + }, + }, + .limit = 0, + }); + defer published_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 2), published_after_failure.graph_metric_results.len); + for (published_after_failure.graph_metric_results) |result| { + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, result.status.state); + try std.testing.expectEqual(published_generation, result.status.published_generation); + try std.testing.expect(result.scores.len > 0); + for (result.scores) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc:new-hub")); + } + } + try std.testing.expectEqualStrings("doc:authority", published_after_failure.graph_metric_results[0].scores[0].node); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .published }, + .{ .name = "hits_hub", .freshness = .published }, + }; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:hub-a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:authority", traversal_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 2), traversal_after_failure.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("hits_authority", traversal_after_failure.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(traversal_after_failure.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), traversal_after_failure.graph_results[0].nodes[0].metrics[0].score.?, 0.001); + try std.testing.expectEqual(@as(usize, 2), traversal_after_failure.graph_results[0].metric_status.len); + for (traversal_after_failure.graph_results[0].metric_status) |status| { + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, status.state); + try std.testing.expectEqual(published_generation, status.published_generation); + } + + const published_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .published, + }}; + var published_order_query = published_graph_query; + published_order_query.order_by = &published_metric_orders; + var traversal_order_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_order_query }}, + .limit = 0, + }); + defer traversal_order_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_order_after_failure.graph_results.len); + try std.testing.expectEqualStrings("doc:authority", traversal_order_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, traversal_order_after_failure.graph_results[0].metric_status[0].state); + + const published_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.5, + .freshness = .published, + }}; + var published_filter_query = published_graph_query; + published_filter_query.where_metric = &published_metric_filters; + var traversal_filter_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_filter_query }}, + .limit = 0, + }); + defer traversal_filter_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_filter_after_failure.graph_results.len); + try std.testing.expectEqualStrings("doc:authority", traversal_filter_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, traversal_filter_after_failure.graph_results[0].metric_status[0].state); + + var rerank_after_failure = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_after_failure.deinit(); + try std.testing.expectEqual(@as(u32, 4), rerank_after_failure.total_hits); + const rerank_status = rerank_after_failure.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, rerank_status.state); + try std.testing.expectEqual(published_generation, rerank_status.published_generation); + var saw_authority_score = false; + var saw_new_hub_missing_score = false; + for (rerank_after_failure.hits) |hit| { + const details = hit.score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(published_generation, details.published_generation); + if (std.mem.eql(u8, hit.id, "doc:authority")) { + saw_authority_score = details.metric_score != null; + } else if (std.mem.eql(u8, hit.id, "doc:new-hub")) { + saw_new_hub_missing_score = details.metric_score == null; + } + } + try std.testing.expect(saw_authority_score); + try std.testing.expect(saw_new_hub_missing_score); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{ + .{ .name = "hits_authority", .freshness = .fresh }, + .{ .name = "hits_hub", .freshness = .fresh }, + }; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "hits_authority", + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "hits_authority", + .op = .gte, + .value = 0.5, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +test "db graph metric runtime planned scheduler sweeps pagerank across reopened handles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":20,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const initial_tick = try coordinator.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + try std.testing.expect(initial_tick.active_builds > 0); + } + + const workers = [_][]const u8{ "reopened-pagerank-a", "reopened-pagerank-b", "reopened-pagerank-c" }; + var finished = false; + var step_index: usize = 0; + while (step_index < 2000) : (step_index += 1) { + const worker = blk: { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + break :blk try worker_db.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = workers[step_index % workers.len], + .max_pages = 1, + }); + }; + + const coordinator = blk: { + var coordinator_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator_db.close(); + const sweep = try coordinator_db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + { + const graph_entry = coordinator_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation != 0 and status.phase == .complete) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(status.iterations_completed > 0); + finished = true; + } + } + break :blk sweep; + }; + if (finished) break; + if (!worker.progressed() and !coordinator.progressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + var published_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[0].scores[0].score >= published_result.graph_metric_results[0].scores[1].score); + } +} + +test "db graph metric runtime planned scheduler reopened coordinators do not duplicate pagerank publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "db-publish-race-worker-a", "db-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker = blk: { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + break :blk try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "pagerank", workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator = blk: { + var coordinator_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator_db.close(); + break :blk try coordinator_db.runGraphMetricPlannedCoordinatorStep("graph_idx", "pagerank"); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker.claimed_page and !worker.completed_page and !coordinator.advanced_phase) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const materialized = try coordinator.runGraphMetricPlannedWorkerPageStep("graph_idx", "pagerank", "db-publish-race-materializer"); + try std.testing.expect(materialized.completed_page); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "pagerank"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "pagerank"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "pagerank", "db-publish-race-cleaner"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + + var published_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + } +} + +test "db graph metric runtime planned scheduler reopened coordinators do not duplicate eigenvector publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "db-eigenvector-publish-race-worker-a", "db-eigenvector-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker = blk: { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + break :blk try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "eigenvector", workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator = blk: { + var coordinator_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator_db.close(); + break :blk try coordinator_db.runGraphMetricPlannedCoordinatorStep("graph_idx", "eigenvector"); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker.claimed_page and !worker.completed_page and !coordinator.advanced_phase) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const materialized = try coordinator.runGraphMetricPlannedWorkerPageStep("graph_idx", "eigenvector", "db-eigenvector-publish-race-materializer"); + try std.testing.expect(materialized.completed_page); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "eigenvector"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "eigenvector"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + } + + { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "eigenvector", "db-eigenvector-publish-race-cleaner"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(@as(usize, 1), status.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, status.recent_events[0].kind); + + var published_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + } +} + +test "db graph metric runtime planned scheduler reopened coordinators do not duplicate hits publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var target_generation: u64 = 0; + { + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub_a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub_b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + target_generation = graph_entry.index.edge_generation; + } + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + var started = try coordinator.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + } + + const workers = [_][]const u8{ "db-hits-publish-race-worker-a", "db-hits-publish-race-worker-b" }; + var reached_publish = false; + var step_index: usize = 0; + while (step_index < 400) : (step_index += 1) { + const worker = blk: { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + break :blk try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "hits_authority", workers[step_index % workers.len]); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + const coordinator = blk: { + var coordinator_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator_db.close(); + break :blk try coordinator_db.runGraphMetricPlannedCoordinatorStep("graph_idx", "hits_authority"); + }; + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer status.deinit(alloc); + if (status.phase == .publish_generation) { + reached_publish = true; + break; + } + } + + if (!worker.claimed_page and !worker.completed_page and !coordinator.advanced_phase) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(reached_publish); + + { + var coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer coordinator.close(); + + const materialized = try coordinator.runGraphMetricPlannedWorkerPageStep("graph_idx", "hits_authority", "db-hits-publish-race-materializer"); + try std.testing.expect(materialized.completed_page); + const publish = try coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "hits_authority"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.publish_generation, publish.phase); + try std.testing.expect(publish.advanced_phase); + + const graph_entry = coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } + + { + var duplicate_coordinator = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer duplicate_coordinator.close(); + + const duplicate = try duplicate_coordinator.runGraphMetricPlannedCoordinatorStep("graph_idx", "hits_authority"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, duplicate.phase); + try std.testing.expect(!duplicate.advanced_phase); + try std.testing.expect(!duplicate.published); + + const graph_entry = duplicate_coordinator.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + } + + { + var worker_db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer worker_db.close(); + + var cleanup_finished = false; + for (0..12) |_| { + const cleanup = try worker_db.runGraphMetricPlannedWorkerPageStep("graph_idx", "hits_authority", "db-hits-publish-race-cleaner"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.cleanup_old_generations, cleanup.phase); + try std.testing.expect(!cleanup.advanced_phase); + if (cleanup.published) { + cleanup_finished = true; + break; + } + } + try std.testing.expect(cleanup_finished); + } + + { + var reader = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer reader.close(); + + const graph_entry = reader.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority.deinit(alloc); + var hub = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, authority.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, authority.phase); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, hub.phase); + try std.testing.expectEqual(target_generation, authority.published_generation); + try std.testing.expectEqual(authority.published_generation, hub.published_generation); + try std.testing.expectEqual(@as(usize, 1), authority.recent_events.len); + try std.testing.expectEqual(@as(usize, 1), hub.recent_events.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, authority.recent_events[0].kind); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricEventKind.publish, hub.recent_events[0].kind); + + var authority_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer authority_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), authority_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, authority_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, authority_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), authority_result.graph_metric_results[0].scores.len); + + var hub_result = try reader.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer hub_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), hub_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, hub_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), hub_result.graph_metric_results[0].scores.len); + } +} + +test "db graph metric runtime planned maintenance drains background pagerank work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + break :blk graph_entry.index.edge_generation; + }; + + const maintenance = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "planned-maintenance-pagerank", + .max_rounds = 200, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expectEqual(@as(usize, 1), maintenance.builds_started); + try std.testing.expect(maintenance.pages_completed > 0); + try std.testing.expect(maintenance.phases_advanced > 0); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[0].scores[0].score >= published_result.graph_metric_results[0].scores[1].score); +} + +test "db graph metric runtime background drains pagerank through planned maintenance" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + break :blk graph_entry.index.edge_generation; + }; + + const resources = db.core.asyncResources(); + var runtime = try GraphMetricRuntime.init( + alloc, + resources.store, + resources.index_manager, + resources.apply_mutex, + db.backend_runtime, + .{ + .enabled = true, + .runtime_id = "runtime-pagerank-combined", + .planned_options = .{ + .worker_id = "runtime-pagerank-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }, + ); + defer runtime.deinit(); + + { + const initial_stats = runtime.stats(); + try std.testing.expect(initial_stats.enabled); + try std.testing.expectEqual(Role.combined, initial_stats.role); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-pagerank-combined"), initial_stats.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "local"), initial_stats.owner_id_hash); + try std.testing.expect(initial_stats.lease_key_hash != 0); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-pagerank-worker"), initial_stats.worker_id_hash); + try std.testing.expectEqual(@as(usize, 1), initial_stats.worker_count); + try std.testing.expectEqual(@as(u64, 0), initial_stats.ticks_started); + try std.testing.expectEqual(@as(u64, 0), initial_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, 0), initial_stats.durable_progress_ticks); + try std.testing.expectEqual(@as(?[]const u8, null), initial_stats.last_error_name); + } + + var steps: usize = 0; + while (try runtime.runOnce()) { + steps += 1; + if (steps > 200) return error.TestUnexpectedResult; + } + try std.testing.expect(steps > 0); + + { + const drained_stats = runtime.stats(); + try std.testing.expectEqual(@as(u64, @intCast(steps + 1)), drained_stats.ticks_started); + try std.testing.expectEqual(drained_stats.ticks_started, drained_stats.ticks_completed); + try std.testing.expectEqual(@as(u64, @intCast(steps)), drained_stats.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), drained_stats.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), drained_stats.error_ticks); + try std.testing.expectEqual(@as(?[]const u8, null), drained_stats.last_error_name); + try std.testing.expect(!drained_stats.last_result.durableProgressed()); + } + + db.graph_metric_runtime = &runtime; + defer db.graph_metric_runtime = null; + { + const mapped_stats = try db.stats(alloc); + defer types.freeDBStats(alloc, mapped_stats); + try std.testing.expect(mapped_stats.graph_metric_runtime.enabled); + try std.testing.expectEqual(types.GraphMetricRuntimeRole.combined, mapped_stats.graph_metric_runtime.role.?); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-pagerank-combined"), mapped_stats.graph_metric_runtime.runtime_id_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "local"), mapped_stats.graph_metric_runtime.owner_id_hash); + try std.testing.expectEqual(runtime.stats().lease_key_hash, mapped_stats.graph_metric_runtime.lease_key_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, "runtime-pagerank-worker"), mapped_stats.graph_metric_runtime.worker_id_hash); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.worker_count); + try std.testing.expectEqual(@as(u64, @intCast(steps + 1)), mapped_stats.graph_metric_runtime.ticks_started); + try std.testing.expectEqual(mapped_stats.graph_metric_runtime.ticks_started, mapped_stats.graph_metric_runtime.ticks_completed); + try std.testing.expectEqual(@as(u64, @intCast(steps)), mapped_stats.graph_metric_runtime.durable_progress_ticks); + try std.testing.expectEqual(@as(u64, 1), mapped_stats.graph_metric_runtime.idle_ticks); + try std.testing.expectEqual(@as(u64, 0), mapped_stats.graph_metric_runtime.error_ticks); + try std.testing.expectEqual(@as(u64, 0), mapped_stats.graph_metric_runtime.last_builds_started); + try std.testing.expect(!mapped_stats.graph_metric_runtime.last_budget_exhausted); + } + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + } + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime planned maintenance reports budget exhaustion and resumes" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + const first_tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "planned-maintenance-budgeted", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(first_tick.budget_exhausted); + try std.testing.expect(first_tick.durableProgressed()); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expect(status.state != .fresh or status.phase != .complete); + } + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var finished = false; + var saw_budget_exhausted = first_tick.budget_exhausted; + var tick_index: usize = 0; + while (tick_index < 200) : (tick_index += 1) { + const tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "planned-maintenance-budgeted", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + saw_budget_exhausted = saw_budget_exhausted or tick.budget_exhausted; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expect(status.iterations_completed > 0); + finished = true; + break; + } + if (!tick.progressed()) return error.GraphMetricBuildNoEligiblePage; + } + try std.testing.expect(saw_budget_exhausted); + try std.testing.expect(finished); + + // The numerical job is complete, but its topology pin can still require a + // bounded reclamation checkpoint. Drain that independent lifecycle too. + var maintenance_idle = false; + for (0..8) |_| { + const no_work = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "planned-maintenance-budgeted", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(no_work.worker_steps <= 1); + if (!no_work.durableProgressed()) { + try std.testing.expect(!no_work.budget_exhausted); + maintenance_idle = true; + break; + } + } + try std.testing.expect(maintenance_idle); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:d", published_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime planned pagerank production budget matches local oracle" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank_local\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"pagerank_planned\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + var writes = std.ArrayListUnmanaged(types.BatchWrite).empty; + defer { + for (writes.items) |write| { + alloc.free(write.key); + alloc.free(write.value); + } + writes.deinit(alloc); + } + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:src-{d:0>3}", .{i}); + errdefer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + errdefer alloc.free(value); + try writes.append(alloc, .{ .key = key, .value = value }); + } + const hub_key = try alloc.dupe(u8, "doc:hub"); + errdefer alloc.free(hub_key); + const hub_value = try alloc.dupe(u8, "{\"title\":\"hub\"}"); + errdefer alloc.free(hub_value); + try writes.append(alloc, .{ .key = hub_key, .value = hub_value }); + + try db.batch(.{ + .writes = writes.items, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var local_refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "pagerank_local"); + defer local_refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, local_refreshed.state); + try std.testing.expectEqual(target_generation, local_refreshed.published_generation); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + const first_tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(first_tick.budget_exhausted); + try std.testing.expect(first_tick.durableProgressed()); + try std.testing.expect(first_tick.rounds_executed <= 1); + + var total = first_tick; + var finished = false; + var saw_budget_exhausted = first_tick.budget_exhausted; + var tick_index: usize = 0; + while (tick_index < 400) : (tick_index += 1) { + const tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + total.add(tick); + saw_budget_exhausted = saw_budget_exhausted or tick.budget_exhausted; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank_planned"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + try std.testing.expectEqual(target_generation, status.published_generation); + finished = true; + break; + } + if (!tick.progressed()) return error.GraphMetricBuildNoEligiblePage; + } + try std.testing.expect(finished); + try std.testing.expect(saw_budget_exhausted); + try std.testing.expect(total.rounds_executed > 1); + try std.testing.expect(total.pages_completed > 1); + try std.testing.expect(total.phases_advanced > 0); + try std.testing.expect(total.published > 0); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var local_status = try graph_entry.index.graphMetricStatus("pagerank_local"); + defer local_status.deinit(alloc); + var planned_status = try graph_entry.index.graphMetricStatus("pagerank_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + + const top_limit: usize = 32; + const local_top = try graph_entry.index.graphMetricTopK("pagerank_local", top_limit); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph_entry.index.graphMetricTopK("pagerank_planned", top_limit); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + try std.testing.expectEqualStrings("doc:hub", planned_top[0].node); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "db graph metric runtime planned eigenvector production budget matches local oracle" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector_local\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"manual\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"eigenvector_planned\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + var writes = std.ArrayListUnmanaged(types.BatchWrite).empty; + defer { + for (writes.items) |write| { + alloc.free(write.key); + alloc.free(write.value); + } + writes.deinit(alloc); + } + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:src-{d:0>3}", .{i}); + errdefer alloc.free(key); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"source {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:hub\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + errdefer alloc.free(value); + try writes.append(alloc, .{ .key = key, .value = value }); + } + const hub_key = try alloc.dupe(u8, "doc:hub"); + errdefer alloc.free(hub_key); + const hub_value = try alloc.dupe(u8, "{\"title\":\"hub\"}"); + errdefer alloc.free(hub_value); + try writes.append(alloc, .{ .key = hub_key, .value = hub_value }); + + try db.batch(.{ + .writes = writes.items, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var local_refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "eigenvector_local"); + defer local_refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, local_refreshed.state); + try std.testing.expectEqual(target_generation, local_refreshed.published_generation); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + const first_tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(first_tick.budget_exhausted); + try std.testing.expect(first_tick.durableProgressed()); + try std.testing.expect(first_tick.rounds_executed <= 1); + + var total = first_tick; + var finished = false; + var saw_budget_exhausted = first_tick.budget_exhausted; + var tick_index: usize = 0; + while (tick_index < 600) : (tick_index += 1) { + const tick = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + total.add(tick); + saw_budget_exhausted = saw_budget_exhausted or tick.budget_exhausted; + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector_planned"); + defer status.deinit(alloc); + if (status.state == .fresh and status.phase == .complete) { + try std.testing.expectEqual(target_generation, status.published_generation); + finished = true; + break; + } + if (!tick.progressed()) return error.GraphMetricBuildNoEligiblePage; + } + try std.testing.expect(finished); + try std.testing.expect(saw_budget_exhausted); + try std.testing.expect(total.rounds_executed > 1); + try std.testing.expect(total.pages_completed > 1); + try std.testing.expect(total.phases_advanced > 0); + try std.testing.expect(total.published > 0); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var local_status = try graph_entry.index.graphMetricStatus("eigenvector_local"); + defer local_status.deinit(alloc); + var planned_status = try graph_entry.index.graphMetricStatus("eigenvector_planned"); + defer planned_status.deinit(alloc); + try std.testing.expectEqual(local_status.published_generation, planned_status.published_generation); + try std.testing.expectEqual(local_status.iterations_completed, planned_status.iterations_completed); + try std.testing.expectEqual(local_status.converged, planned_status.converged); + try std.testing.expectApproxEqAbs(local_status.delta, planned_status.delta, 0.0000001); + + const top_limit: usize = 32; + const local_top = try graph_entry.index.graphMetricTopK("eigenvector_local", top_limit); + defer { + for (local_top) |*score| score.deinit(alloc); + alloc.free(local_top); + } + const planned_top = try graph_entry.index.graphMetricTopK("eigenvector_planned", top_limit); + defer { + for (planned_top) |*score| score.deinit(alloc); + alloc.free(planned_top); + } + try std.testing.expectEqual(local_top.len, planned_top.len); + for (local_top, planned_top) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expect(std.math.isFinite(local.score)); + try std.testing.expect(std.math.isFinite(planned.score)); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "db graph metric runtime planned hits production budget matches local oracle" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var local_path_buf: [256]u8 = undefined; + const local_path = TestHelpers.tempPath(&local_path_buf); + defer TestHelpers.cleanupTempDir(local_path); + var planned_path_buf: [256]u8 = undefined; + const planned_path = TestHelpers.tempPath(&planned_path_buf); + defer TestHelpers.cleanupTempDir(planned_path); + + var local_db = try DB.open(alloc, std.mem.span(local_path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer local_db.close(); + var planned_db = try DB.open(alloc, std.mem.span(planned_path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer planned_db.close(); + + const config_json = + "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}"; + try local_db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = config_json, + }); + try planned_db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = config_json, + }); + + var writes = std.ArrayListUnmanaged(types.BatchWrite).empty; + defer { + for (writes.items) |write| { + alloc.free(write.key); + alloc.free(write.value); + } + writes.deinit(alloc); + } + for (0..130) |i| { + const key = try std.fmt.allocPrint(alloc, "doc:hub-{d:0>3}", .{i}); + const value = try std.fmt.allocPrint( + alloc, + "{{\"title\":\"hub {d}\",\"_edges\":{{\"graph_idx\":{{\"cites\":[{{\"target\":\"doc:authority\",\"weight\":1.0}}]}}}}}}", + .{i}, + ); + writes.append(alloc, .{ .key = key, .value = value }) catch |err| { + alloc.free(key); + alloc.free(value); + return err; + }; + } + const authority_key = try alloc.dupe(u8, "doc:authority"); + const authority_value = try alloc.dupe(u8, "{\"title\":\"authority\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}"); + writes.append(alloc, .{ .key = authority_key, .value = authority_value }) catch |err| { + alloc.free(authority_key); + alloc.free(authority_value); + return err; + }; + + try local_db.batch(.{ + .writes = writes.items, + .sync_level = .write, + }); + try planned_db.batch(.{ + .writes = writes.items, + .sync_level = .write, + }); + try local_db.runDerivedUntil(local_db.core.nextDerivedSequence()); + try planned_db.runDerivedUntil(planned_db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = planned_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + + var local_authority_refreshed = try local_db.refreshGraphMetric(alloc, "graph_idx", "hits_authority"); + defer local_authority_refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, local_authority_refreshed.state); + try std.testing.expectEqual(target_generation, local_authority_refreshed.published_generation); + + { + const pending = planned_db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + const first_tick = try planned_db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(first_tick.budget_exhausted); + try std.testing.expect(first_tick.durableProgressed()); + try std.testing.expect(first_tick.rounds_executed <= 1); + + var total = first_tick; + var finished = false; + var saw_budget_exhausted = first_tick.budget_exhausted; + var tick_index: usize = 0; + while (tick_index < 800) : (tick_index += 1) { + const tick = try planned_db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_ids = &.{ "budget-worker-a", "budget-worker-b" }, + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + total.add(tick); + saw_budget_exhausted = saw_budget_exhausted or tick.budget_exhausted; + const graph_entry = planned_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + if (authority_status.state == .fresh and authority_status.phase == .complete and hub_status.state == .fresh and hub_status.phase == .complete) { + try std.testing.expectEqual(target_generation, authority_status.published_generation); + try std.testing.expectEqual(authority_status.published_generation, hub_status.published_generation); + finished = true; + break; + } + if (!tick.progressed()) return error.GraphMetricBuildNoEligiblePage; + } + try std.testing.expect(finished); + try std.testing.expect(saw_budget_exhausted); + try std.testing.expect(total.rounds_executed > 1); + try std.testing.expect(total.pages_completed > 1); + try std.testing.expect(total.phases_advanced > 0); + try std.testing.expect(total.published > 0); + + const local_graph = local_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const planned_graph = planned_db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var local_authority_status = try local_graph.index.graphMetricStatus("hits_authority"); + defer local_authority_status.deinit(alloc); + var local_hub_status = try local_graph.index.graphMetricStatus("hits_hub"); + defer local_hub_status.deinit(alloc); + var planned_authority_status = try planned_graph.index.graphMetricStatus("hits_authority"); + defer planned_authority_status.deinit(alloc); + var planned_hub_status = try planned_graph.index.graphMetricStatus("hits_hub"); + defer planned_hub_status.deinit(alloc); + try std.testing.expectEqual(local_authority_status.published_generation, planned_authority_status.published_generation); + try std.testing.expectEqual(local_hub_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(planned_authority_status.published_generation, planned_hub_status.published_generation); + try std.testing.expectEqual(local_authority_status.iterations_completed, planned_authority_status.iterations_completed); + try std.testing.expectEqual(local_hub_status.iterations_completed, planned_hub_status.iterations_completed); + try std.testing.expectEqual(local_authority_status.converged, planned_authority_status.converged); + try std.testing.expectEqual(local_hub_status.converged, planned_hub_status.converged); + try std.testing.expectApproxEqAbs(local_authority_status.delta, planned_authority_status.delta, 0.0000001); + try std.testing.expectApproxEqAbs(local_hub_status.delta, planned_hub_status.delta, 0.0000001); + + const local_authorities = try local_graph.index.graphMetricTopK("hits_authority", 32); + defer { + for (local_authorities) |*score| score.deinit(alloc); + alloc.free(local_authorities); + } + const planned_authorities = try planned_graph.index.graphMetricTopK("hits_authority", 32); + defer { + for (planned_authorities) |*score| score.deinit(alloc); + alloc.free(planned_authorities); + } + try std.testing.expectEqual(local_authorities.len, planned_authorities.len); + try std.testing.expectEqualStrings("doc:authority", planned_authorities[0].node); + for (local_authorities, planned_authorities) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expect(std.math.isFinite(local.score)); + try std.testing.expect(std.math.isFinite(planned.score)); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } + + const local_hubs = try local_graph.index.graphMetricTopK("hits_hub", 32); + defer { + for (local_hubs) |*score| score.deinit(alloc); + alloc.free(local_hubs); + } + const planned_hubs = try planned_graph.index.graphMetricTopK("hits_hub", 32); + defer { + for (planned_hubs) |*score| score.deinit(alloc); + alloc.free(planned_hubs); + } + try std.testing.expectEqual(local_hubs.len, planned_hubs.len); + for (local_hubs, planned_hubs) |local, planned| { + try std.testing.expectEqualStrings(local.node, planned.node); + try std.testing.expect(std.math.isFinite(local.score)); + try std.testing.expect(std.math.isFinite(planned.score)); + try std.testing.expectApproxEqAbs(local.score, planned.score, 0.0000001); + } +} + +test "db graph metric runtime planned maintenance drains background centrality family work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const metric_names = [_][]const u8{ "degree", "eigenvector", "hits_authority", "hits_hub" }; + for (metric_names) |metric_name| { + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + } + break :blk graph_entry.index.edge_generation; + }; + + const maintenance = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "planned-maintenance-centrality", + .max_rounds = 400, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expectEqual(@as(usize, 3), maintenance.builds_started); + try std.testing.expect(maintenance.pages_completed > 0); + try std.testing.expect(maintenance.phases_advanced > 0); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const metric_names = [_][]const u8{ "degree", "eigenvector", "hits_authority", "hits_hub" }; + var authority_generation: u64 = 0; + for (metric_names) |metric_name| { + var status = try graph_entry.index.graphMetricStatus(metric_name); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expectEqual(target_generation, status.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.complete, status.phase); + try std.testing.expect(status.iterations_completed > 0); + if (std.mem.eql(u8, metric_name, "hits_authority")) { + authority_generation = status.published_generation; + } else if (std.mem.eql(u8, metric_name, "hits_hub")) { + try std.testing.expectEqual(authority_generation, status.published_generation); + } + } + } + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results.len); + try std.testing.expectEqualStrings("doc:authority", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expectEqualStrings("doc:authority", published_result.graph_metric_results[1].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[2].scores[0].score >= published_result.graph_metric_results[2].scores[1].score); +} + +test "db graph metric runtime planned scheduler sweeps active eigenvector work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"manual\",\"max_iterations\":20,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const workers = [_][]const u8{ "eigenvector-sweep-a", "eigenvector-sweep-b", "eigenvector-sweep-c" }; + var finished = false; + var step_index: usize = 0; + while (step_index < 2000) : (step_index += 1) { + const worker = try db.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = workers[step_index % workers.len], + .max_pages = 1, + }); + const coordinator = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer status.deinit(alloc); + if (status.state == .fresh and status.published_generation != 0 and status.phase == .complete) { + try std.testing.expect(status.iterations_completed > 0); + finished = true; + break; + } + } + if (!worker.progressed() and !coordinator.progressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 3, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:b", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expect(published_result.graph_metric_results[0].scores[0].score > published_result.graph_metric_results[0].scores[1].score); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), published_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime planned scheduler sweeps active hits work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + + const workers = [_][]const u8{ "hits-sweep-a", "hits-sweep-b", "hits-sweep-c" }; + var finished = false; + var step_index: usize = 0; + while (step_index < 2000) : (step_index += 1) { + const worker = try db.runGraphMetricPlannedWorkerSweep(.{ + .worker_id = workers[step_index % workers.len], + .max_pages = 1, + }); + const coordinator = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = false, + }); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + if (authority_status.state == .fresh and authority_status.published_generation != 0 and authority_status.phase == .complete) { + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expectEqual(authority_status.published_generation, hub_status.published_generation); + try std.testing.expectEqual(authority_status.iterations_completed, hub_status.iterations_completed); + try std.testing.expect(authority_status.iterations_completed > 0); + finished = true; + break; + } + } + if (!worker.progressed() and !coordinator.progressed()) { + return error.GraphMetricBuildNoEligiblePage; + } + } + try std.testing.expect(finished); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, published_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(published_result.graph_metric_results[0].status.published_generation, published_result.graph_metric_results[1].status.published_generation); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results[0].scores.len); + try std.testing.expectEqual(@as(usize, 3), published_result.graph_metric_results[1].scores.len); + try std.testing.expectEqualStrings("doc:authority", published_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), published_result.graph_metric_results[0].scores[0].score, 0.001); + try std.testing.expect(published_result.graph_metric_results[1].scores[0].score >= published_result.graph_metric_results[1].scores[1].score); + try std.testing.expect(published_result.graph_metric_results[1].scores[1].score > published_result.graph_metric_results[1].scores[2].score); +} + +test "db graph metric runtime query public reads fail not ready before first publish" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .full_index, + }); + try db.runUntilIdle(); + + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + })); + + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 10, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .published, + .weight = 1.0, + }, + .limit = 2, + .include_stored = false, + })); + + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 2, + .include_stored = false, + })); +} + +test "db graph metric runtime query freshness distinguishes published stale scores from fresh requirement" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + var published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(refreshed.published_generation, published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), published_result.graph_metric_results[0].scores.len); + for (published_result.graph_metric_results[0].scores) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc:c")); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.001); + } + + const active_target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + var building = try graph_entry.index.ensureGraphMetricPlannedBuild("manual_degree", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric("manual_degree"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + break :blk target_generation; + }; + + var building_published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer building_published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), building_published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building_published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(refreshed.published_generation, building_published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(active_target_generation, building_published_result.graph_metric_results[0].status.building_generation); + try std.testing.expectEqual(@as(usize, 2), building_published_result.graph_metric_results[0].scores.len); + for (building_published_result.graph_metric_results[0].scores) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc:c")); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.001); + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var failed = try graph_entry.index.failGraphMetricPlannedBuild("manual_degree", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(refreshed.published_generation, failed.published_generation); + } + + var failed_published_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 10, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer failed_published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), failed_published_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed_published_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(refreshed.published_generation, failed_published_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), failed_published_result.graph_metric_results[0].scores.len); + for (failed_published_result.graph_metric_results[0].scores) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc:c")); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), score.score, 0.001); + } + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); +} + +test "db graph metric runtime query rerank applies published metric scores to search hits" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .full_index, + }); + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + + var result = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 3, + .include_stored = false, + }); + defer result.deinit(); + + try std.testing.expectEqual(@as(u32, 3), result.total_hits); + try std.testing.expectEqual(@as(usize, 3), result.hits.len); + try std.testing.expectEqualStrings("doc:b", result.hits[0].id); + try std.testing.expectApproxEqAbs(@as(f32, 3.0), result.hits[0].score orelse return error.TestUnexpectedResult, 0.001); + try std.testing.expectApproxEqAbs(@as(f32, 2.0), result.hits[1].score orelse return error.TestUnexpectedResult, 0.001); + try std.testing.expectApproxEqAbs(@as(f32, 2.0), result.hits[2].score orelse return error.TestUnexpectedResult, 0.001); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:d", .value = "{\"title\":\"delta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .full_index, + }); + try db.runUntilIdle(); + + const active_target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + var building = try graph_entry.index.ensureGraphMetricPlannedBuild("manual_degree", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric("manual_degree"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + + var active_status = try graph_entry.index.graphMetricStatus("manual_degree"); + defer active_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, active_status.state); + try std.testing.expectEqual(refreshed.published_generation, active_status.published_generation); + try std.testing.expectEqual(target_generation, active_status.building_generation); + break :blk target_generation; + }; + + var stale_ok = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer stale_ok.deinit(); + try std.testing.expectEqualStrings("doc:b", stale_ok.hits[0].id); + try std.testing.expectApproxEqAbs(@as(f32, 3.0), stale_ok.hits[0].score orelse return error.TestUnexpectedResult, 0.001); + const stale_ok_status = stale_ok.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, stale_ok_status.state); + try std.testing.expectEqual(refreshed.published_generation, stale_ok_status.published_generation); + try std.testing.expectEqual(active_target_generation, stale_ok_status.building_generation); + + var explicit_expression = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .published, + .base_weight = 0.0, + .weight = 2.0, + .missing_score = -10.0, + }, + .limit = 4, + .include_stored = false, + }); + defer explicit_expression.deinit(); + try std.testing.expectEqualStrings("doc:b", explicit_expression.hits[0].id); + try std.testing.expectApproxEqAbs(@as(f32, 4.0), explicit_expression.hits[0].score orelse return error.TestUnexpectedResult, 0.001); + const top_details = explicit_expression.hits[0].score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("graph_idx", top_details.index_name); + try std.testing.expectEqualStrings("manual_degree", top_details.metric_name); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), top_details.base_score, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 0.0), top_details.base_weight, 0.001); + try std.testing.expect(top_details.metric_score != null); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top_details.metric_score.?, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top_details.metric_score_used, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top_details.metric_weight, 0.001); + try std.testing.expect(!top_details.missing_score_used); + try std.testing.expectApproxEqAbs(@as(f64, 4.0), top_details.final_score, 0.001); + try std.testing.expectEqual(refreshed.published_generation, top_details.published_generation); + try std.testing.expectEqualStrings("doc:d", explicit_expression.hits[3].id); + try std.testing.expectApproxEqAbs(@as(f32, -20.0), explicit_expression.hits[3].score orelse return error.TestUnexpectedResult, 0.001); + const missing_details = explicit_expression.hits[3].score_details orelse return error.TestUnexpectedResult; + try std.testing.expect(missing_details.metric_score == null); + try std.testing.expect(missing_details.missing_score_used); + try std.testing.expectApproxEqAbs(@as(f64, -10.0), missing_details.metric_score_used, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, -20.0), missing_details.final_score, 0.001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var failed = try graph_entry.index.failGraphMetricPlannedBuild("manual_degree", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(refreshed.published_generation, failed.published_generation); + } + + var failed_rerank = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer failed_rerank.deinit(); + try std.testing.expectEqualStrings("doc:b", failed_rerank.hits[0].id); + try std.testing.expectApproxEqAbs(@as(f32, 3.0), failed_rerank.hits[0].score orelse return error.TestUnexpectedResult, 0.001); + const failed_rerank_status = failed_rerank.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed_rerank_status.state); + try std.testing.expectEqual(refreshed.published_generation, failed_rerank_status.published_generation); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "manual_degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +test "db graph metric runtime query not ready semantics distinguish projection from ranking" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "manual_degree", + .freshness = .published, + }}; + const base_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var projection_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = base_query }}, + .limit = 0, + }); + defer projection_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), projection_result.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), projection_result.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", projection_result.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), projection_result.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("manual_degree", projection_result.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(projection_result.graph_results[0].nodes[0].metrics[0].score == null); + try std.testing.expectEqual(@as(usize, 1), projection_result.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, projection_result.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(@as(u64, 0), projection_result.graph_results[0].metric_status[0].published_generation); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "manual_degree", + .freshness = .fresh, + }}; + var fresh_projection_query = base_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const published_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "manual_degree", + .freshness = .published, + }}; + var order_query = base_query; + order_query.order_by = &published_metric_orders; + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = order_query }}, + .limit = 0, + })); + + const published_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "manual_degree", + .op = .gte, + .value = 0.5, + .freshness = .published, + }}; + var filter_query = base_query; + filter_query.where_metric = &published_metric_filters; + try std.testing.expectError(error.MetricNotReady, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = filter_query }}, + .limit = 0, + })); +} + +test "db graph metric runtime query freshness distinguishes stale projection from fresh ordering and filtering" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "manual_degree", + .freshness = .published, + }}; + const published_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var published_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_query }}, + .limit = 0, + }); + defer published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", published_result.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("manual_degree", published_result.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(published_result.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), published_result.graph_results[0].nodes[0].metrics[0].score.?, 0.001); + try std.testing.expectEqual(@as(usize, 1), published_result.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, published_result.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(refreshed.published_generation, published_result.graph_results[0].metric_status[0].published_generation); + + const active_target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + var building = try graph_entry.index.ensureGraphMetricPlannedBuild("manual_degree", target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + + const prepare = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-prepare"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, prepare.phase); + try std.testing.expect(prepare.claimed_page); + try std.testing.expect(prepare.completed_page); + + const advance_prepare = try graph_entry.index.runGraphMetricPlannedCoordinatorStepForMetric("manual_degree"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.prepare_generation, advance_prepare.phase); + try std.testing.expect(advance_prepare.advanced_phase); + + const scan = try graph_entry.index.runGraphMetricPlannedWorkerPageStepForMetric("manual_degree", "worker-scan"); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricBuildPhase.scan_edges_and_out_degree, scan.phase); + try std.testing.expect(scan.claimed_page); + try std.testing.expect(scan.completed_page); + break :blk target_generation; + }; + + var building_published_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_query }}, + .limit = 0, + }); + defer building_published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), building_published_result.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), building_published_result.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", building_published_result.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), building_published_result.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("manual_degree", building_published_result.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(building_published_result.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), building_published_result.graph_results[0].nodes[0].metrics[0].score.?, 0.001); + try std.testing.expectEqual(@as(usize, 1), building_published_result.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building_published_result.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(refreshed.published_generation, building_published_result.graph_results[0].metric_status[0].published_generation); + try std.testing.expectEqual(active_target_generation, building_published_result.graph_results[0].metric_status[0].building_generation); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "manual_degree", + .freshness = .fresh, + }}; + var fresh_projection_query = published_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "manual_degree", + .freshness = .fresh, + }}; + var fresh_order_query = published_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = "manual_degree", + .op = .gte, + .value = 0.5, + .freshness = .fresh, + }}; + var fresh_filter_query = published_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var failed = try graph_entry.index.failGraphMetricPlannedBuild("manual_degree", error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(refreshed.published_generation, failed.published_generation); + } + + var failed_published_result = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_query }}, + .limit = 0, + }); + defer failed_published_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), failed_published_result.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), failed_published_result.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", failed_published_result.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), failed_published_result.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("manual_degree", failed_published_result.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(failed_published_result.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), failed_published_result.graph_results[0].nodes[0].metrics[0].score.?, 0.001); + try std.testing.expectEqual(@as(usize, 1), failed_published_result.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed_published_result.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(refreshed.published_generation, failed_published_result.graph_results[0].metric_status[0].published_generation); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); +} + +test "db graph metric runtime degree canary gate tracks queued active and capped degree work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "degree-canary-worker", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, true, 0, 1, 0, 0); + const queued_decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(.{}); + try std.testing.expectEqual(@as(usize, 0), queued_decision.control_records); + try std.testing.expect(queued_decision.queued_degree_control_records > 0); + const capped_queued_decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(.{ + .max_control_records = queued_decision.queued_degree_control_records - 1, + }); + try std.testing.expect(!capped_queued_decision.shouldRunPlanned()); + try std.testing.expectEqual(queued_decision.queued_degree_control_records, capped_queued_decision.queued_degree_control_records); + + const start = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = true, + }); + try std.testing.expectEqual(@as(usize, 1), start.builds_started); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, true, 1, 0, 0, 0); + + const active_decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(.{}); + try std.testing.expect(active_decision.control_records > 0); + try std.testing.expect(active_decision.shouldRunPlanned()); + + const capped_decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(.{ + .max_control_records = active_decision.control_records - 1, + }); + try std.testing.expect(!capped_decision.shouldRunPlanned()); + try std.testing.expectEqual(active_decision.control_records, capped_decision.control_records); + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, false, 0, 0, 0, 0); +} + +test "db graph metric runtime degree canary gate blocks non degree queued work" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, false, 0, 1, 0, 1); + + const degree_canary = try db.core.index_manager.shouldRunGraphMetricDegreeCanary(.{}); + try std.testing.expect(!degree_canary); +} + +test "db graph metric runtime degree canary runUntilIdle uses planned maintenance for one degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .degree_canary, + .graph_metric_idle_planned_options = .{ + .worker_id = "degree-canary-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime degree canary planned maintenance reports bounded rounds and resumes" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .degree_canary, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(db.graph_metric_idle_degree_canary_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued_degree); + + const budgeted = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "degree-canary-round-budget", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expectEqual(@as(usize, 1), budgeted.rounds_executed); + try std.testing.expect(budgeted.budget_exhausted); + try std.testing.expect(budgeted.durableProgressed()); + + const resumed = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "degree-canary-round-budget", + .max_rounds = 200, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }); + try std.testing.expect(!resumed.budget_exhausted); + try std.testing.expect(resumed.rounds_executed > 0); + try std.testing.expect(resumed.rounds_executed <= 200); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime degree canary runUntilIdle preserves published scores while rebuild is active" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .degree_canary, + .graph_metric_idle_planned_options = .{ + .worker_id = "degree-canary-freshness", + .max_rounds = 200, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try db.runUntilIdle(); + + var initial = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer initial.deinit(); + try std.testing.expectEqual(@as(usize, 1), initial.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[0].status.state); + const published_generation = initial.graph_metric_results[0].status.published_generation; + try std.testing.expect(published_generation > 0); + try std.testing.expectEqualStrings("doc:b", initial.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), initial.graph_metric_results[0].scores[0].score, 0.001); + + try db.batch(.{ + .writes = &.{.{ + .key = "doc:d", + .value = "{\"title\":\"delta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}", + }}, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + db.graph_metric_idle_planned_options.max_rounds = 1; + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + + var published = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published.deinit(); + try std.testing.expectEqual(@as(usize, 1), published.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, published.graph_metric_results[0].status.state); + try std.testing.expectEqual(published_generation, published.graph_metric_results[0].status.published_generation); + try std.testing.expect(published.graph_metric_results[0].status.building_generation > published_generation); + try std.testing.expectEqualStrings("doc:b", published.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), published.graph_metric_results[0].scores[0].score, 0.001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .published, + }}; + const traversal_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = traversal_query }}, + .limit = 0, + }); + defer traversal.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:b", traversal.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings("degree", traversal.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), traversal.graph_results[0].nodes[0].metrics[0].score orelse return error.TestUnexpectedResult, 0.001); + try std.testing.expectEqual(@as(usize, 1), traversal.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, traversal.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(published_generation, traversal.graph_results[0].metric_status[0].published_generation); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = "degree", + .freshness = .fresh, + }}; + var fresh_traversal_query = traversal_query; + fresh_traversal_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_traversal_query }}, + .limit = 0, + })); + + var rerank = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank.deinit(); + try std.testing.expectEqualStrings("doc:b", rerank.hits[0].id); + const rerank_status = rerank.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, rerank_status.state); + try std.testing.expectEqual(published_generation, rerank_status.published_generation); + const rerank_details = rerank.hits[0].score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("degree", rerank_details.metric_name); + try std.testing.expectEqual(published_generation, rerank_details.published_generation); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), rerank_details.metric_score orelse return error.TestUnexpectedResult, 0.001); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +test "db graph metric runtime degree canary runUntilIdle fails fast when active planned work is outside guardrails" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .degree_canary, + .graph_metric_idle_planned_options = .{ + .worker_id = "degree-canary-capped-active", + .max_rounds = 200, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_degree_canary_options = .{ + .max_control_records = 0, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const start = try db.runGraphMetricPlannedCoordinatorSweep(.{ + .max_metrics = 8, + .start_background_builds = true, + }); + try std.testing.expectEqual(@as(usize, 1), start.builds_started); + + const decision = try db.core.index_manager.graphMetricDegreeCanaryDecision(db.graph_metric_idle_degree_canary_options); + try std.testing.expectEqual(@as(usize, 1), decision.active_degree_builds); + try std.testing.expect(decision.control_records > decision.max_control_records); + try std.testing.expect(!decision.shouldRunPlanned()); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } +} + +test "db graph metric runtime degree canary runUntilIdle falls back to local oracle for mixed metrics" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .degree_canary, + .graph_metric_idle_planned_options = .{ + .worker_id = "degree-canary-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectDegreeCanaryDecision(db.core.index_manager, .{}, false, 0, 1, 0, 1); + + try db.runUntilIdle(); + + var degree_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer degree_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), degree_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, degree_result.graph_metric_results[0].status.state); + + var pagerank_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer pagerank_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), pagerank_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, pagerank_result.graph_metric_results[0].status.state); +} + +test "db graph metric runtime default gate runUntilIdle publishes configured graph pagerank metrics" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"max_iterations\":40,\"tolerance\":0.000001,\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"max_iterations\":20,\"tolerance\":0.000001,\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"max_iterations\":20,\"tolerance\":0.000001,\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"max_iterations\":20,\"tolerance\":0.000001,\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}],\"related\":[{\"target\":\"doc:x\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + .{ .key = "doc:x", .value = "{\"title\":\"excluded\"}" }, + }, + .sync_level = .write, + }); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + var degree_status = try graph_entry.index.graphMetricStatus("degree"); + defer degree_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, degree_status.state); + var eigenvector_status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer eigenvector_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, eigenvector_status.state); + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, authority_status.state); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, hub_status.state); + } + + try db.runUntilIdle(); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("pagerank"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expect(status.published_generation > 0); + try std.testing.expect(status.converged or status.iterations_completed == 40); + var degree_status = try graph_entry.index.graphMetricStatus("degree"); + defer degree_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, degree_status.state); + try std.testing.expect(degree_status.converged); + try std.testing.expectEqual(@as(u32, 1), degree_status.iterations_completed); + var eigenvector_status = try graph_entry.index.graphMetricStatus("eigenvector"); + defer eigenvector_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, eigenvector_status.state); + try std.testing.expect(eigenvector_status.converged or eigenvector_status.iterations_completed == 20); + var authority_status = try graph_entry.index.graphMetricStatus("hits_authority"); + defer authority_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, authority_status.state); + try std.testing.expect(authority_status.converged or authority_status.iterations_completed == 20); + var hub_status = try graph_entry.index.graphMetricStatus("hits_hub"); + defer hub_status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, hub_status.state); + try std.testing.expect(hub_status.converged or hub_status.iterations_completed == 20); + + const top = try graph_entry.index.graphMetricTopK("pagerank", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 4), top.len); + for (top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:x")); + + const degree_top = try graph_entry.index.graphMetricTopK("degree", 10); + defer { + for (degree_top) |*score| score.deinit(alloc); + alloc.free(degree_top); + } + try std.testing.expectEqual(@as(usize, 4), degree_top.len); + try std.testing.expectEqualStrings("doc:b", degree_top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 3.0), degree_top[0].score, 0.001); + for (degree_top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:x")); + + const eigenvector_top = try graph_entry.index.graphMetricTopK("eigenvector", 10); + defer { + for (eigenvector_top) |*score| score.deinit(alloc); + alloc.free(eigenvector_top); + } + try std.testing.expectEqual(@as(usize, 4), eigenvector_top.len); + for (eigenvector_top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:x")); + + const authority_top = try graph_entry.index.graphMetricTopK("hits_authority", 10); + defer { + for (authority_top) |*score| score.deinit(alloc); + alloc.free(authority_top); + } + try std.testing.expectEqual(@as(usize, 4), authority_top.len); + for (authority_top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:x")); + + const hub_top = try graph_entry.index.graphMetricTopK("hits_hub", 10); + defer { + for (hub_top) |*score| score.deinit(alloc); + alloc.free(hub_top); + } + try std.testing.expectEqual(@as(usize, 4), hub_top.len); + for (hub_top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:x")); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 0), metric_result.hits.len); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + + var degree_metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer degree_metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), degree_metric_result.graph_metric_results.len); + try std.testing.expectEqual(@as(usize, 1), degree_metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:b", degree_metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 3.0), degree_metric_result.graph_metric_results[0].scores[0].score, 0.001); +} + +test "db graph metric runtime operations manual refresh rebuild and delete operate on configured metric" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"manual_degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.runUntilIdle(); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("manual_degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, status.state); + } + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + try std.testing.expect(refreshed.published_generation > 0); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runUntilIdle(); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("manual_degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, status.state); + } + + var rebuilt = try db.rebuildGraphMetric(alloc, "graph_idx", "manual_degree"); + defer rebuilt.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, rebuilt.state); + try std.testing.expect(rebuilt.published_generation >= refreshed.published_generation); + + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const top = try graph_entry.index.graphMetricTopK("manual_degree", 1); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 1), top.len); + try std.testing.expectEqualStrings("doc:b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top[0].score, 0.001); + + var deleted = try db.deleteGraphMetricMaterialization(alloc, "graph_idx", "manual_degree"); + defer deleted.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.disabled, deleted.state); + try std.testing.expectEqual(@as(u64, 0), deleted.published_generation); + try std.testing.expect(!deleted.maintenance_paused); + try std.testing.expect(!deleted.build_queued); + + try std.testing.expectError(error.MetricNotReady, graph_entry.index.graphMetricTopK("manual_degree", 1)); + + var refreshed_after_delete = try db.refreshGraphMetric(alloc, "graph_idx", "manual_degree"); + defer refreshed_after_delete.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed_after_delete.state); +} + +test "db graph metric runtime operations pause and resume controls background maintenance" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + }, + .sync_level = .write, + }); + + try db.runUntilIdle(); + + var initial = try db.refreshGraphMetric(alloc, "graph_idx", "degree"); + defer initial.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, initial.state); + try std.testing.expect(!initial.maintenance_paused); + const first_generation = initial.published_generation; + try std.testing.expect(first_generation > 0); + + var paused = try db.pauseGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer paused.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, paused.state); + try std.testing.expect(paused.maintenance_paused); + try std.testing.expectEqual(first_generation, paused.published_generation); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expectEqual(@as(usize, 1), pending.metrics_scanned); + try std.testing.expectEqual(@as(usize, 1), pending.paused_metrics); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_pages); + try std.testing.expect(!pending.hasWork()); + } + + const planned_while_paused = try db.runGraphMetricPlannedMaintenanceForIdle(.{ + .worker_id = "paused-degree-worker", + .max_rounds = 4, + .max_metrics_per_round = 8, + .max_pages_per_round = 4, + }); + try std.testing.expect(!planned_while_paused.durableProgressed()); + try std.testing.expectEqual(@as(usize, 0), planned_while_paused.builds_started); + try std.testing.expectEqual(@as(usize, 0), planned_while_paused.worker_steps); + try std.testing.expectEqual(@as(usize, 0), planned_while_paused.coordinator_steps); + try std.testing.expectEqual(@as(usize, 0), planned_while_paused.pages_completed); + try std.testing.expectEqual(@as(usize, 0), planned_while_paused.published); + + try db.runUntilIdle(); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expect(status.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.stale, status.state); + try std.testing.expectEqual(first_generation, status.published_edge_generation); + + const top = try graph_entry.index.graphMetricTopK("degree", 10); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 2), top.len); + for (top) |score| try std.testing.expect(!std.mem.eql(u8, score.node, "doc:c")); + } + + var refreshed_while_paused = try db.refreshGraphMetric(alloc, "graph_idx", "degree"); + defer refreshed_while_paused.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed_while_paused.state); + try std.testing.expect(refreshed_while_paused.maintenance_paused); + try std.testing.expect(refreshed_while_paused.published_generation > first_generation); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const top = try graph_entry.index.graphMetricTopK("degree", 1); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 1), top.len); + try std.testing.expectEqualStrings("doc:b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), top[0].score, 0.001); + } + + var resumed = try db.resumeGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer resumed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, resumed.state); + try std.testing.expect(!resumed.maintenance_paused); + + const resumed_generation = resumed.published_generation; + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:d", .value = "{\"title\":\"delta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runUntilIdle(); + + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expect(!status.maintenance_paused); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expect(status.published_generation > resumed_generation); + + const top = try graph_entry.index.graphMetricTopK("degree", 1); + defer { + for (top) |*score| score.deinit(alloc); + alloc.free(top); + } + try std.testing.expectEqual(@as(usize, 1), top.len); + try std.testing.expectEqualStrings("doc:b", top[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 3.0), top[0].score, 0.001); + } + + var deleted = try db.deleteGraphMetricMaterialization(alloc, "graph_idx", "degree"); + defer deleted.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.disabled, deleted.state); + try std.testing.expect(!deleted.build_queued); + try db.runUntilIdle(); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.disabled, status.state); + try std.testing.expectEqual(@as(u64, 0), status.published_generation); + } + + var reenabled = try db.resumeGraphMetricMaintenance(alloc, "graph_idx", "degree"); + defer reenabled.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.not_ready, reenabled.state); + try std.testing.expect(reenabled.build_queued); + try db.runUntilIdle(); + { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var status = try graph_entry.index.graphMetricStatus("degree"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + } +} + +test "db graph metric runtime default gate runUntilIdle can use planned graph metric maintenance when enabled" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .planned, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle planned graph metric maintenance reports budget exhaustion and resumes" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .planned, + .graph_metric_idle_planned_options = .{ + .worker_id = "planned-idle-budgeted", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const decision = try db.core.index_manager.graphMetricPlannedAutoIdleDecision(db.graph_metric_idle_auto_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 0), decision.active_builds); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued); + try std.testing.expectEqual(@as(usize, 0), decision.ineligible_queued); + } + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto chooses planned for one degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-degree-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto chooses planned for one small pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-pagerank-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const decision = try db.core.index_manager.graphMetricPlannedAutoIdleDecision(db.graph_metric_idle_auto_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 0), decision.active_builds); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued); + try std.testing.expectEqual(@as(usize, 0), decision.ineligible_queued); + } + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto can cap larger pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-large-pagerank-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_pagerank_iterations = 3, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":4,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, false, 0, 0, 0, 1); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + // A bounded tick leaves durable work queued/active, never runs an + // unlimited fallback. Raising admission/drain limits resumes that work. + db.graph_metric_idle_auto_options = .{}; + db.graph_metric_idle_planned_options = .{}; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto can widen pagerank planned gate" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-wide-pagerank-planned", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_pagerank_iterations = 4, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":4,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const decision = try db.core.index_manager.graphMetricPlannedAutoIdleDecision(db.graph_metric_idle_auto_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 0), decision.active_builds); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued); + try std.testing.expectEqual(@as(usize, 0), decision.ineligible_queued); + } + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto chooses bounded planned for multi metric indexes" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-multi-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 2, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }, + .{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[1].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto chooses planned for one small eigenvector" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-eigenvector-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto chooses planned for compatible hits by default" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-hits-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); +} + +test "db graph metric runtime default gate runUntilIdle default graph metric maintenance auto resumes active planned pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ + .worker_id = "default-auto-active-pagerank", + .max_rounds = 0, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"manual\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "pagerank", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 1, 0, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses planned for one small pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-planned-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses planned for one degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-degree-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses planned for one small eigenvector" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-eigenvector-idle", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses bounded planned for multi metric indexes" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-legacy-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 2, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }, + .{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[1].scores[0].node); +} + +test "db graph metric runtime default gate prepares topology while numerical capacity is occupied" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = + \\{"metrics":{"degree":{"enabled":true,"kind":"degree","refresh":"manual"},"rank":{"enabled":true,"kind":"pagerank","refresh":"background","max_iterations":2}}} + , + }); + try db.batch(.{ + .writes = &.{ + .{ .key = "a", .value = "{\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"b\"}]}}}" }, + .{ .key = "b", .value = "{}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + const entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + var active = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "degree", entry.index.edge_generation); + defer active.deinit(alloc); + const cfg = for (entry.metric_configs) |cfg| { + if (std.mem.eql(u8, cfg.name, "rank")) break cfg; + } else return error.MetricNotConfigured; + const options = index_manager_mod.IndexManager.GraphMetricPlannedSchedulerSweepOptions{ + .max_metrics = 8, + .auto_idle_options = .{ .max_active_builds = 1, .max_active_builds_per_index = 1 }, + }; + // Keep the degree job active: only preparation workers are advanced. + var admitted = false; + for (0..16) |_| { + const sweep = try db.runGraphMetricPlannedCoordinatorSweep(options); + try std.testing.expectEqual(@as(usize, 0), sweep.builds_started); + if (try entry.index.runGraphMetricTopologyPreparationStep("prepare-at-cap")) { + admitted = true; + break; + } + } + try std.testing.expect(admitted); + for (0..128) |_| { + if (try entry.index.prepareGraphMetricTopology(cfg, entry.index.edge_generation)) break; + _ = try entry.index.runGraphMetricTopologyPreparationStep("prepare-at-cap"); + } else return error.TopologyPreparationDidNotSeal; + const sweep = try db.runGraphMetricPlannedCoordinatorSweep(options); + try std.testing.expectEqual(@as(usize, 0), sweep.builds_started); + var waiting = try entry.index.graphMetricStatus("rank"); + defer waiting.deinit(alloc); + try std.testing.expectEqual(@as(u64, 0), waiting.build_job_id); + var degree = try entry.index.graphMetricStatus("degree"); + defer degree.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, degree.state); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance defers queued work at per-index cap" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-bounded-fair-cap", + .max_rounds = 200, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_active_builds_per_index = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":3,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"background\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 1, 0); + + try db.runUntilIdle(); + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }, + .{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 1, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance can cap larger pagerank" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-large-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_pagerank_iterations = 3, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"pagerank\":{\"enabled\":true,\"kind\":\"pagerank\",\"refresh\":\"background\",\"max_iterations\":4,\"tolerance\":0.000000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, false, 0, 0, 0, 1); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + // Raising admission/drain limits resumes bounded queued/active work. + db.graph_metric_idle_auto_options = .{}; + db.graph_metric_idle_planned_options = .{}; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "pagerank", + .query = .{ + .index_name = "graph_idx", + .metric_name = "pagerank", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqualStrings("doc:d", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance can cap larger eigenvector" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-large-eigenvector-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_eigenvector_iterations = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, false, 0, 0, 0, 1); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + // Raising admission/drain limits resumes bounded queued/active work. + db.graph_metric_idle_auto_options = .{}; + db.graph_metric_idle_planned_options = .{}; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance can widen eigenvector planned gate" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-wide-eigenvector-planned", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_eigenvector_iterations = 2, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"background\",\"max_iterations\":2,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const decision = try db.core.index_manager.graphMetricPlannedAutoIdleDecision(db.graph_metric_idle_auto_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 0), decision.active_builds); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued); + try std.testing.expectEqual(@as(usize, 0), decision.ineligible_queued); + } + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses planned for compatible hits by default" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-hits-default-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); +} + +test "db graph metric runtime default gate standalone HITS lanes use resumable planned maintenance" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + for ([_]graph_mod.GraphMetricKind{ .hits_authority, .hits_hub }) |kind| { + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_planned_options = .{ .max_rounds = 1, .max_pages_per_round = 1 }, + }); + defer db.close(); + const config = try std.fmt.allocPrint(alloc, "{{\"metrics\":{{\"standalone\":{{\"enabled\":true,\"kind\":\"{s}\",\"refresh\":\"background\",\"max_iterations\":2}}}}}}", .{@tagName(kind)}); + defer alloc.free(config); + try db.addIndex(.{ .name = "graph_idx", .kind = .graph, .config_json = config }); + try db.batch(.{ .graph_writes = &.{ + .{ .index_name = "graph_idx", .source = "a", .target = "b", .edge_type = "cites", .weight = 1 }, + .{ .index_name = "graph_idx", .source = "c", .target = "b", .edge_type = "cites", .weight = 1 }, + }, .sync_level = .write }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 1, 0, 0); + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + db.graph_metric_idle_planned_options = .{}; + try db.runUntilIdle(); + var status = try db.core.graphIndex("graph_idx").?.index.graphMetricStatus("standalone"); + defer status.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, status.state); + try std.testing.expect(status.published_generation != 0); + } +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance bounds independent incompatible hits lifecycles" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-hits-incompatible-fallback", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_hits_iterations = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.00001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + // Incompatible HITS definitions are independent planned metrics; + // neither side may suppress the other as though they shared one lifecycle. + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 0, 2, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + // Raising admission/drain limits resumes bounded queued/active work. + db.graph_metric_idle_auto_options = .{}; + db.graph_metric_idle_planned_options = .{}; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance chooses planned for one compatible small hits pair" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-hits-opt-in", + .max_rounds = 1, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + .graph_metric_idle_auto_options = .{ + .max_hits_iterations = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"background\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .write, + }); + + try db.runDerivedUntil(db.core.nextDerivedSequence()); + { + const decision = try db.core.index_manager.graphMetricPlannedAutoIdleDecision(db.graph_metric_idle_auto_options); + try std.testing.expect(decision.shouldRunPlanned()); + try std.testing.expectEqual(@as(usize, 0), decision.active_builds); + try std.testing.expectEqual(@as(usize, 1), decision.eligible_queued); + try std.testing.expectEqual(@as(usize, 0), decision.ineligible_queued); + } + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + // Preparation is admitted independently; only numerical jobs count as active. + try std.testing.expectEqual(@as(usize, 1), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); + try std.testing.expectEqual(@as(usize, 3), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqual(@as(usize, 3), metric_result.graph_metric_results[1].scores.len); + try std.testing.expectEqualStrings("doc:authority", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance resumes active planned degree" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-active-degree", + .max_rounds = 0, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"degree\":{\"enabled\":true,\"kind\":\"degree\",\"refresh\":\"manual\",\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\"}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "degree", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 1, 0, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "degree", + .query = .{ + .index_name = "graph_idx", + .metric_name = "degree", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqualStrings("doc:b", metric_result.graph_metric_results[0].scores[0].node); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance resumes active planned eigenvector" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-active-eigenvector", + .max_rounds = 0, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"eigenvector\":{\"enabled\":true,\"kind\":\"eigenvector\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:a", .value = "{\"title\":\"alpha\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:b", .value = "{\"title\":\"beta\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:d\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:c", .value = "{\"title\":\"gamma\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:b\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:d", .value = "{\"title\":\"delta\"}" }, + }, + .sync_level = .write, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "eigenvector", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 1, 0, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "eigenvector", + .query = .{ + .index_name = "graph_idx", + .metric_name = "eigenvector", + .top_k = 2, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 1), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results[0].scores.len); +} + +test "db graph metric runtime default gate runUntilIdle auto graph metric maintenance resumes active planned hits pair" { + const DB = @import("../mod.zig").DB; + const alloc = std.testing.allocator; + + var path_buf: [256]u8 = undefined; + const path = TestHelpers.tempPath(&path_buf); + defer TestHelpers.cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + .graph_metric_idle_maintenance = .auto, + .graph_metric_idle_planned_options = .{ + .worker_id = "auto-active-hits", + .max_rounds = 0, + .max_metrics_per_round = 8, + .max_pages_per_round = 1, + }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = "{\"metrics\":{\"hits_authority\":{\"enabled\":true,\"kind\":\"hits_authority\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}},\"hits_hub\":{\"enabled\":true,\"kind\":\"hits_hub\",\"refresh\":\"manual\",\"max_iterations\":1,\"tolerance\":0.000001,\"edge_filter\":{\"types\":[\"cites\"]}}}}", + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const target_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + break :blk graph_entry.index.edge_generation; + }; + var started = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", "hits_authority", target_generation); + defer started.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, started.state); + try std.testing.expectEqual(target_generation, started.building_generation); + try expectPlannedAutoIdleDecision(db.core.index_manager, db.graph_metric_idle_auto_options, true, 1, 0, 0, 0); + + try std.testing.expectError(error.RunUntilIdleDidNotConverge, db.runUntilIdle()); + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 1), pending.active_builds); + } + + db.graph_metric_idle_planned_options.max_rounds = 200; + try db.runUntilIdle(); + + { + const pending = db.pendingWorkStats().graph_metric; + try std.testing.expect(!pending.hasWork()); + try std.testing.expectEqual(@as(usize, 0), pending.queued_builds); + try std.testing.expectEqual(@as(usize, 0), pending.active_builds); + } + + var metric_result = try db.search(alloc, .{ + .graph_metric_queries = &.{ + .{ + .name = "authority", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_authority", + .top_k = 3, + .freshness = .fresh, + }, + }, + .{ + .name = "hub", + .query = .{ + .index_name = "graph_idx", + .metric_name = "hits_hub", + .top_k = 3, + .freshness = .fresh, + }, + }, + }, + .limit = 0, + }); + defer metric_result.deinit(); + try std.testing.expectEqual(@as(usize, 2), metric_result.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[0].status.state); + try std.testing.expectEqual(target_generation, metric_result.graph_metric_results[0].status.published_generation); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, metric_result.graph_metric_results[1].status.state); + try std.testing.expectEqual(metric_result.graph_metric_results[0].status.published_generation, metric_result.graph_metric_results[1].status.published_generation); + try std.testing.expectEqual(@as(usize, 3), metric_result.graph_metric_results[0].scores.len); + try std.testing.expectEqual(@as(usize, 3), metric_result.graph_metric_results[1].scores.len); + try std.testing.expectEqualStrings("doc:authority", metric_result.graph_metric_results[0].scores[0].node); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), metric_result.graph_metric_results[0].scores[0].score, 0.001); + try std.testing.expect(metric_result.graph_metric_results[1].scores[0].score >= metric_result.graph_metric_results[1].scores[1].score); + try std.testing.expect(metric_result.graph_metric_results[1].scores[1].score > metric_result.graph_metric_results[1].scores[2].score); +} diff --git a/zig/pkg/antfly/src/storage/db/mod.zig b/zig/pkg/antfly/src/storage/db/mod.zig index fc1a3dd207..9701ac6ee4 100644 --- a/zig/pkg/antfly/src/storage/db/mod.zig +++ b/zig/pkg/antfly/src/storage/db/mod.zig @@ -55,6 +55,7 @@ pub const background_runtime = @import("../background_runtime.zig"); pub const io_threaded_runtime = @import("derived/io_threaded_runtime.zig"); pub const ttl_runtime = @import("maintenance/ttl_runtime.zig"); pub const transaction_runtime = @import("maintenance/transaction_runtime.zig"); +pub const graph_metric_runtime = @import("maintenance/graph_metric_runtime.zig"); pub const document_query = @import("document_query.zig"); pub const query_projection = @import("query/projection.zig"); pub const document_mapper = @import("document_mapper.zig"); diff --git a/zig/pkg/antfly/src/storage/db/ownership.zig b/zig/pkg/antfly/src/storage/db/ownership.zig index 6311d508c3..7f4e41ccc6 100644 --- a/zig/pkg/antfly/src/storage/db/ownership.zig +++ b/zig/pkg/antfly/src/storage/db/ownership.zig @@ -30,9 +30,13 @@ pub const Stats = struct { lease_owned: bool = false, has_lease: bool = false, acquisition_count: u64 = 0, + takeover_count: u64 = 0, lease_acquire_failures: u64 = 0, lost_leases: u64 = 0, last_acquired_ms: u64 = 0, + lease_expires_at_ms: u64 = 0, + lease_renew_after_ms: u64 = 0, + renewal_count: u64 = 0, lease_epoch: u64 = 0, }; @@ -43,11 +47,14 @@ pub const State = struct { lease_ttl_ms: u64, has_lease: bool, acquisition_count: u64, + takeover_count: u64, lease_acquire_failures: u64, lost_leases: u64, last_acquired_ms: u64, - lease_epoch: u64, lease_expires_at_ms: u64, + lease_renew_after_ms: u64, + renewal_count: u64, + lease_epoch: u64, pub fn init(alloc: Allocator, store: anytype, key: []const u8, config: Config) !State { return .{ @@ -57,11 +64,14 @@ pub const State = struct { .lease_ttl_ms = config.lease_ttl_ms, .has_lease = !config.lease_owned, .acquisition_count = 0, + .takeover_count = 0, .lease_acquire_failures = 0, .lost_leases = 0, .last_acquired_ms = 0, - .lease_epoch = 0, .lease_expires_at_ms = 0, + .lease_renew_after_ms = 0, + .renewal_count = 0, + .lease_epoch = 0, }; } @@ -72,6 +82,12 @@ pub const State = struct { self.* = undefined; } + pub fn deinitPreserveLease(self: *State, alloc: Allocator) void { + self.lease.deinit(); + alloc.free(self.owner_id); + self.* = undefined; + } + pub fn ensureLease(self: *State, now_ms: u64) !bool { if (!self.lease_owned) { self.has_lease = true; @@ -79,6 +95,10 @@ pub const State = struct { } const had_lease = self.has_lease; + // A lease cannot be taken by a conforming peer before its durable + // expiry. Keep the overwhelmingly common runtime tick on this + // in-memory path and renew early enough to tolerate scheduler stalls. + if (had_lease and now_ms < self.lease_renew_after_ms) return true; const acquired = try self.lease.tryAcquireFenced(self.owner_id, now_ms, self.lease_ttl_ms); if (acquired.acquired) { self.has_lease = true; @@ -88,6 +108,9 @@ pub const State = struct { self.acquisition_count += 1; self.last_acquired_ms = now_ms; } + if (had_lease and acquired.kind == .renewed) self.renewal_count += 1; + if (acquired.kind == .takeover) self.takeover_count += 1; + self.updateRenewalDeadline(); return true; } @@ -95,6 +118,20 @@ pub const State = struct { return false; } + fn updateRenewalDeadline(self: *State) void { + // Renew with one third of the TTL remaining. A deterministic + // per-owner jitter spreads writers sharing the same TTL without + // making tests or restart behavior nondeterministic. + const renewal_slack = @max(@as(u64, 1), self.lease_ttl_ms / 3); + const jitter_window = self.lease_ttl_ms / 10; + const jitter = if (jitter_window == 0) + 0 + else + std.hash.Wyhash.hash(0, self.owner_id) % (jitter_window + 1); + const base_renew_after = self.lease_expires_at_ms -| renewal_slack; + self.lease_renew_after_ms = base_renew_after -| jitter; + } + pub fn heartbeat(self: *State, now_ms: u64) !bool { if (!self.lease_owned) return true; if (!self.has_lease or self.lease_epoch == 0) return false; @@ -109,6 +146,8 @@ pub const State = struct { return false; } self.lease_expires_at_ms = std.math.add(u64, now_ms, self.lease_ttl_ms) catch std.math.maxInt(u64); + self.renewal_count += 1; + self.updateRenewalDeadline(); return true; } @@ -129,6 +168,8 @@ pub const State = struct { self.lease_expires_at_ms = 0; self.lost_leases += 1; } + self.lease_expires_at_ms = 0; + self.lease_renew_after_ms = 0; } pub fn release(self: *State) void { @@ -138,6 +179,23 @@ pub const State = struct { self.has_lease = !self.lease_owned; self.lease_epoch = 0; self.lease_expires_at_ms = 0; + self.lease_renew_after_ms = 0; + } + + pub fn releaseHeldLease(self: *State) !bool { + const released = if (self.lease_owned and self.has_lease) + try self.lease.releaseFenced(self.owner_id, self.lease_epoch) + else + false; + self.has_lease = !self.lease_owned; + self.lease_epoch = 0; + self.lease_expires_at_ms = 0; + self.lease_renew_after_ms = 0; + return released; + } + + pub fn loadLease(self: *State, alloc: Allocator) !?lease_mod.LeaseRecord { + return try self.lease.load(alloc); } pub fn stats(self: *const State) Stats { @@ -145,9 +203,13 @@ pub const State = struct { .lease_owned = self.lease_owned, .has_lease = self.has_lease, .acquisition_count = self.acquisition_count, + .takeover_count = self.takeover_count, .lease_acquire_failures = self.lease_acquire_failures, .lost_leases = self.lost_leases, .last_acquired_ms = self.last_acquired_ms, + .lease_expires_at_ms = self.lease_expires_at_ms, + .lease_renew_after_ms = self.lease_renew_after_ms, + .renewal_count = self.renewal_count, .lease_epoch = self.lease_epoch, }; } @@ -204,6 +266,39 @@ test "ownership state tracks lease takeover and loss" { try std.testing.expect(owner_b.has_lease); } +test "ownership state renews only at the cached renewal deadline" { + const alloc = std.testing.allocator; + var backend = mem_backend.Backend.init(alloc, .{}); + defer backend.close(); + var runtime = try backend.runtimeStore(alloc, .{ .name = "lease-renewal" }); + defer runtime.deinit(); + var owner = try State.init(alloc, runtime, "\x00\x00__metadata__:ownership_renewal_test", .{ + .lease_owned = true, + .owner_id = "worker-renewal", + .lease_ttl_ms = 30_000, + }); + defer owner.deinit(alloc); + + try std.testing.expect(try owner.ensureLease(1_000)); + const first_deadline = owner.lease_renew_after_ms; + try std.testing.expect(first_deadline > 1_000); + var first = (try owner.loadLease(alloc)) orelse return error.TestExpectedLease; + defer lease_mod.deinitRecord(alloc, &first); + try std.testing.expectEqual(@as(u64, 31_000), first.expires_at_ms); + + try std.testing.expect(try owner.ensureLease(first_deadline - 1)); + try std.testing.expectEqual(@as(u64, 0), owner.renewal_count); + var unchanged = (try owner.loadLease(alloc)) orelse return error.TestExpectedLease; + defer lease_mod.deinitRecord(alloc, &unchanged); + try std.testing.expectEqual(first.expires_at_ms, unchanged.expires_at_ms); + + try std.testing.expect(try owner.ensureLease(first_deadline)); + try std.testing.expectEqual(@as(u64, 1), owner.renewal_count); + var renewed = (try owner.loadLease(alloc)) orelse return error.TestExpectedLease; + defer lease_mod.deinitRecord(alloc, &renewed); + try std.testing.expectEqual(first_deadline + 30_000, renewed.expires_at_ms); +} + test "ownership state works with memory backend store" { const alloc = std.testing.allocator; var backend = mem_backend.Backend.init(alloc, .{}); diff --git a/zig/pkg/antfly/src/storage/db/query/graph_exec.zig b/zig/pkg/antfly/src/storage/db/query/graph_exec.zig index 9ccbfd2f45..7198c53e73 100644 --- a/zig/pkg/antfly/src/storage/db/query/graph_exec.zig +++ b/zig/pkg/antfly/src/storage/db/query/graph_exec.zig @@ -15,6 +15,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const types = @import("../types.zig"); +const graph_mod = @import("../../../graph/graph.zig"); const graph_query_mod = @import("../../../graph/query.zig"); const graph_pattern_mod = @import("../../../graph/pattern.zig"); const graph_node_identity = @import("../../../graph/node_identity.zig"); @@ -1605,7 +1606,7 @@ pub fn executeSingleNonPatternQueryWithSetsWithBudgets( target_keys, budgets.work, ); - errdefer graph_result.deinit(alloc); + defer graph_result.deinit(alloc); if (!executor.predicate_aware and searchRequestHasGraphPredicates(req)) { graph_result.nodes = try filterGraphResultNodes( alloc, @@ -1627,16 +1628,29 @@ pub fn executeSingleNonPatternQueryWithSetsWithBudgets( ); const name = try alloc.dupe(u8, named.name); + errdefer alloc.free(name); + const metric_status = if (named.query.include_metric_status) + try cloneGraphMetricStatusesFromGraph(alloc, graph_result.metric_status) + else + @constCast((&[_]types.GraphMetricStatus{})[0..]); + errdefer types.freeGraphMetricStatuses(alloc, metric_status); const nodes = graph_result.nodes; graph_result.nodes = &.{}; + const metric_values_slab = graph_result.metric_values_slab; + graph_result.metric_values_slab = &.{}; + const metric_value_names = graph_result.metric_value_names; + graph_result.metric_value_names = &.{}; return .{ .name = name, .nodes = nodes, + .metric_values_slab = metric_values_slab, + .metric_value_names = metric_value_names, .paths = &.{}, .matches = &.{}, .hits = hits, .total_hits = total_hits, + .metric_status = metric_status, }; } @@ -1674,6 +1688,75 @@ fn buildPathGraphSearchResult( }; } +fn cloneGraphMetricStatusesFromGraph( + alloc: Allocator, + statuses: []const graph_query_mod.GraphMetricStatus, +) ![]types.GraphMetricStatus { + if (statuses.len == 0) return &.{}; + const out = try alloc.alloc(types.GraphMetricStatus, statuses.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*status| status.deinit(alloc); + alloc.free(out); + } + for (statuses, 0..) |status, i| { + const name = try alloc.dupe(u8, status.name); + var name_moved = false; + errdefer if (!name_moved) alloc.free(name); + var edge_filter = try status.edge_filter.cloneAlloc(alloc); + var edge_filter_moved = false; + errdefer if (!edge_filter_moved) edge_filter.deinit(alloc); + const recent_events = if (status.recent_events.len > 0) + try alloc.dupe(graph_mod.GraphIndex.GraphMetricEvent, status.recent_events) + else + @constCast((&[_]graph_mod.GraphIndex.GraphMetricEvent{})[0..]); + var recent_events_moved = false; + errdefer if (!recent_events_moved and recent_events.len > 0) alloc.free(recent_events); + const last_error = if (status.last_error.len > 0) try alloc.dupe(u8, status.last_error) else ""; + var last_error_moved = false; + errdefer if (!last_error_moved and last_error.len > 0) alloc.free(last_error); + const build_worker_id = if (status.build_worker_id.len > 0) try alloc.dupe(u8, status.build_worker_id) else ""; + var build_worker_id_moved = false; + errdefer if (!build_worker_id_moved and build_worker_id.len > 0) alloc.free(build_worker_id); + out[i] = .{ + .name = name, + .state = status.state, + .phase = status.phase, + .edge_filter = edge_filter, + .metadata_version = status.metadata_version, + .config_fingerprint = status.config_fingerprint, + .maintenance_paused = status.maintenance_paused, + .build_queued = status.build_queued, + .published_generation = if (status.published_edge_generation != 0) status.published_edge_generation else status.published_generation, + .edge_generation = status.edge_generation, + .target_edge_generation = status.target_edge_generation, + .queued_generation = status.queued_generation, + .building_generation = status.building_generation, + .build_job_id = status.build_job_id, + .build_started_at_ms = status.build_started_at_ms, + .build_iteration = status.build_iteration, + .build_lease_expires_at_ms = status.build_lease_expires_at_ms, + .build_worker_id = build_worker_id, + .retry_count = status.retry_count, + .last_error = last_error, + .progress = status.progress, + .converged = status.converged, + .iterations_completed = status.iterations_completed, + .delta = status.delta, + .computed_at_ms = status.computed_at_ms, + .last_event = status.last_event, + .recent_events = recent_events, + }; + name_moved = true; + edge_filter_moved = true; + recent_events_moved = true; + last_error_moved = true; + build_worker_id_moved = true; + initialized += 1; + } + return out; +} + pub fn executeSearchGraphWithSets( alloc: Allocator, req: types.SearchRequest, @@ -6725,3 +6808,125 @@ test "graph query result doc-set resolution receives identity generation" { try std.testing.expect(harness.saw_generation); } +test "db query result shape executeSingleNonPatternQueryWithSets hides metric status unless requested" { + const alloc = std.testing.allocator; + + const Harness = struct { + fn findShortestPath( + _: ?*anyopaque, + _: Allocator, + _: *const types.NamedGraphQuery, + _: []const u8, + _: []const u8, + _: *graph_pattern_mod.WorkBudget, + ) anyerror!?types.GraphPath { + return null; + } + + fn findKShortestPaths( + _: ?*anyopaque, + alloc_inner: Allocator, + _: *const types.NamedGraphQuery, + _: []const u8, + _: []const u8, + _: *graph_pattern_mod.WorkBudget, + ) anyerror![]types.GraphPath { + return try alloc_inner.alloc(types.GraphPath, 0); + } + + fn executeGraphQuery( + _: ?*anyopaque, + alloc_inner: Allocator, + _: *const types.NamedGraphQuery, + _: []const []const u8, + _: [][]u8, + _: *graph_pattern_mod.WorkBudget, + ) anyerror!graph_query_mod.GraphQueryResult { + const metric_status = try alloc_inner.alloc(graph_query_mod.GraphMetricStatus, 1); + metric_status[0] = .{ + .name = try alloc_inner.dupe(u8, "pagerank"), + .state = .fresh, + .published_generation = 5, + .edge_generation = 5, + .target_edge_generation = 5, + .progress = 1.0, + .converged = true, + }; + return .{ + .nodes = try alloc_inner.alloc(graph_query_mod.GraphResultNode, 0), + .matches = &.{}, + .metric_status = metric_status, + }; + } + + fn loadProjectedDocument( + _: ?*anyopaque, + _: Allocator, + _: types.SearchRequest, + _: []const u8, + ) anyerror!?[]u8 { + return null; + } + }; + + const graph_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = "pagerank", + .freshness = .published, + }}; + var named = types.NamedGraphQuery{ + .name = "tree_search", + .query = .{ + .query_type = .traverse, + .index_name = "doc_hierarchy", + .start_nodes = .{ .keys = &.{"doc:root"} }, + .params = .{}, + .order_by = &graph_metric_orders, + }, + }; + const executor = NonPatternQueryExecutor{ + .ctx = null, + .find_shortest_path = Harness.findShortestPath, + .find_k_shortest_paths = Harness.findKShortestPaths, + .execute_graph_query = Harness.executeGraphQuery, + .load_projected_document = Harness.loadProjectedDocument, + }; + + var hidden = try executeSingleNonPatternQueryWithSets(alloc, .{ .limit = 10 }, &named, &.{}, executor); + defer hidden.deinit(alloc); + try std.testing.expectEqual(@as(usize, 0), hidden.metric_status.len); + + named.query.include_metric_status = true; + var included = try executeSingleNonPatternQueryWithSets(alloc, .{ .limit = 10 }, &named, &.{}, executor); + defer included.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), included.metric_status.len); + try std.testing.expectEqualStrings("pagerank", included.metric_status[0].name); + try std.testing.expectEqual(@as(u64, 5), included.metric_status[0].published_generation); +} + +test "graph metric status clone owns active build worker id" { + const alloc = std.testing.allocator; + + const worker_id = try alloc.dupe(u8, "worker-a"); + defer alloc.free(worker_id); + const statuses = [_]graph_query_mod.GraphMetricStatus{.{ + .name = "pagerank", + .state = .building, + .phase = .computing, + .build_queued = true, + .building_generation = 7, + .build_job_id = 12345, + .build_iteration = 3, + .build_worker_id = worker_id, + }}; + + const cloned = try cloneGraphMetricStatusesFromGraph(alloc, &statuses); + defer { + for (cloned) |*status| status.deinit(alloc); + alloc.free(cloned); + } + + try std.testing.expectEqual(@as(usize, 1), cloned.len); + try std.testing.expectEqualStrings("worker-a", cloned[0].build_worker_id); + try std.testing.expect(cloned[0].build_worker_id.ptr != worker_id.ptr); + try std.testing.expectEqual(@as(u64, 12345), cloned[0].build_job_id); +} diff --git a/zig/pkg/antfly/src/storage/db/test_support.zig b/zig/pkg/antfly/src/storage/db/test_support.zig new file mode 100644 index 0000000000..196b598582 --- /dev/null +++ b/zig/pkg/antfly/src/storage/db/test_support.zig @@ -0,0 +1,908 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const builtin = @import("builtin"); +const platform = @import("antfly_platform"); + +const Allocator = std.mem.Allocator; + +const asset_producer_mod = @import("enrichment/asset_producer.zig"); +const enrichment_artifact_codec = @import("enrichment/artifact_codec.zig"); +const embedder_mod = @import("enrichment/embedder.zig"); +const graph_mod = @import("../../graph/graph.zig"); +const graph_query_mod = @import("../../graph/query.zig"); +const hbc_mod = @import("../hbc_adapter.zig"); +const index_manager_mod = @import("catalog/index_manager.zig"); +const lsm_backend_mod = @import("../lsm_backend/mod.zig"); +const promotion_runtime_mod = @import("promotion_runtime.zig"); +const transactions_mod = @import("../transactions.zig"); +const types = @import("types.zig"); + +fn getenv(name: [*:0]const u8) ?[]const u8 { + return platform.env.getenv(name); +} + +fn threadedIo() if (builtin.os.tag == .freestanding) void else std.Io.Threaded { + if (builtin.os.tag == .freestanding) return; + return std.Io.Threaded.init(std.heap.page_allocator, .{}); +} + +fn spinOrYield() void { + if (builtin.os.tag == .freestanding) std.atomic.spinLoopHint() else std.Thread.yield() catch {}; +} + +fn sleepPollInterval() void { + platform.clock.Clock.real().sleepMs(10); +} + +pub const default_test_wait_attempts: usize = 100; +pub const slow_test_wait_attempts: usize = 500; + +pub fn expectDenseEmbeddingArtifactValue(alloc: Allocator, value: []const u8, source_hash: ?u64, dims: usize) !void { + const header = try enrichment_artifact_codec.decodeHeader(value); + try std.testing.expectEqual(enrichment_artifact_codec.codec_version, header.version); + try std.testing.expectEqual(enrichment_artifact_codec.Kind.dense_embedding, header.kind); + try std.testing.expectEqual(source_hash != null, header.flags.has_source_hash); + if (source_hash) |expected_hash| try std.testing.expectEqual(expected_hash, header.source_hash); + + const vector = try enrichment_artifact_codec.decodeDenseEmbeddingAlloc(alloc, value); + defer alloc.free(vector); + try std.testing.expectEqual(dims, vector.len); +} + +var temp_path_nonce: u64 = 0; + +pub fn profileBenchTestsEnabled() bool { + if (comptime builtin.os.tag == .freestanding) return false; + return getenv("ANTFLY_RUN_PROFILE_BENCH_TESTS") != null; +} + +pub fn lockApply(db: anytype) void { + db.core.lockApply(); +} + +pub fn stressDenseBackend() hbc_mod.StorageBackend { + const raw = getenv("ANTFLY_STRESS_DENSE_BACKEND") orelse return .lsm; + if (std.ascii.eqlIgnoreCase(raw, "lmdb")) return .lmdb; + return .lsm; +} + +pub fn allocStressDenseDocJson(alloc: Allocator, dims: usize, doc_index: usize) ![]u8 { + var out = std.ArrayListUnmanaged(u8).empty; + errdefer out.deinit(alloc); + + try out.appendSlice(alloc, "{\"title\":\"dense\",\"_embeddings\":{\"dv_v1\":["); + for (0..dims) |dim_index| { + if (dim_index > 0) try out.append(alloc, ','); + + var value_buf: [64]u8 = undefined; + const rendered = try std.fmt.bufPrint(&value_buf, "{d}", .{index_manager_mod.stressDenseValue(doc_index, dim_index)}); + try out.appendSlice(alloc, rendered); + } + try out.appendSlice(alloc, "]}}"); + + const owned = try alloc.dupe(u8, out.items); + out.deinit(alloc); + return owned; +} + +pub fn tempPath(buf: []u8) [*:0]const u8 { + const base = "/tmp/antfly-db-test-"; + const ts = platform.time.monotonicNs(); + const pid: u32 = @intCast(std.posix.system.getpid()); + const nonce = @atomicRmw(u64, &temp_path_nonce, .Add, 1, .monotonic); + const path = std.fmt.bufPrint(buf, "{s}{d}-{d}-{d}\x00", .{ base, pid, ts, nonce }) catch unreachable; + return @ptrCast(path.ptr); +} + +pub fn cleanupTempDir(path: [*:0]const u8) void { + var io_impl = threadedIo(); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), std.mem.span(path)) catch {}; +} + +pub fn cleanupSnapshotDirForPath(path: [*:0]const u8) void { + var snapshots_buf: [512]u8 = undefined; + const snapshots = std.fmt.bufPrint(&snapshots_buf, "{s}.snapshots", .{std.mem.span(path)}) catch return; + var io_impl = threadedIo(); + defer io_impl.deinit(); + std.Io.Dir.cwd().deleteTree(io_impl.io(), snapshots) catch {}; +} + +pub fn corruptNonEmptyFilesUnderDir(alloc: Allocator, root_path: []const u8) !usize { + var io_impl = threadedIo(); + defer io_impl.deinit(); + const io = io_impl.io(); + + var root_dir = try std.Io.Dir.cwd().openDir(io, root_path, .{ .iterate = true }); + defer root_dir.close(io); + + var walker = try root_dir.walk(alloc); + defer walker.deinit(); + + var corrupted: usize = 0; + while (try walker.next(io)) |entry| { + if (entry.kind != .file) continue; + const full_path = try std.fmt.allocPrint(alloc, "{s}/{s}", .{ root_path, entry.path }); + defer alloc.free(full_path); + const stat = try std.Io.Dir.cwd().statFile(io, full_path, .{}); + if (stat.size == 0) continue; + + const bytes = try alloc.alloc(u8, @intCast(stat.size)); + defer alloc.free(bytes); + for (bytes, 0..) |*byte, i| { + byte.* = @truncate((i *% 131) +% 17); + } + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = full_path, + .data = bytes, + }); + corrupted += 1; + } + return corrupted; +} + +pub fn cacheBlockHitsForBench(stats: anytype) u64 { + var hits: u64 = 0; + if (@hasField(@TypeOf(stats), "run_table_block")) { + hits += stats.run_table_block.hits; + } + if (@hasField(@TypeOf(stats), "run_table_physical_block")) { + hits += stats.run_table_physical_block.hits; + } + return hits; +} + +/// A queued obsolete path only counts as reclaimable once no reader pins +/// the backend; transient background readers (index loads, status probes) +/// mask it as pinned_by_readers for a moment. Poll instead of asserting a +/// single snapshot so loaded CI machines don't flake. +pub fn expectObsoletePathsReclaimable(backend: *lsm_backend_mod.Backend, expected: u64) !void { + var attempts: usize = 0; + while (backend.snapshotMaintenanceStats().obsolete_paths_reclaimable != expected) : (attempts += 1) { + if (attempts >= 2000) { + return std.testing.expectEqual(expected, backend.snapshotMaintenanceStats().obsolete_paths_reclaimable); + } + spinOrYield(); + } +} + +pub fn verifyDbSingleVectorFailedPlannedRebuildPreservesPublishedPublicReads( + comptime DB: type, + alloc: Allocator, + metric_name: []const u8, + metric_kind: []const u8, +) !void { + var path_buf: [256]u8 = undefined; + const path = tempPath(&path_buf); + defer cleanupTempDir(path); + + var db = try DB.open(alloc, std.mem.span(path), .{ + .start_index_workers = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + + try db.addIndex(.{ + .name = "ft_v1", + .kind = .full_text, + .config_json = "{\"store\":true}", + }); + const graph_config = try std.fmt.allocPrint( + alloc, + "{{\"metrics\":{{\"{s}\":{{\"enabled\":true,\"kind\":\"{s}\",\"refresh\":\"manual\",\"max_iterations\":4,\"tolerance\":0.000001,\"edge_filter\":{{\"types\":[\"cites\"]}}}}}}}}", + .{ metric_name, metric_kind }, + ); + defer alloc.free(graph_config); + try db.addIndex(.{ + .name = "graph_idx", + .kind = .graph, + .config_json = graph_config, + }); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:hub-a", .value = "{\"title\":\"hub a\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:hub-b", .value = "{\"title\":\"hub b\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + .{ .key = "doc:authority", .value = "{\"title\":\"authority\"}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + var refreshed = try db.refreshGraphMetric(alloc, "graph_idx", metric_name); + defer refreshed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, refreshed.state); + const published_generation = refreshed.published_generation; + try std.testing.expect(published_generation > 0); + + var initial = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 3, + .freshness = .fresh, + }, + }}, + .limit = 0, + }); + defer initial.deinit(); + try std.testing.expectEqual(@as(usize, 1), initial.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.fresh, initial.graph_metric_results[0].status.state); + try std.testing.expectEqual(published_generation, initial.graph_metric_results[0].status.published_generation); + try std.testing.expect(initial.graph_metric_results[0].scores.len > 0); + + try db.batch(.{ + .writes = &.{ + .{ .key = "doc:new-hub", .value = "{\"title\":\"new hub\",\"_edges\":{\"graph_idx\":{\"cites\":[{\"target\":\"doc:authority\",\"weight\":1.0}]}}}" }, + }, + .sync_level = .full_index, + }); + try db.runDerivedUntil(db.core.nextDerivedSequence()); + + const rebuilding_generation = blk: { + const graph_entry = db.core.graphIndex("graph_idx") orelse return error.IndexNotFound; + const target_generation = graph_entry.index.edge_generation; + try std.testing.expect(target_generation > published_generation); + var building = try db.ensureGraphMetricPlannedBuild(alloc, "graph_idx", metric_name, target_generation); + defer building.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.building, building.state); + try std.testing.expectEqual(target_generation, building.building_generation); + break :blk target_generation; + }; + + var failed = try db.failGraphMetricPlannedBuild(alloc, "graph_idx", metric_name, error.InvalidGraphMetricScore); + defer failed.deinit(alloc); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, failed.state); + try std.testing.expectEqual(published_generation, failed.published_generation); + try std.testing.expectEqual(rebuilding_generation, failed.target_edge_generation); + + var published_after_failure = try db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 4, + .freshness = .published, + }, + }}, + .limit = 0, + }); + defer published_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), published_after_failure.graph_metric_results.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, published_after_failure.graph_metric_results[0].status.state); + try std.testing.expectEqual(published_generation, published_after_failure.graph_metric_results[0].status.published_generation); + try std.testing.expect(published_after_failure.graph_metric_results[0].scores.len > 0); + for (published_after_failure.graph_metric_results[0].scores) |score| { + try std.testing.expect(!std.mem.eql(u8, score.node, "doc:new-hub")); + try std.testing.expect(std.math.isFinite(score.score)); + } + + const published_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = metric_name, + .freshness = .published, + }}; + const published_graph_query = graph_query_mod.GraphQuery{ + .query_type = .neighbors, + .index_name = "graph_idx", + .start_nodes = .{ .keys = &.{"doc:hub-a"} }, + .params = .{ .edge_types = &.{"cites"}, .direction = .out, .max_depth = 1 }, + .metrics = &published_metric_reads, + .include_metric_status = true, + }; + var traversal_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_graph_query }}, + .limit = 0, + }); + defer traversal_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results.len); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results[0].nodes.len); + try std.testing.expectEqualStrings("doc:authority", traversal_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results[0].nodes[0].metrics.len); + try std.testing.expectEqualStrings(metric_name, traversal_after_failure.graph_results[0].nodes[0].metrics[0].name); + try std.testing.expect(traversal_after_failure.graph_results[0].nodes[0].metrics[0].score != null); + try std.testing.expect(std.math.isFinite(traversal_after_failure.graph_results[0].nodes[0].metrics[0].score.?)); + try std.testing.expectEqual(@as(usize, 1), traversal_after_failure.graph_results[0].metric_status.len); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, traversal_after_failure.graph_results[0].metric_status[0].state); + try std.testing.expectEqual(published_generation, traversal_after_failure.graph_results[0].metric_status[0].published_generation); + + const published_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = metric_name, + .freshness = .published, + }}; + var published_order_query = published_graph_query; + published_order_query.order_by = &published_metric_orders; + var traversal_order_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_order_query }}, + .limit = 0, + }); + defer traversal_order_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_order_after_failure.graph_results.len); + try std.testing.expectEqualStrings("doc:authority", traversal_order_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, traversal_order_after_failure.graph_results[0].metric_status[0].state); + + const published_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = metric_name, + .op = .gte, + .value = 0.0, + .freshness = .published, + }}; + var published_filter_query = published_graph_query; + published_filter_query.where_metric = &published_metric_filters; + var traversal_filter_after_failure = try db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = published_filter_query }}, + .limit = 0, + }); + defer traversal_filter_after_failure.deinit(); + try std.testing.expectEqual(@as(usize, 1), traversal_filter_after_failure.graph_results.len); + try std.testing.expectEqualStrings("doc:authority", traversal_filter_after_failure.graph_results[0].nodes[0].key); + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, traversal_filter_after_failure.graph_results[0].metric_status[0].state); + + var rerank_after_failure = try db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .freshness = .published, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + }); + defer rerank_after_failure.deinit(); + try std.testing.expectEqual(@as(u32, 4), rerank_after_failure.total_hits); + const rerank_status = rerank_after_failure.graph_metric_rerank_status orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(graph_mod.GraphIndex.GraphMetricState.failed, rerank_status.state); + try std.testing.expectEqual(published_generation, rerank_status.published_generation); + var saw_authority_score = false; + var saw_new_hub_missing_score = false; + for (rerank_after_failure.hits) |hit| { + const details = hit.score_details orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(published_generation, details.published_generation); + if (std.mem.eql(u8, hit.id, "doc:authority")) { + saw_authority_score = details.metric_score != null; + } else if (std.mem.eql(u8, hit.id, "doc:new-hub")) { + saw_new_hub_missing_score = details.metric_score == null; + } + } + try std.testing.expect(saw_authority_score); + try std.testing.expect(saw_new_hub_missing_score); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_metric_queries = &.{.{ + .name = "central", + .query = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .top_k = 1, + .freshness = .fresh, + }, + }}, + .limit = 0, + })); + + const fresh_metric_reads = [_]graph_query_mod.GraphMetricRead{.{ + .name = metric_name, + .freshness = .fresh, + }}; + var fresh_projection_query = published_graph_query; + fresh_projection_query.metrics = &fresh_metric_reads; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_projection_query }}, + .limit = 0, + })); + + const fresh_metric_orders = [_]graph_query_mod.GraphMetricOrder{.{ + .name = metric_name, + .freshness = .fresh, + }}; + var fresh_order_query = published_graph_query; + fresh_order_query.order_by = &fresh_metric_orders; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_order_query }}, + .limit = 0, + })); + + const fresh_metric_filters = [_]graph_query_mod.GraphMetricFilter{.{ + .name = metric_name, + .op = .gte, + .value = 0.0, + .freshness = .fresh, + }}; + var fresh_filter_query = published_graph_query; + fresh_filter_query.where_metric = &fresh_metric_filters; + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .graph_queries = &.{.{ .name = "neighbors", .query = fresh_filter_query }}, + .limit = 0, + })); + + try std.testing.expectError(error.MetricStale, db.search(alloc, .{ + .index_name = "ft_v1", + .full_text = .{ .match_all = {} }, + .graph_metric_rerank = .{ + .index_name = "graph_idx", + .metric_name = metric_name, + .freshness = .fresh, + .weight = 1.0, + }, + .limit = 4, + .include_stored = false, + })); +} + +pub fn waitForSearchResult(alloc: Allocator, db: anytype, req: types.SearchRequest, min_hits: u32) !types.SearchResult { + return waitForSearchResultWithAttempts(alloc, db, req, min_hits, default_test_wait_attempts); +} + +pub fn waitForSearchResultWithAttempts(alloc: Allocator, db: anytype, req: types.SearchRequest, min_hits: u32, max_attempts: usize) !types.SearchResult { + var last = try db.search(alloc, req); + var attempts: usize = 0; + while (last.total_hits < min_hits and attempts < max_attempts) : (attempts += 1) { + last.deinit(); + sleepPollInterval(); + last = try db.search(alloc, req); + } + if (last.total_hits < min_hits) { + last.deinit(); + return error.Timeout; + } + return last; +} + +pub fn waitForDenseSearchResult(alloc: Allocator, db: anytype, req: types.SearchRequest, min_hits: u32) !types.SearchResult { + return waitForDenseSearchResultWithAttempts(alloc, db, req, min_hits, default_test_wait_attempts); +} + +pub fn waitForDenseSearchResultWithAttempts(alloc: Allocator, db: anytype, req: types.SearchRequest, min_hits: u32, max_attempts: usize) !types.SearchResult { + const dense = req.dense orelse return error.InvalidArgument; + var last_profiled = try db.searchDenseProfiled(alloc, req, dense); + var last = last_profiled.result; + var attempts: usize = 0; + while (last.total_hits < min_hits and attempts < max_attempts) : (attempts += 1) { + last.deinit(); + sleepPollInterval(); + last_profiled = try db.searchDenseProfiled(alloc, req, dense); + last = last_profiled.result; + } + if (last.total_hits < min_hits) { + last.deinit(); + return error.Timeout; + } + return last; +} + +pub fn waitForDenseIndexResultsWithAttempts(index: *hbc_mod.HBCIndex, query: []const f32, k: usize, min_hits: usize, max_attempts: usize) !hbc_mod.SearchResults { + var last = try index.searchWithRequest(.{ + .query = query, + .k = k, + }); + var attempts: usize = 0; + while (last.getHits().len < min_hits and attempts < max_attempts) : (attempts += 1) { + last.deinit(); + sleepPollInterval(); + last = try index.searchWithRequest(.{ + .query = query, + .k = k, + }); + } + if (last.getHits().len < min_hits) { + last.deinit(); + return error.Timeout; + } + return last; +} + +pub fn waitForAppliedSequenceAdvance( + alloc: Allocator, + db: anytype, + index_name: []const u8, + previous: u64, +) !u64 { + var applied = try db.core.loadAppliedSequence(alloc, index_name); + var attempts: usize = 0; + while (applied <= previous and attempts < 100) : (attempts += 1) { + sleepPollInterval(); + applied = try db.core.loadAppliedSequence(alloc, index_name); + } + if (applied <= previous) return error.Timeout; + return applied; +} + +pub fn waitForRawDelete(alloc: Allocator, db: anytype, key: []const u8, max_attempts: usize) !void { + var attempts: usize = 0; + while (attempts < max_attempts) : (attempts += 1) { + const raw = try db.get(alloc, key); + if (raw == null) return; + alloc.free(raw.?); + sleepPollInterval(); + } + return error.Timeout; +} + +pub fn putDenseEmbeddingArtifactForTest(db: anytype, alloc: Allocator, artifact_key: []const u8, source_hash: ?u64, vector: []const f32) !void { + const payload = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, source_hash, vector); + defer alloc.free(payload); + try db.core.store.put(artifact_key, payload); + try db.core.putArtifactPresenceMarker(); +} + +pub fn putSparseEmbeddingArtifactForTest( + db: anytype, + alloc: Allocator, + artifact_key: []const u8, + source_hash: ?u64, + indices: []const u32, + values: []const f32, +) !void { + const payload = try enrichment_artifact_codec.encodeSparseEmbeddingAlloc(alloc, source_hash, indices, values); + defer alloc.free(payload); + try db.core.store.put(artifact_key, payload); + try db.core.putArtifactPresenceMarker(); +} + +pub const CountingDenseEmbedder = struct { + deterministic: embedder_mod.DeterministicDenseEmbedder = .{}, + calls: usize = 0, + + fn embedDense(ptr: *anyopaque, alloc: Allocator, embedding_name: []const u8, text: []const u8, dims: u32) ![]f32 { + const self: *CountingDenseEmbedder = @ptrCast(@alignCast(ptr)); + self.calls += 1; + return try embedder_mod.DeterministicDenseEmbedder.embedDense(&self.deterministic, alloc, embedding_name, text, dims); + } + + pub fn interface(self: *CountingDenseEmbedder) embedder_mod.DenseEmbedder { + return .{ + .ptr = self, + .dense_embed_fn = embedDense, + .deinit_fn = null, + }; + } +}; + +pub const GateDenseEmbedder = struct { + allowed_successes: std.atomic.Value(usize) = .init(1), + successful_requests: std.atomic.Value(usize) = .init(0), + total_requests: std.atomic.Value(usize) = .init(0), + blocked_requests: std.atomic.Value(usize) = .init(0), + blocked_error: anyerror = error.EmbedRateLimited, + + fn containsIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + var i: usize = 0; + while (i + needle.len <= haystack.len) : (i += 1) { + var matched = true; + for (needle, 0..) |needle_ch, j| { + if (std.ascii.toLower(haystack[i + j]) != needle_ch) { + matched = false; + break; + } + } + if (matched) return true; + } + return false; + } + + fn vectorForText(text: []const u8) [3]f32 { + if (containsIgnoreCase(text, "alpha") or containsIgnoreCase(text, "concept")) { + return .{ 1.0, 0.0, 0.0 }; + } + if (containsIgnoreCase(text, "beta")) { + return .{ 0.0, 1.0, 0.0 }; + } + if (containsIgnoreCase(text, "retrieval") or containsIgnoreCase(text, "semantic")) { + return .{ 0.8, 0.2, 0.0 }; + } + return .{ 0.0, 0.0, 1.0 }; + } + + fn embedDense(ptr: *anyopaque, alloc: Allocator, _: []const u8, text: []const u8, dims: u32) ![]f32 { + const self: *GateDenseEmbedder = @ptrCast(@alignCast(ptr)); + _ = self.total_requests.fetchAdd(1, .monotonic); + const previous_successes = self.successful_requests.fetchAdd(1, .acq_rel); + if (previous_successes >= self.allowed_successes.load(.acquire)) { + _ = self.successful_requests.fetchSub(1, .acq_rel); + _ = self.blocked_requests.fetchAdd(1, .monotonic); + return self.blocked_error; + } + if (dims != 3) return error.InvalidVectorDimensions; + const vector = try alloc.alloc(f32, 3); + const values = vectorForText(text); + @memcpy(vector, &values); + return vector; + } + + pub fn interface(self: *GateDenseEmbedder) embedder_mod.DenseEmbedder { + return .{ + .ptr = self, + .dense_embed_fn = embedDense, + .deinit_fn = null, + }; + } + + pub fn allowAll(self: *GateDenseEmbedder) void { + self.allowed_successes.store(std.math.maxInt(usize), .release); + } + + pub fn snapshot(self: *GateDenseEmbedder) struct { + total_requests: usize, + blocked_requests: usize, + successful_requests: usize, + } { + return .{ + .total_requests = self.total_requests.load(.acquire), + .blocked_requests = self.blocked_requests.load(.acquire), + .successful_requests = self.successful_requests.load(.acquire), + }; + } +}; + +pub const GateSparseEmbedder = struct { + deterministic: embedder_mod.DeterministicSparseEmbedder = .{}, + allowed_successes: std.atomic.Value(usize) = .init(0), + successful_requests: std.atomic.Value(usize) = .init(0), + blocked_requests: std.atomic.Value(usize) = .init(0), + blocked_error: anyerror = error.ResourceTemporarilyUnavailable, + + fn embedSparse(ptr: *anyopaque, alloc: Allocator, embedding_name: []const u8, text: []const u8) !embedder_mod.SparseEmbedding { + const self: *GateSparseEmbedder = @ptrCast(@alignCast(ptr)); + const previous_successes = self.successful_requests.fetchAdd(1, .acq_rel); + if (previous_successes >= self.allowed_successes.load(.acquire)) { + _ = self.successful_requests.fetchSub(1, .acq_rel); + _ = self.blocked_requests.fetchAdd(1, .monotonic); + return self.blocked_error; + } + return try embedder_mod.DeterministicSparseEmbedder.embedSparse(&self.deterministic, alloc, embedding_name, text); + } + + pub fn interface(self: *GateSparseEmbedder) embedder_mod.SparseEmbedder { + return .{ + .ptr = self, + .sparse_embed_fn = embedSparse, + .deinit_fn = null, + }; + } + + pub fn allowAll(self: *GateSparseEmbedder) void { + self.allowed_successes.store(std.math.maxInt(usize), .release); + } +}; + +pub const CountingSparseEmbedder = struct { + deterministic: embedder_mod.DeterministicSparseEmbedder = .{}, + calls: usize = 0, + + fn embedSparse(ptr: *anyopaque, alloc: Allocator, embedding_name: []const u8, text: []const u8) !embedder_mod.SparseEmbedding { + const self: *CountingSparseEmbedder = @ptrCast(@alignCast(ptr)); + self.calls += 1; + return try embedder_mod.DeterministicSparseEmbedder.embedSparse(&self.deterministic, alloc, embedding_name, text); + } + + pub fn interface(self: *CountingSparseEmbedder) embedder_mod.SparseEmbedder { + return .{ + .ptr = self, + .sparse_embed_fn = embedSparse, + .deinit_fn = null, + }; + } +}; + +pub const TestTransactionRecoveryResolver = struct { + pub fn resolve(_: *anyopaque, _: transactions_mod.TxnId, _: []const u8, _: transactions_mod.TxnStatus, _: u64) !void {} +}; + +pub const TxnResolverRecorder = struct { + mutex: std.atomic.Mutex = .unlocked, + calls: u32 = 0, + + pub fn resolve(ctx_ptr: *anyopaque, txn_id: transactions_mod.TxnId, participant: []const u8, status: transactions_mod.TxnStatus, commit_version: u64) anyerror!void { + _ = txn_id; + _ = status; + _ = commit_version; + if (!std.mem.eql(u8, participant, "remote")) return error.UnexpectedParticipant; + const self: *TxnResolverRecorder = @ptrCast(@alignCast(ctx_ptr)); + _ = platform.sync.lockAtomic(&self.mutex); + defer self.mutex.unlock(); + self.calls += 1; + } +}; + +/// Embedder that returns a fixed vector for any text, so a backfilled mention +/// embedding deterministically matches a candidate's name_embedding. +pub const FixedVectorEmbedder = struct { + pub fn interface(self: *FixedVectorEmbedder) embedder_mod.DenseEmbedder { + return .{ .ptr = self, .dense_embed_fn = embed }; + } + + fn embed(ptr: *anyopaque, alloc: Allocator, embedding_name: []const u8, text: []const u8, dims: u32) anyerror![]f32 { + _ = ptr; + _ = embedding_name; + _ = text; + _ = dims; + const v = try alloc.alloc(f32, 4); + v[0] = 1.0; + v[1] = 0.0; + v[2] = 0.0; + v[3] = 0.0; + return v; + } +}; + +/// Thread-safe capturing entity sink for the promotion integration test. +pub const FakePromotionSink = struct { + const Upsert = struct { table: []u8, key: []u8, doc: []u8 }; + + alloc: std.mem.Allocator, + mutex: std.atomic.Mutex = .unlocked, + upserts: std.ArrayListUnmanaged(Upsert) = .empty, + + pub fn deinit(self: *FakePromotionSink) void { + for (self.upserts.items) |u| { + self.alloc.free(u.table); + self.alloc.free(u.key); + self.alloc.free(u.doc); + } + self.upserts.deinit(self.alloc); + } + + pub fn sink(self: *FakePromotionSink) promotion_runtime_mod.EntitySink { + return .{ .ptr = self, .vtable = &vtable }; + } + + const vtable = promotion_runtime_mod.EntitySink.VTable{ .upsert = upsertFn }; + + fn upsertFn(ptr: *anyopaque, allocator: std.mem.Allocator, table: []const u8, key: []const u8, doc_json: []const u8) anyerror!void { + _ = allocator; + const self: *FakePromotionSink = @ptrCast(@alignCast(ptr)); + _ = platform.sync.lockAtomic(&self.mutex); + defer self.mutex.unlock(); + const t = try self.alloc.dupe(u8, table); + errdefer self.alloc.free(t); + const k = try self.alloc.dupe(u8, key); + errdefer self.alloc.free(k); + const d = try self.alloc.dupe(u8, doc_json); + errdefer self.alloc.free(d); + try self.upserts.append(self.alloc, .{ .table = t, .key = k, .doc = d }); + } + + pub fn findKey(self: *FakePromotionSink, key: []const u8) ?[]const u8 { + _ = platform.sync.lockAtomic(&self.mutex); + defer self.mutex.unlock(); + for (self.upserts.items) |u| { + if (std.mem.eql(u8, u.key, key)) return u.doc; + } + return null; + } +}; + +pub const TestAssetProducer = struct { + calls: usize = 0, + generator_calls: usize = 0, + reader_calls: usize = 0, + transcriber_calls: usize = 0, + extractor_calls: usize = 0, + reader_output: ?[]const u8 = null, + transcriber_output: ?[]const u8 = null, + extractor_output: ?[]const u8 = null, + + pub fn producer(self: *@This()) asset_producer_mod.Producer { + return .{ + .ptr = self, + .vtable = &.{ .produce = produce }, + }; + } + + fn produce(ptr: *anyopaque, alloc: Allocator, request: asset_producer_mod.Request) ![]u8 { + const self: *@This() = @ptrCast(@alignCast(ptr)); + self.calls += 1; + switch (request.producer_type) { + .copy => {}, + .document_extraction => {}, + .generator => self.generator_calls += 1, + .reader => self.reader_calls += 1, + .transcriber => self.transcriber_calls += 1, + .extractor => self.extractor_calls += 1, + } + if (request.producer_type == .extractor) { + if (self.extractor_output) |output| return try alloc.dupe(u8, output); + return try std.fmt.allocPrint(alloc, "{{\"relations\":[{{\"type\":\"mentions\",\"target\":{{\"document_id\":{f}}}}}]}}", .{std.json.fmt(request.source_text, .{})}); + } + if (request.producer_type == .reader) { + if (self.reader_output) |output| return try alloc.dupe(u8, output); + } + if (request.producer_type == .transcriber) { + if (self.transcriber_output) |output| return try alloc.dupe(u8, output); + } + return try std.fmt.allocPrint(alloc, "{s}:{s}", .{ @tagName(request.producer_type), request.source_text }); + } +}; + +pub fn SharedReadLockHold(comptime DB: type) type { + return struct { + db: *DB, + acquired: std.atomic.Value(u8) = .init(0), + release: std.atomic.Value(u8) = .init(0), + + pub fn run(self: *@This()) void { + self.db.core.lockApplyShared(); + self.acquired.store(1, .monotonic); + while (self.release.load(.monotonic) == 0) { + spinOrYield(); + } + self.db.core.unlockApplyShared(); + } + }; +} + +pub fn ConcurrentReadProbe(comptime DB: type) type { + return struct { + db: *DB, + started: std.atomic.Value(u8) = .init(0), + done: std.atomic.Value(u8) = .init(0), + failed: std.atomic.Value(u8) = .init(0), + + pub fn runSearch(self: *@This()) void { + self.started.store(1, .monotonic); + var result = self.db.search(std.heap.c_allocator, .{ + .index_name = "ft_v1", + .full_text = .{ .match = .{ .field = "title", .text = "alpha" } }, + }) catch { + self.failed.store(1, .monotonic); + return; + }; + defer result.deinit(); + self.done.store(1, .monotonic); + } + + pub fn runScan(self: *@This()) void { + self.started.store(1, .monotonic); + var result = self.db.scan(std.heap.c_allocator, "", "", .{ + .include_documents = true, + .limit = 10, + }) catch { + self.failed.store(1, .monotonic); + return; + }; + defer result.deinit(std.heap.c_allocator); + self.done.store(1, .monotonic); + } + }; +} + +pub fn ConcurrentWriteProbe(comptime DB: type) type { + return struct { + db: *DB, + started: std.atomic.Value(u8) = .init(0), + done: std.atomic.Value(u8) = .init(0), + failed: std.atomic.Value(u8) = .init(0), + + pub fn runBatch(self: *@This()) void { + self.started.store(1, .monotonic); + self.db.batch(.{ + .writes = &.{ + .{ .key = "doc:b", .value = "{\"title\":\"bravo\"}" }, + }, + }) catch { + self.failed.store(1, .monotonic); + return; + }; + self.done.store(1, .monotonic); + } + }; +} diff --git a/zig/pkg/antfly/src/storage/db/types.zig b/zig/pkg/antfly/src/storage/db/types.zig index e51ce0b6ec..5b1fcd9dba 100644 --- a/zig/pkg/antfly/src/storage/db/types.zig +++ b/zig/pkg/antfly/src/storage/db/types.zig @@ -26,6 +26,7 @@ const shard_mod = @import("../shard.zig"); const transactions_mod = @import("../transactions.zig"); const reranking_mod = @import("antfly_reranking"); const doc_identity_mod = @import("doc_identity.zig"); +const graph_edge_types = @import("graph_edge_types.zig"); const resource_manager_mod = @import("../resource_manager.zig"); const index_repair_status = @import("../../common/index_repair_status.zig"); const dense_native_storage_phase = @import("../../common/dense_native_storage_phase.zig"); @@ -337,23 +338,8 @@ pub fn validateMergeArtifacts(req: BatchRequest) !void { } } -pub const GraphEdgeWrite = struct { - index_name: []const u8, - source: []const u8, - target: []const u8, - edge_type: []const u8, - weight: f64 = 1.0, - created_at: u64 = 0, - updated_at: u64 = 0, - metadata_json: []const u8 = "", -}; - -pub const GraphEdgeDelete = struct { - index_name: []const u8, - source: []const u8, - target: []const u8, - edge_type: []const u8, -}; +pub const GraphEdgeWrite = graph_edge_types.GraphEdgeWrite; +pub const GraphEdgeDelete = graph_edge_types.GraphEdgeDelete; pub const IndexKind = enum { full_text, @@ -1478,6 +1464,8 @@ pub const SearchRequest = struct { dense_queries: []const NamedDenseQuery = &.{}, sparse_queries: []const NamedSparseQuery = &.{}, graph_queries: []const NamedGraphQuery = &.{}, + graph_metric_queries: []const NamedGraphMetricQuery = &.{}, + graph_metric_rerank: ?GraphMetricRerank = null, /// Trusted operator-owned graph admission ceilings. Public request parsing /// never reads these from JSON, and shard transport must not serialize them. graph_execution_limits: @import("../../graph/work_budget.zig").Limits = .{}, @@ -1610,6 +1598,8 @@ const hierarchy_children_rejected_fields = [_][]const u8{ "dense_queries", "sparse_queries", "graph_queries", + "graph_metric_queries", + "graph_metric_rerank", "graph_query_transport", "merge_config", "reranker", @@ -1849,6 +1839,52 @@ pub const NamedGraphQuery = struct { query: graph_query_mod.GraphQuery, }; +pub const GraphMetricFreshness = enum { + published, + fresh, +}; + +pub const GraphMetricQuery = struct { + index_name: []const u8, + metric_name: []const u8, + top_k: u32 = 10, + freshness: GraphMetricFreshness = .published, +}; + +pub const NamedGraphMetricQuery = struct { + name: []const u8, + query: GraphMetricQuery, +}; + +pub const GraphMetricRerank = struct { + index_name: []const u8, + metric_name: []const u8, + freshness: GraphMetricFreshness = .published, + candidate_count: ?u32 = null, + base_weight: f64 = 1.0, + weight: f64 = 1.0, + missing_score: f64 = 0.0, +}; + +pub const graph_metric_rerank_max_candidates: u32 = 10_000; +pub const graph_metric_rerank_default_oversample: u32 = 4; + +pub fn graphMetricRerankCandidateCount(rerank: GraphMetricRerank, offset: u32, limit: u32) u32 { + if (rerank.candidate_count) |count| return count; + const page_boundary = offset +| limit; + const adaptive = offset +| (limit *| graph_metric_rerank_default_oversample); + return @min(graph_metric_rerank_max_candidates, @max(page_boundary, adaptive)); +} + +pub fn validateGraphMetricRerankWindow(rerank: GraphMetricRerank, offset: u32, limit: u32) !void { + const page_boundary = offset +| limit; + if (page_boundary > graph_metric_rerank_max_candidates) return error.QueryCandidateBudgetExceeded; + if (rerank.candidate_count) |count| { + if (count == 0 or count > graph_metric_rerank_max_candidates or count < page_boundary) + return error.InvalidQueryRequest; + } +} + pub const NamedFullTextQuery = struct { name: []const u8, index_name: []const u8, @@ -1874,6 +1910,44 @@ pub const MergeConfig = struct { weights: []const fusion_mod.NamedWeight = &.{}, }; +pub const GraphMetricRerankScoreDetails = struct { + index_name: []u8, + metric_name: []u8, + base_score: f64 = 0.0, + base_weight: f64 = 1.0, + metric_score: ?f64 = null, + metric_score_used: f64 = 0.0, + metric_weight: f64 = 1.0, + missing_score_used: bool = false, + final_score: f64 = 0.0, + published_generation: u64 = 0, + + pub fn clone(self: GraphMetricRerankScoreDetails, alloc: Allocator) !GraphMetricRerankScoreDetails { + const index_name = try alloc.dupe(u8, self.index_name); + errdefer alloc.free(index_name); + const metric_name = try alloc.dupe(u8, self.metric_name); + errdefer alloc.free(metric_name); + return .{ + .index_name = index_name, + .metric_name = metric_name, + .base_score = self.base_score, + .base_weight = self.base_weight, + .metric_score = self.metric_score, + .metric_score_used = self.metric_score_used, + .metric_weight = self.metric_weight, + .missing_score_used = self.missing_score_used, + .final_score = self.final_score, + .published_generation = self.published_generation, + }; + } + + pub fn deinit(self: *GraphMetricRerankScoreDetails, alloc: Allocator) void { + alloc.free(self.index_name); + alloc.free(self.metric_name); + self.* = undefined; + } +}; + pub const SearchHit = struct { id: []u8, /// Internal graph-hydration namespace. Null means the query's source @@ -1883,6 +1957,7 @@ pub const SearchHit = struct { native_text_doc_id: ?u32 = null, /// Higher-is-better relevance score used by every public query path. score: ?f32 = null, + score_details: ?GraphMetricRerankScoreDetails = null, /// Metric-native dense-vector distance. Lower values are better. This is /// retained separately so score ordering never depends on the metric. distance: ?f32 = null, @@ -1899,6 +1974,7 @@ pub const SearchHit = struct { errdefer { alloc.free(cloned.id); if (cloned.source_table) |table| alloc.free(table); + if (cloned.score_details) |*details| details.deinit(alloc); freeIndexScores(alloc, cloned.index_scores); freeJsonValues(alloc, cloned.sort_values); if (cloned.stored_data) |data| alloc.free(data); @@ -1910,6 +1986,7 @@ pub const SearchHit = struct { cloned.doc_ordinal = self.doc_ordinal; cloned.native_text_doc_id = self.native_text_doc_id; cloned.score = self.score; + cloned.score_details = if (self.score_details) |details| try details.clone(alloc) else null; cloned.distance = self.distance; cloned.index_scores = try cloneIndexScores(alloc, self.index_scores); cloned.sort_values = try cloneJsonValues(alloc, self.sort_values); @@ -1937,6 +2014,7 @@ pub const SearchHit = struct { pub fn deinit(self: *SearchHit, alloc: Allocator) void { alloc.free(self.id); if (self.source_table) |table| alloc.free(table); + if (self.score_details) |*details| details.deinit(alloc); freeIndexScores(alloc, self.index_scores); freeJsonValues(alloc, self.sort_values); if (self.stored_data) |data| alloc.free(data); @@ -2200,25 +2278,61 @@ pub const SearchResult = struct { shard_identity_read_generations: []ShardIdentityReadGeneration = &.{}, sort_profile: ?SortProfile = null, graph_results: []GraphSearchResult = &.{}, + graph_metric_results: []GraphMetricResult = &.{}, + graph_metric_rerank_status: ?GraphMetricStatus = null, pub fn deinit(self: *SearchResult) void { for (self.hits) |*hit| hit.deinit(self.alloc); if (self.hits.len > 0) self.alloc.free(self.hits); for (self.graph_results) |*graph_result| graph_result.deinit(self.alloc); if (self.graph_results.len > 0) self.alloc.free(self.graph_results); + for (self.graph_metric_results) |*metric_result| metric_result.deinit(self.alloc); + if (self.graph_metric_results.len > 0) self.alloc.free(self.graph_metric_results); + if (self.graph_metric_rerank_status) |*status| status.deinit(self.alloc); if (self.shard_identity_read_generations.len > 0) self.alloc.free(self.shard_identity_read_generations); self.* = undefined; } }; +pub const GraphMetricScore = struct { + node: []u8, + score: f64, + + pub fn deinit(self: *GraphMetricScore, alloc: Allocator) void { + alloc.free(self.node); + self.* = undefined; + } +}; + +pub const GraphMetricResult = struct { + name: []u8, + index_name: []u8, + metric_name: []u8, + scores: []GraphMetricScore, + status: GraphMetricStatus, + + pub fn deinit(self: *GraphMetricResult, alloc: Allocator) void { + alloc.free(self.name); + alloc.free(self.index_name); + alloc.free(self.metric_name); + for (self.scores) |*score| score.deinit(alloc); + if (self.scores.len > 0) alloc.free(self.scores); + self.status.deinit(alloc); + self.* = undefined; + } +}; + pub const GraphSearchResult = struct { name: []u8, nodes: []graph_query_mod.GraphResultNode = &.{}, + metric_values_slab: []graph_query_mod.GraphMetricValue = &.{}, + metric_value_names: [][]u8 = &.{}, paths: []GraphPath = &.{}, matches: []GraphPatternMatch = &.{}, aggregates: []GraphAggregateResult = &.{}, hits: []SearchHit, total_hits: u32, + metric_status: []GraphMetricStatus = &.{}, truncated: bool = false, /// Detach request-scoped retained-state release hooks at the result @@ -2232,6 +2346,9 @@ pub const GraphSearchResult = struct { alloc.free(self.name); for (self.nodes) |*node| node.deinit(alloc); if (self.nodes.len > 0) alloc.free(self.nodes); + if (self.metric_values_slab.len > 0) alloc.free(self.metric_values_slab); + for (self.metric_value_names) |name| alloc.free(name); + if (self.metric_value_names.len > 0) alloc.free(self.metric_value_names); for (self.paths) |path| paths_mod.freePath(alloc, path); if (self.paths.len > 0) alloc.free(self.paths); for (self.matches) |*match| match.deinit(alloc); @@ -2240,10 +2357,136 @@ pub const GraphSearchResult = struct { if (self.aggregates.len > 0) alloc.free(self.aggregates); for (self.hits) |*hit| hit.deinit(alloc); if (self.hits.len > 0) alloc.free(self.hits); + freeGraphMetricStatuses(alloc, self.metric_status); + self.* = undefined; + } +}; + +pub const GraphMetricStatus = struct { + name: []u8, + state: graph_mod.GraphIndex.GraphMetricState = .not_ready, + phase: graph_mod.GraphIndex.GraphMetricBuildPhase = .idle, + edge_filter: graph_mod.GraphMetricEdgeFilter = .{}, + metadata_version: u32 = 0, + config_fingerprint: u64 = 0, + maintenance_paused: bool = false, + build_queued: bool = false, + published_generation: u64 = 0, + edge_generation: u64 = 0, + target_edge_generation: u64 = 0, + queued_generation: u64 = 0, + building_generation: u64 = 0, + build_job_id: u64 = 0, + build_started_at_ms: u64 = 0, + build_iteration: u32 = 0, + build_lease_expires_at_ms: u64 = 0, + build_worker_id: []const u8 = "", + build_cursor: []const u8 = "", + build_completed_units: u64 = 0, + build_total_units: u64 = 0, + build_pages: []GraphMetricBuildPageStatus = &.{}, + build_pages_truncated: bool = false, + retry_count: u64 = 0, + last_error: []const u8 = "", + progress: f64 = 0.0, + converged: bool = false, + iterations_completed: u32 = 0, + delta: f64 = 0.0, + computed_at_ms: u64 = 0, + last_event: ?graph_mod.GraphIndex.GraphMetricEvent = null, + recent_events: []graph_mod.GraphIndex.GraphMetricEvent = &.{}, + + pub fn cloneAlloc(self: GraphMetricStatus, alloc: Allocator) !GraphMetricStatus { + var out = self; + out.name = try alloc.dupe(u8, self.name); + out.edge_filter = .{}; + out.build_worker_id = ""; + out.build_cursor = ""; + out.build_pages = &.{}; + out.last_error = ""; + out.recent_events = &.{}; + errdefer out.deinit(alloc); + out.edge_filter = try self.edge_filter.cloneAlloc(alloc); + out.build_worker_id = try alloc.dupe(u8, self.build_worker_id); + out.build_cursor = try alloc.dupe(u8, self.build_cursor); + out.last_error = try alloc.dupe(u8, self.last_error); + out.recent_events = try alloc.dupe(graph_mod.GraphIndex.GraphMetricEvent, self.recent_events); + out.build_pages = try cloneGraphMetricBuildPageStatuses(alloc, self.build_pages); + return out; + } + + pub fn deinit(self: *GraphMetricStatus, alloc: Allocator) void { + alloc.free(self.name); + self.edge_filter.deinit(alloc); + if (self.build_worker_id.len > 0) alloc.free(self.build_worker_id); + if (self.build_cursor.len > 0) alloc.free(self.build_cursor); + for (self.build_pages) |*page| page.deinit(alloc); + if (self.build_pages.len > 0) alloc.free(self.build_pages); + if (self.last_error.len > 0) alloc.free(self.last_error); + if (self.recent_events.len > 0) alloc.free(self.recent_events); + self.* = undefined; + } +}; + +pub const GraphMetricBuildPageStatus = struct { + phase: graph_mod.GraphIndex.GraphMetricBuildPhase = .idle, + iteration: u32 = 0, + page_id: u64 = 0, + state: graph_mod.GraphIndex.GraphMetricBuildPageState = .pending, + range_kind: graph_mod.GraphIndex.GraphMetricBuildPageRangeKind = .full, + worker_id: []const u8 = "", + lease_expires_at_ms: u64 = 0, + attempt: u64 = 0, + cursor: []const u8 = "", + completed_units: u64 = 0, + total_units: u64 = 0, + last_error: []const u8 = "", + + pub fn deinit(self: *GraphMetricBuildPageStatus, alloc: Allocator) void { + if (self.worker_id.len > 0) alloc.free(self.worker_id); + if (self.cursor.len > 0) alloc.free(self.cursor); + if (self.last_error.len > 0) alloc.free(self.last_error); self.* = undefined; } }; +pub fn freeGraphMetricStatuses(alloc: Allocator, statuses: []GraphMetricStatus) void { + for (statuses) |*status| status.deinit(alloc); + if (statuses.len > 0) alloc.free(statuses); +} + +pub fn cloneGraphMetricStatuses(alloc: Allocator, statuses: []const GraphMetricStatus) ![]GraphMetricStatus { + const out = try alloc.alloc(GraphMetricStatus, statuses.len); + var initialized: usize = 0; + errdefer { + for (out[0..initialized]) |*status| status.deinit(alloc); + alloc.free(out); + } + for (statuses, 0..) |status, i| { + out[i] = try status.cloneAlloc(alloc); + initialized += 1; + } + return out; +} + +pub fn cloneGraphMetricBuildPageStatuses(alloc: Allocator, source: []const GraphMetricBuildPageStatus) ![]GraphMetricBuildPageStatus { + const out = try alloc.alloc(GraphMetricBuildPageStatus, source.len); + for (out) |*page| page.* = .{}; + errdefer { + for (out) |*page| page.deinit(alloc); + alloc.free(out); + } + for (source, out) |original, *page| { + page.* = original; + page.worker_id = ""; + page.cursor = ""; + page.last_error = ""; + page.worker_id = try alloc.dupe(u8, original.worker_id); + page.cursor = try alloc.dupe(u8, original.cursor); + page.last_error = try alloc.dupe(u8, original.last_error); + } + return out; +} pub const GraphAggregateResult = struct { name: []u8, value: u128, @@ -2540,6 +2783,117 @@ pub const TextMergeStats = struct { max_pending_bytes: u64 = 0, }; +pub const GraphMetricRuntimeRole = enum { + combined, + coordinator, + worker, + worker_pool, +}; + +pub const GraphMetricRuntimeStats = struct { + enabled: bool = false, + role: ?GraphMetricRuntimeRole = null, + runtime_id_hash: u64 = 0, + owner_id_hash: u64 = 0, + lease_key_hash: u64 = 0, + worker_id_hash: u64 = 0, + worker_count: u64 = 0, + lease_owned: bool = false, + has_lease: bool = false, + acquisition_count: u64 = 0, + takeover_count: u64 = 0, + lease_acquire_failures: u64 = 0, + lost_leases: u64 = 0, + last_acquired_ms: u64 = 0, + lease_expires_at_ms: u64 = 0, + lease_renew_after_ms: u64 = 0, + renewal_count: u64 = 0, + started: bool = false, + shutdown: bool = false, + notified: bool = false, + ticks_started: u64 = 0, + ticks_completed: u64 = 0, + durable_progress_ticks: u64 = 0, + idle_ticks: u64 = 0, + error_ticks: u64 = 0, + last_error_name: ?[]const u8 = null, + total_metrics_scanned: u64 = 0, + total_active_builds: u64 = 0, + total_builds_started: u64 = 0, + total_worker_steps: u64 = 0, + total_coordinator_steps: u64 = 0, + total_retired_input_records: u64 = 0, + total_pages_claimed: u64 = 0, + total_pages_completed: u64 = 0, + total_phases_advanced: u64 = 0, + total_published: u64 = 0, + total_failed_builds: u64 = 0, + last_metrics_scanned: u64 = 0, + last_active_builds: u64 = 0, + last_builds_started: u64 = 0, + last_worker_steps: u64 = 0, + last_coordinator_steps: u64 = 0, + last_retired_input_records: u64 = 0, + last_pages_claimed: u64 = 0, + last_pages_completed: u64 = 0, + last_phases_advanced: u64 = 0, + last_published: u64 = 0, + last_failed_builds: u64 = 0, + last_budget_exhausted: bool = false, + + pub fn hasRuntimeFacts(self: @This()) bool { + return self.enabled or + self.role != null or + self.runtime_id_hash != 0 or + self.owner_id_hash != 0 or + self.lease_key_hash != 0 or + self.worker_id_hash != 0 or + self.worker_count != 0 or + self.lease_owned or + self.has_lease or + self.acquisition_count != 0 or + self.takeover_count != 0 or + self.lease_acquire_failures != 0 or + self.lost_leases != 0 or + self.last_acquired_ms != 0 or + self.lease_expires_at_ms != 0 or + self.lease_renew_after_ms != 0 or + self.renewal_count != 0 or + self.started or + self.shutdown or + self.notified or + self.ticks_started != 0 or + self.ticks_completed != 0 or + self.durable_progress_ticks != 0 or + self.idle_ticks != 0 or + self.error_ticks != 0 or + self.last_error_name != null or + self.total_metrics_scanned != 0 or + self.total_active_builds != 0 or + self.total_builds_started != 0 or + self.total_worker_steps != 0 or + self.total_coordinator_steps != 0 or + self.total_retired_input_records != 0 or + self.total_pages_claimed != 0 or + self.total_pages_completed != 0 or + self.total_phases_advanced != 0 or + self.total_published != 0 or + self.total_failed_builds != 0 or + self.last_metrics_scanned != 0 or + self.last_active_builds != 0 or + self.last_builds_started != 0 or + self.last_worker_steps != 0 or + self.last_coordinator_steps != 0 or + self.last_retired_input_records != 0 or + self.last_pages_claimed != 0 or + self.last_pages_completed != 0 or + self.last_phases_advanced != 0 or + self.last_published != 0 or + self.last_failed_builds != 0 or + self.last_budget_exhausted; + } +}; + pub fn accumulateTextMergeStats(dst: *TextMergeStats, src: TextMergeStats) void { dst.enabled = dst.enabled or src.enabled; dst.active_indexes +|= src.active_indexes; @@ -2662,6 +3016,7 @@ pub const DBStats = struct { ttl_cleanup: TTLCleanupStats = .{}, transaction_recovery: TransactionRecoveryStats = .{}, text_merge: TextMergeStats = .{}, + graph_metric_runtime: GraphMetricRuntimeStats = .{}, term_doc_freq_cache_hits: u64 = 0, term_doc_freq_cache_misses: u64 = 0, async_indexing: AsyncIndexingStats = .{}, @@ -3379,6 +3734,7 @@ pub const DBIndexStats = struct { algebraic_graph_traversal_rejected_count: u64 = 0, algebraic_graph_traversal_fallback_count: u64 = 0, algebraic_graph_traversal_result_node_count: u64 = 0, + graph_metric_status: []GraphMetricStatus = &.{}, algebraic_observed_query_shape_count: u64 = 0, algebraic_recommendation_count: u64 = 0, algebraic_adaptive_candidate_count: u64 = 0, @@ -3935,6 +4291,7 @@ pub fn freeResolverReplayDiagnostics(alloc: Allocator, stats: ResolverReplayDiag } pub fn freeDBIndexStatsItem(alloc: Allocator, item: DBIndexStats) void { + freeGraphMetricStatuses(alloc, item.graph_metric_status); alloc.free(item.name); for (item.source_replay) |source| alloc.free(source.artifact_name); if (item.source_replay.len > 0) alloc.free(item.source_replay); @@ -3988,3 +4345,22 @@ pub fn freeDBStats(alloc: Allocator, stats: DBStats) void { for (stats.indexes) |item| freeDBIndexStatsItem(alloc, item); if (stats.indexes.len > 0) alloc.free(stats.indexes); } + +test "graph metric index stats cleanup owns nested status payloads" { + const alloc = std.testing.allocator; + var item: DBIndexStats = .{ .name = try alloc.dupe(u8, "graph_idx"), .kind = .graph }; + defer freeDBIndexStatsItem(alloc, item); + const statuses = blk: { + const owned = try alloc.alloc(GraphMetricStatus, 1); + errdefer alloc.free(owned); + owned[0] = .{ .name = try alloc.dupe(u8, "pagerank") }; + break :blk owned; + }; + item.graph_metric_status = statuses; + statuses[0].build_worker_id = try alloc.dupe(u8, "worker"); + statuses[0].build_cursor = try alloc.dupe(u8, "cursor"); + statuses[0].last_error = try alloc.dupe(u8, "diagnostic"); + statuses[0].build_pages = try alloc.alloc(GraphMetricBuildPageStatus, 1); + statuses[0].build_pages[0] = .{}; + statuses[0].build_pages[0].cursor = try alloc.dupe(u8, "page-cursor"); +} diff --git a/zig/pkg/antfly/src/storage/lsm_backend.zig b/zig/pkg/antfly/src/storage/lsm_backend.zig index c065c0440b..8732c6520c 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend.zig @@ -1479,6 +1479,9 @@ pub const Backend = struct { }; allocator: Allocator, + /// Opt-in gate shared by every graph-store view of this backend. Durable + /// roots already enforce a single open writer across processes/handles. + serialized_write_mutex: std.atomic.Mutex = .unlocked, mu: std.atomic.Mutex = .unlocked, // Cached score used by best-effort maintenance scheduling and metrics. // A value of 1 can still mean "known debt, exact score not refreshed yet". @@ -11184,6 +11187,43 @@ test "lsm backend probe owns active mutable point values across later writes" { try std.testing.expectEqual(@as(u64, 2), after.point_value_copies - before.point_value_copies); } +test "graph metric sorted batch presence avoids value retention and respects overlays" { + const alloc = std.testing.allocator; + var storage = storage_io.MemoryStorage.init(alloc); + defer storage.deinit(); + var cache = Cache.init(alloc, DefaultCacheSizeBytes); + defer cache.deinit(); + var backend = try Backend.open(alloc, "/graph-presence-batch", .{ .flush_threshold = 1, .storage = storage.storage(), .cache = &cache }); + defer backend.close(); + var runtime = try backend.runtimeStore(alloc, .{ .name = "graph" }); + defer runtime.deinit(); + const payload = [_]u8{'x'} ** (64 * 1024); + { + var write = try runtime.beginWrite(); + errdefer write.abort(); + try write.put("a", &payload); + try write.put("b", &payload); + try write.commit(); + } + while (try backend.runMaintenanceStep()) {} + var batch = try runtime.beginBatch(); + defer batch.abort(); + try std.testing.expect(batch.vtable.contains_many_sorted != null); + const before = backend.snapshotReadStats(); + var present: [4]bool = undefined; + try batch.containsManySorted(&.{ "a", "a", "b", "missing" }, &present); + try std.testing.expectEqualSlices(bool, &.{ true, true, true, false }, &present); + const after = backend.snapshotReadStats(); + try std.testing.expectEqual(@as(u64, 0), after.point_value_copies - before.point_value_copies); + try batch.delete("a"); + try batch.put("c", "overlay"); + try batch.containsManySorted(&.{ "a", "b", "c", "missing" }, &present); + try std.testing.expectEqualSlices(bool, &.{ false, true, true, false }, &present); + try std.testing.expectError(error.InvalidBatch, batch.containsManySorted(&.{ "b", "a" }, present[0..2])); + try std.testing.expectError(error.InvalidBatch, batch.containsManySorted(&.{"a"}, &present)); + try batch.containsManySorted(&.{}, present[0..0]); +} + test "lsm backend stable probe batch borrows pinned run values without recopying" { var storage = storage_io.MemoryStorage.init(std.testing.allocator); defer storage.deinit(); diff --git a/zig/pkg/antfly/src/storage/lsm_backend/runtime.zig b/zig/pkg/antfly/src/storage/lsm_backend/runtime.zig index 67d66f1642..4bb37edc4b 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/runtime.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/runtime.zig @@ -3960,6 +3960,57 @@ pub fn BoundWriteTxn(comptime BackendType: type) type { return self.backend.getMergedWithOverlay(&self.backend.mutable, &self.mutable, self.namespace, key); } + pub fn containsManySorted(self: *@This(), keys: []const []const u8, present: []bool) !void { + if (self.closed) return error.TransactionClosed; + if (keys.len != present.len or !keysAreSorted(keys)) return error.InvalidBatch; + @memset(present, false); + self.backend.recordGetManySorted(keys.len); + self.backend.recordGetManySortedLocality(keys); + var offset: usize = 0; + while (offset < keys.len) { + const end = @min(keys.len, offset + 256); + // Values only borrow the live view while locked. Pins, table + // decode scratch and run metadata are released after each page, + // never appended to the write batch's retained value inventory. + const locked = lockBackend(BackendType, self.backend); + defer unlockBackend(BackendType, self.backend, locked); + var layout = try CurrentReadLayout(BackendType).init(self.backend, self.allocator); + defer layout.deinit(); + var blocks = std.ArrayListUnmanaged(cache_mod.Handle).empty; + defer releaseHeldBlocks(&blocks, self.backend.allocator); + var values = std.ArrayListUnmanaged([]u8).empty; + defer { + for (values.items) |value| self.allocator.free(value); + values.deinit(self.allocator); + } + var indexes = RunBatchIndexHandles{ .allocator = self.metadata_allocator }; + defer indexes.deinit(); + var group: ?usize = null; + var hint: ?BorrowedReadHint = null; + for (keys[offset..end], present[offset..end]) |key, *exists| { + var bulk = self.bulk_appends.entries.items.len; + while (bulk != 0) { + bulk -= 1; + const entry = self.bulk_appends.entries.items[bulk]; + if (compareEntryTo(entry, self.namespace, key) == .eq) { + exists.* = !entry.tombstone; + break; + } + } else { + if (self.mutable.findIndex(self.namespace, key)) |i| { + exists.* = !self.mutable.entries.items[i].tombstone; + continue; + } + exists.* = if (getFromSnapshotRuns(self.backend, &self.backend.mutable, layout.immutable_memtables, layout.runs, layout.l0_groups, layout.levels, &group, &hint, &blocks, &values, self.allocator, self.namespace, key, true, &indexes)) |_| true else |err| switch (err) { + error.NotFound => false, + else => return err, + }; + } + } + offset = end; + } + } + pub fn getManySorted(self: *@This(), keys: []const []const u8, values: []?[]const u8) !void { if (self.closed) return error.TransactionClosed; if (keys.len != values.len) return error.InvalidBatch; diff --git a/zig/pkg/antfly/src/storage/mem_backend.zig b/zig/pkg/antfly/src/storage/mem_backend.zig index 0b48db2729..c457673c06 100644 --- a/zig/pkg/antfly/src/storage/mem_backend.zig +++ b/zig/pkg/antfly/src/storage/mem_backend.zig @@ -93,6 +93,7 @@ fn releaseState(allocator: Allocator, rc: *RcState) void { } pub const Backend = struct { + serialized_write_mutex: std.atomic.Mutex = .unlocked, allocator: Allocator, open_options: backend_types.OpenOptions, state: ?*RcState = null, diff --git a/zig/pkg/antfly/src/storage/test_manifest.zig b/zig/pkg/antfly/src/storage/test_manifest.zig index f745d8c343..95c7370cb5 100644 --- a/zig/pkg/antfly/src/storage/test_manifest.zig +++ b/zig/pkg/antfly/src/storage/test_manifest.zig @@ -93,10 +93,12 @@ comptime { _ = @import("db/enrichment/enrichment_worker.zig"); _ = @import("db/enrichment/utf8_text.zig"); _ = @import("db/generation_lifecycle.zig"); + _ = @import("db/graph_runtime.zig"); _ = @import("db/graph_asset_state.zig"); _ = @import("db/graph_edge_contender.zig"); _ = @import("db/graph_state_name.zig"); _ = @import("db/lease.zig"); + _ = @import("db/maintenance/graph_metric_runtime.zig"); _ = @import("db/maintenance/sparse_compaction_runtime.zig"); _ = @import("db/maintenance/transaction_runtime.zig"); _ = @import("db/maintenance/ttl_runtime.zig"); diff --git a/zig/pkg/antfly/src/vopr/object_store.zig b/zig/pkg/antfly/src/vopr/object_store.zig index e4f747b4c6..87d9d09bc1 100644 --- a/zig/pkg/antfly/src/vopr/object_store.zig +++ b/zig/pkg/antfly/src/vopr/object_store.zig @@ -1,5 +1,16 @@ // Copyright 2026 Antfly, Inc. -// SPDX-License-Identifier: Elastic-2.0 +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. //! Production serverless object-store protocols over the reusable scripted //! objectstore fault client. Fault selection belongs to VOPR; artifacts, @@ -94,22 +105,25 @@ test "serverless object store VOPR composes real artifact manifest WAL and progr defer artifacts.deinit(); // A short provider write followed by timeout must not be mistaken for a - // published content-addressed artifact. Retrying replaces it with the full - // immutable body, so checksum-derived identity and bytes agree. + // published content-addressed artifact. Immutable create-only writes must + // reject the corrupt occupant rather than overwrite an identity that a + // reader may already have pinned. A separate valid artifact exercises the + // remaining publication protocols. faults.partiallyCommitNextPut(3, error.Timeout); try std.testing.expectError(error.Timeout, artifacts.put("complete-artifact")); - var artifact = try artifacts.put("complete-artifact"); + try std.testing.expectError(error.ArtifactIntegrityMismatch, artifacts.put("complete-artifact")); + var artifact = try artifacts.put("valid-artifact"); defer artifact.deinit(alloc); faults.failNextGet(error.Canceled); try std.testing.expectError(error.Canceled, artifacts.getAlloc(artifact.artifact_id)); const artifact_bytes = try artifacts.getAlloc(artifact.artifact_id); defer alloc.free(artifact_bytes); - try std.testing.expectEqualStrings("complete-artifact", artifact_bytes); + try std.testing.expectEqualStrings("valid-artifact", artifact_bytes); - // Duplicate provider completion is harmless for an unconditional - // content-addressed artifact write. + // Duplicate provider completion is harmless for a create-only artifact + // write: an authenticated existing body satisfies the retry. faults.duplicateNextPut(); - var duplicate = try artifacts.put("complete-artifact"); + var duplicate = try artifacts.put("valid-artifact"); defer duplicate.deinit(alloc); try std.testing.expectEqualStrings(artifact.artifact_id, duplicate.artifact_id); diff --git a/zig/tools/test_run_bounded_zig_build.py b/zig/tools/test_run_bounded_zig_build.py index 5cb204940e..03dfe8f556 100644 --- a/zig/tools/test_run_bounded_zig_build.py +++ b/zig/tools/test_run_bounded_zig_build.py @@ -3,6 +3,7 @@ import importlib.util import io import os +import re import sys import unittest from pathlib import Path @@ -19,6 +20,21 @@ class BoundedZigBuildTest(unittest.TestCase): + def test_ci_scheduler_caps_admit_the_storage_compile_claim(self): + build = (SCRIPT.parents[1] / "build.zig").read_text(encoding="utf-8") + workflow = (SCRIPT.parents[2] / ".github/workflows/zig-tests.yml").read_text( + encoding="utf-8" + ) + claim = re.search(r"\.distributed => (\d+) \* 1024 \* 1024 \* 1024", build) + self.assertIsNotNone( + claim, "update this contract when storage claims change shape" + ) + required = int(claim.group(1)) * 1024**3 + caps = re.findall(r"--max-rss-cap (\d+)", workflow) + self.assertTrue(caps) + for cap in caps: + self.assertGreaterEqual(int(cap), required) + def test_environment_override_is_used_as_exact_budget(self): with mock.patch.dict(os.environ, {launcher.MAX_RSS_ENV: "123456"}): self.assertEqual(123456, launcher.detect_max_rss()) @@ -56,6 +72,18 @@ def test_uncapped_build_uses_detected_host_budget(self): ): self.assertEqual(32_000, launcher.detect_max_rss()) + def test_ci_cap_admits_storage_kernel_without_overriding_small_cgroup(self): + cap = 22 * 1024 * 1024 * 1024 + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch.object( + launcher, "detect_memory_limit", return_value=64 * 1024**3 + ): + self.assertEqual(cap, launcher.detect_max_rss(cap)) + with mock.patch.object( + launcher, "detect_memory_limit", return_value=16 * 1024**3 + ): + self.assertEqual(int(16 * 1024**3 * 0.8), launcher.detect_max_rss(cap)) + def test_command_adds_missing_scheduler_options(self): command = launcher.build_command( "zig",