-
Notifications
You must be signed in to change notification settings - Fork 16
Expand file tree
/
Copy pathcheck.ts
More file actions
3214 lines (3049 loc) · 189 KB
/
Copy pathcheck.ts
File metadata and controls
3214 lines (3049 loc) · 189 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bun
/**
* check.ts — the single entry point for "run everything that verifies this repo".
*
* bun run check.ts # `local` tier (default) — run before considering work done
* bun run check.ts fast # what CI runs — the absolute-minimum commit gate
* bun run check.ts full # `local` + every manual-only gate (the real "run all tests")
*
* CI (`.github/workflows/build.yml`) runs EXACTLY `bun run check.ts fast` — nothing else. The
* registry below IS the encoding of every gate that verifies this repo; the tiers
* (fast ⊂ local ⊂ full) are views over it. POLICY: the fast tier stays the absolute minimum (sole
* maintainer; AI-velocity commits; CI minutes are the scarce resource) — new gates default to
* `local` or `full`, and promoting anything into `fast` is a maintainer decision. Honest reporting
* is the point: every run prints the FULL registry as a table (PASS / FAIL / SKIPPED / STUB /
* not-in-tier + durations), so a gate that did not run is always *visibly* not-run — the failure
* mode this runner cures (a gate that exists but is in nobody's habit) is impossible to miss.
*
* Flags:
* --keep-going run all in-tier gates even after a failure (default: fail fast — later cargo
* gates depend on the build, so stopping early avoids cascade noise).
* --skip-missing downgrade a missing verify.ts oracle from FAIL to SKIPPED(oracle absent).
* --refresh-fuzz re-run fuzz/generate.sh before the fuzz gates even if generated/ exists.
* --cache-transparency enable the flag-gated `verify_cache_transparency` full-tier gate (two verify
* runs — cached vs GATE_CACHE=0 — asserted byte-identical; otherwise SKIPPED).
* --only <a,b> run ONLY these gates, in registry order, among the named tier's in-tier set
* (`--only a,b` or `--only=a,b`). A selected run is NEVER a tier run: the full
* registry still prints with the deselected gates as `NOT RUN (--only)`, the
* SUMMARY header says PARTIAL, the self-log is `check-only-<stamp>.log` (outside
* every tier median), and the last line is a receipt, not the tier verdict. Cite
* it as "gates X, Y ran green" — never as a tier verdict. See "GATE SELECTION".
*
* CONCURRENCY: gates run one at a time UNLESS the registry says otherwise. A gate may declare a
* `concurrent` group; gates in the same group, contiguous in the registry, run as one batch with at
* most `CHECK_JOBS` (default 4) in flight, slowest-measured first. Everything else is a barrier, so
* registry order still means what it says and the fmt→clippy→build→test chain stays strictly ordered.
* Today exactly one group exists: the `#[ignore]`d manual-only heavy gates. A batched gate's output
* is BUFFERED and emitted as one block on completion — interleaved cargo output is unreadable and
* would mis-attribute the per-gate cache rollups the timings parser reads. Bound is memory, not
* cores: `CHECK_JOBS=1` restores fully sequential execution.
*
* PEAK RESOURCE: what has to stay bounded is not the gate count but the PRODUCT
* `(gates in flight) × (rustc per gate) × (per-rustc resident set)` — so each batched gate is handed
* a memory-derived `CARGO_BUILD_JOBS` (`CHECK_CARGO_JOBS` overrides), and neither factor scales with
* `nproc`. The sequential path is untouched. A `local`/`full` run also preflights free memory and
* free scratch space up front: below the memory floor it degrades to sequential, below the disk floor
* it refuses, because a tier commits to its peak in its first seconds and cannot discover a cap
* mid-run. `CHECK_SKIP_PREFLIGHT=1` bypasses both floors; `fast` (CI) is untouched.
*
* LOGGING: every run tees its FULL output to a timestamped `draft/logs/check-<tier>-<stamp>.log`
* (`check-only-<stamp>.log` for a `--only` run — a name outside the tier regex on purpose, see
* "GATE SELECTION"; path printed at start and end) — evidence preservation is the tool's job, not a piping habit.
* Never pipe a run through `tail`/`grep` as its only capture; cite the printed log path instead.
*
* FUZZING: the `full` tier walks the byte-fuzzer's two targets live (`fuzz_bounded_run`), bounded to
* `FUZZ_BUDGET_S` seconds per target (default 120) and run one libFuzzer process at a time. It is a
* smoke-walk of the reachable hostile-input surface, not the periodic deep run — that stays manual
* (`fuzz/README.md`). Deliberately not gate-cached: a randomized exploration is not a pure function
* of the tree's bytes.
*
* NETWORK: local/full runs start with a retried dep-universe-lock refresh plus `cargo fetch` warm-up
* (workspace + fuzz + tests/warmup dep-universe manifest), then force CARGO_NET_OFFLINE=true for every gate — nested-
* cargo cells resolve from the cargo cache instead of hitting crates.io per cell, which removes
* the registry-transient flake class outright (tests/README.md § "Offline-after-warmup").
* CHECK_ONLINE=1 skips the offline forcing; a pre-set CARGO_NET_OFFLINE=true skips the fetch.
* The fast tier (CI) is untouched.
*
* SELF-COMPLETENESS (the systematic catch, TDD): the first gate `self_checks` runs eight meta-checks
* so a new gate that nobody registers fails the run rather than silently not existing:
* 1. ignored-test classification — every `#[ignore]` test must be registered here as either a
* manual gate (run it) or a known-failing stub (never run it, shown as STUB).
* 2. matrix-script coverage — every `cddl-matrix/*.ts` (minus lib.ts) must be wired to a gate.
* 3. CI-is-fast-tier invariant — build.yml must invoke `bun run check.ts fast` and must contain
* NO other run step (all CI work flows through the registry's fast tier, so growing CI is an
* explicit, reviewed registry edit — not a workflow edit agents make in passing). It says
* nothing about the workflow's `paths:` trigger filter, which is not a run step: covering the
* trees a fast gate READS is a filter edit, and the promoted doc scanners depend on one.
* 4. concurrency declarations are well-formed (`cmd`-only, group members contiguous).
* 5. `requires:` edges are well-formed — the `--only` dependency fence's only enforcement.
* 6. registry/readme integrity — every gate is named in `tests/README.md`, and prose next to a
* concurrent group carries no authored cardinal count that can drift from the registry.
* 7. ignored-draft index ban — Git's index must contain no `draft/` path. `.gitignore` protects
* ordinary adds but not `git add -f`; checking the index catches both staged and committed
* violations before they can become another history-rewrite incident.
* 8. new-fixture advisory — a Git-added direct `tests/<dir>/input.cddl` names the known
* local-tier corpus-parity verdict while the author can still add its registry row.
*
* Meta-checks mutation-verified red-first at landing (repo idiom):
* - adding a throwaway `#[ignore]` test -> meta-check 1 FAILED (unclassified ignore)
* - adding a throwaway `cddl-matrix/throwaway.ts` -> meta-check 2 FAILED (script wired to no gate)
* - adding a direct `run: cargo test` step to build.yml -> meta-check 3 FAILED (bypasses registry)
* canaries reverted after confirming red.
*/
import {
existsSync, mkdirSync, readFileSync, readdirSync, readlinkSync, rmSync, statSync,
unlinkSync, writeFileSync,
} from "node:fs";
import { homedir, tmpdir } from "node:os";
import { basename, join, relative, resolve } from "node:path";
import {
appendRows, cellCountsFor, compactDur, keptRunKeys, machineId, parseLog, readCells, readDigest,
readLedger, runDigestUpdate, runKeysInOrder, splitLogName, tierWindow, trimCellLines, trimRows,
upsert, writeLedger,
KEEP_RUNS_IN_CELLS, type Digest, type GateRow, type Row, type RunRow,
} from "./cddl-matrix/project_timings.ts";
import { runNoStdCheckGate } from "./cddl-matrix/no_std_check.ts";
const ROOT = import.meta.dir;
const MATRIX = join(ROOT, "cddl-matrix");
const LOGS_DIR = join(ROOT, "draft", "logs");
const LEDGER = join(ROOT, "draft", "timings.jsonl");
const CELLS = join(ROOT, "draft", "timing-cells.jsonl");
// ---- tiers ---------------------------------------------------------------------------------------
const TIERS = ["fast", "local", "full"] as const;
export type Tier = (typeof TIERS)[number];
const rank = (t: Tier) => TIERS.indexOf(t);
// ---- argv ------------------------------------------------------------------------------------------
export interface ParsedArgv { flags: Set<string>; positional: string[]; only: string[] | null }
/**
* argv -> (flags, positional, `--only` selection).
*
* argv is WALKED rather than partitioned by a `startsWith("--")` filter because `--only` is the one
* flag that takes a value: in the space spelling (`--only a,b`) that value is not a flag, and the
* filter would hand it to the tier positional — a selection would silently become "unknown tier".
* Both spellings are accepted (`--only a,b` and `--only=a,b`), and ids may also be split across
* repeated flags. Every argv consumer in this file goes through here, including the two `fn` gates
* that read the tier off argv themselves, so there is exactly one place that knows this shape.
*/
export function parseArgv(argv: string[]): ParsedArgv {
const flags = new Set<string>();
const positional: string[] = [];
let only: string[] | null = null;
const add = (v: string): void => {
only = [...(only ?? []), ...v.split(",").map(s => s.trim()).filter(Boolean)];
};
for (let i = 0; i < argv.length; i++) {
const a = argv[i]!;
if (a === "--only") {
flags.add("--only");
const v = argv[i + 1];
// A valueless `--only` yields an EMPTY selection rather than swallowing the next flag; the
// resolver rejects it by name, which is a better message than "unknown flag".
if (v !== undefined && !v.startsWith("--")) { add(v); i++; } else only ??= [];
continue;
}
if (a.startsWith("--only=")) { flags.add("--only"); add(a.slice("--only=".length)); continue; }
if (a.startsWith("--")) { flags.add(a); continue; }
positional.push(a);
}
return { flags, positional, only };
}
/** The tier a run is at, read off argv the way `main` reads it. */
export function tierFromArgv(argv: string[]): Tier {
const p = parseArgv(argv).positional[0];
return TIERS.includes(p as Tier) ? (p as Tier) : "local";
}
// ---- gate model ----------------------------------------------------------------------------------
// A FOURTH not-run flavour beside `NOT_IN_TIER`, `SKIPPED (earlier failure; fail-fast)` and
// `SKIPPED (reason)`: a gate the run's `--only` selection deliberately left out. Never a reuse of
// SKIPPED — a deliberate omission must not read as an incidental one — and never `not-in-tier`,
// which says the opposite (this gate WOULD have run in a complete run of this tier).
type Status = "PASS" | "FAIL" | "SKIPPED" | "STUB" | "NOT_IN_TIER" | "NOT_RUN_ONLY";
interface Outcome { status: Status; reason?: string }
interface Opts { skipMissing: boolean; refreshFuzz: boolean; cacheTransparency: boolean }
export interface Gate {
id: string;
tier: Tier;
kind: "cmd" | "fn" | "stub";
desc: string;
cmd?: string[]; // kind === "cmd"
cwd?: string; // kind === "cmd"; defaults to ROOT
run?: (o: Opts) => Outcome;// kind === "fn"
ignoredTest?: string; // maps this gate to a `#[ignore]` test (meta-check 1)
script?: string; // cddl-matrix/*.ts this gate drives (meta-check 2)
/**
* OPT-IN gate-level concurrency: this gate may run concurrently with the OTHER gates naming the
* same group. Absent (the default for every gate) means sequential — today's behaviour, byte for
* byte, including inherited stdout. The registry is the encoding of what verifies this repo, so
* concurrency is declared here and is as visible as tier membership; nothing infers it.
*
* `cmd` gates only, and group members must be CONSECUTIVE in the registry (both enforced by
* meta-check 4). Consecutiveness is what keeps registry order meaningful: an ungrouped gate is a
* BARRIER, so `verify` still finishes before `verify_cache_transparency` starts and the
* fmt→clippy→build→test chain stays strictly ordered. A member separated from its group by a
* barrier would run alone — a declaration that silently does nothing, which is the failure class
* meta-check 4 exists to make impossible.
*/
concurrent?: string;
/**
* Gates this one READS THE OUTPUT OF, and therefore cannot be selected without (`--only`).
*
* The registry encodes execution ORDER and concurrency; a tier run masks data dependencies by
* always running whole prefixes, so nothing had to state them until selection existed. A split
* pair does not fail loudly — `coverage_md_diff` alone passes vacuously against a stale-but-
* committed COVERAGE.md — which is why the refusal is hard (v1: refuse, never auto-include) and
* why the `why` is carried here rather than in the error site: the message has to say what the
* split would silently have asserted.
*
* Established by enumerating the registry, not by grep: the pairs are exactly the gates whose
* verdict depends on a file or cache another gate WRITES in the same run. Everything else either
* reads only committed files or owns its scratch root. Meta-check 5 keeps the field honest.
*/
requires?: { gate: string; why: string }[];
}
const CORPUS_PARITY_OWNER_TEST = "wasm_api_parity_axes_and_pins_are_live";
const CORPUS_PARITY_OWNER_GATE = "test";
/** A Git discovery candidate, kept explicit so self-tests pin the added-only boundary. */
export interface NewFixtureCandidate {
status: "added" | "modified" | "deleted";
path: string;
}
/**
* The one proven new-file advisory. This deliberately names the known enumerating test rather than
* pretending `check.ts` can infer arbitrary test ownership from a path. The registry remains the
* single source of truth for the owning gate's tier.
*/
export function corpusParityNewFixtureAdvisories(
candidates: readonly NewFixtureCandidate[],
registry: readonly Pick<Gate, "id" | "tier">[],
): string[] {
const owner = registry.find(g => g.id === CORPUS_PARITY_OWNER_GATE);
if (!owner)
throw new Error(
`new-fixture advisory owner gate '${CORPUS_PARITY_OWNER_GATE}' for ` +
`'${CORPUS_PARITY_OWNER_TEST}' is missing from the live registry`,
);
const paths = [...new Set(
candidates
.filter(c => c.status === "added")
.map(c => c.path)
.filter(path => /^tests\/[^/]+\/input\.cddl$/.test(path)),
)].sort();
return paths.map(path =>
`new-fixture advisory: '${path}' must be added to CORPUS_PARITY_INPUTS or ` +
`CORPUS_PARITY_EXCLUDED; then run '${CORPUS_PARITY_OWNER_TEST}' ` +
`(gate '${owner.id}', ${owner.tier} tier) or the complete ${owner.tier} tier — ` +
`fast does not execute this verdict`,
);
}
/** Pure controls for the advisory's event boundary, ordering, and registry-derived tier. */
export function corpusParityNewFixtureAdvisoriesSelftest(): void {
const localOwner = [{ id: CORPUS_PARITY_OWNER_GATE, tier: "local" as Tier }];
const expected =
"new-fixture advisory: 'tests/new-fixture/input.cddl' must be added to " +
"CORPUS_PARITY_INPUTS or CORPUS_PARITY_EXCLUDED; then run " +
"'wasm_api_parity_axes_and_pins_are_live' (gate 'test', local tier) or the complete " +
"local tier — fast does not execute this verdict";
const one = corpusParityNewFixtureAdvisories(
[{ status: "added", path: "tests/new-fixture/input.cddl" }], localOwner,
);
if (one.length !== 1 || one[0] !== expected)
throw new Error(`new-fixture advisory self-test produced ${JSON.stringify(one)}, expected exactly ${JSON.stringify([expected])}`);
const stable = corpusParityNewFixtureAdvisories([
{ status: "added", path: "tests/zeta/input.cddl" },
{ status: "added", path: "tests/alpha/input.cddl" },
{ status: "added", path: "tests/zeta/input.cddl" },
], localOwner);
if (stable.length !== 2 || !stable[0]!.includes("tests/alpha/input.cddl") || !stable[1]!.includes("tests/zeta/input.cddl"))
throw new Error(`new-fixture advisory self-test did not deduplicate and sort: ${JSON.stringify(stable)}`);
const ignored = corpusParityNewFixtureAdvisories([
{ status: "modified", path: "tests/modified/input.cddl" },
{ status: "deleted", path: "tests/deleted/input.cddl" },
{ status: "added", path: "tests/input.cddl" },
{ status: "added", path: "tests/nested/child/input.cddl" },
{ status: "added", path: "tests/other/not-input.cddl" },
{ status: "added", path: "elsewhere/input.cddl" },
], localOwner);
if (ignored.length)
throw new Error(`new-fixture advisory self-test accepted a non-added or near-miss path: ${JSON.stringify(ignored)}`);
const fullTier = corpusParityNewFixtureAdvisories(
[{ status: "added", path: "tests/tier-proof/input.cddl" }],
[{ id: CORPUS_PARITY_OWNER_GATE, tier: "full" }],
);
if (
fullTier.length !== 1 ||
!fullTier[0]!.includes("(gate 'test', full tier)") ||
!fullTier[0]!.includes("complete full tier") ||
fullTier[0]!.includes("local tier")
)
throw new Error(`new-fixture advisory self-test did not derive the owner tier: ${JSON.stringify(fullTier)}`);
let missingOwner = false;
try {
corpusParityNewFixtureAdvisories([{ status: "added", path: "tests/missing/input.cddl" }], []);
} catch (error) {
missingOwner = String(error).includes("owner gate 'test'");
}
if (!missingOwner)
throw new Error("new-fixture advisory self-test did not reject a missing owner gate");
}
/** The registry fields the README-integrity check owns; kept small for synthetic self-tests. */
export interface GateReadmeLintGate {
id: string;
concurrent?: string;
}
interface TextRange { start: number; end: number }
interface CodeSpan extends TextRange { text: string }
/** Fenced blocks are not prose and cannot contribute inline-code spans. */
function markdownFencedCodeRanges(markdown: string): TextRange[] {
const ranges: TextRange[] = [];
let open: { start: number; marker: "`" | "~"; length: number } | undefined;
let offset = 0;
for (const lineWithEnding of markdown.matchAll(/[^\n]*(?:\n|$)/g)) {
const line = lineWithEnding[0].replace(/\n$/, "").replace(/\r$/, "");
const fence = line.match(/^ {0,3}(`{3,}|~{3,})/);
if (!open && fence) {
open = { start: offset, marker: fence[1][0] as "`" | "~", length: fence[1].length };
} else if (open && fence && fence[1][0] === open.marker && fence[1].length >= open.length &&
new RegExp(`^ {0,3}${open.marker}{${open.length},}[ \\t]*$`).test(line)) {
ranges.push({ start: open.start, end: offset + lineWithEnding[0].length });
open = undefined;
}
offset += lineWithEnding[0].length;
}
if (open) ranges.push({ start: open.start, end: markdown.length });
return ranges;
}
/**
* CommonMark-style code spans outside fences. A delimiter run closes only against an ENTIRE run of
* the same length, never a prefix of a longer run. Newlines become spaces; a single padding space
* is stripped at both ends unless the span is all spaces, matching CommonMark's code-span rule.
*/
function markdownCodeSpans(markdown: string, fences = markdownFencedCodeRanges(markdown)): CodeSpan[] {
const spans: CodeSpan[] = [];
let fenceIndex = 0;
for (let i = 0; i < markdown.length;) {
while (fenceIndex < fences.length && fences[fenceIndex].end <= i) fenceIndex++;
if (fenceIndex < fences.length && i >= fences[fenceIndex].start) {
i = fences[fenceIndex].end;
continue;
}
if (markdown[i] !== "`") {
i++;
continue;
}
let openerEnd = i + 1;
while (markdown[openerEnd] === "`") openerEnd++;
const delimiterLength = openerEnd - i;
let closeStart = openerEnd;
let closed = false;
while (closeStart < markdown.length) {
if (fenceIndex < fences.length && closeStart >= fences[fenceIndex].start) break;
if (markdown[closeStart] !== "`") {
closeStart++;
continue;
}
let closeEnd = closeStart + 1;
while (markdown[closeEnd] === "`") closeEnd++;
if (closeEnd - closeStart === delimiterLength) {
let text = markdown.slice(openerEnd, closeStart).replace(/\r\n?|\n/g, " ");
if (text.length > 1 && text.startsWith(" ") && text.endsWith(" ") && /[^ ]/.test(text))
text = text.slice(1, -1);
spans.push({ start: i, end: closeEnd, text });
i = closeEnd;
closed = true;
break;
}
closeStart = closeEnd;
}
if (!closed) i = openerEnd;
}
return spans;
}
/** Markdown prose paragraphs, excluding headings and code blocks that are not authored prose. */
function markdownProseParagraphs(markdown: string, fences: readonly TextRange[]): TextRange[] {
const paragraphs: TextRange[] = [];
let start: number | undefined;
let end = 0;
let fenceIndex = 0;
const flush = () => {
if (start !== undefined) paragraphs.push({ start, end });
start = undefined;
};
let offset = 0;
for (const lineWithEnding of markdown.matchAll(/[^\n]*(?:\n|$)/g)) {
const line = lineWithEnding[0].replace(/\n$/, "").replace(/\r$/, "");
while (fenceIndex < fences.length && fences[fenceIndex].end <= offset) fenceIndex++;
const fenced = fenceIndex < fences.length && offset >= fences[fenceIndex].start;
if (fenced || /^\s*$/.test(line) || /^ {0,3}#{1,6}\s/.test(line) || /^ {4}/.test(line)) {
flush();
} else {
start ??= offset;
end = offset + lineWithEnding[0].length;
}
offset += lineWithEnding[0].length;
}
flush();
return paragraphs;
}
/** Removes spans by source range, including a span crossing a prose paragraph boundary. */
function withoutCodeSpans(markdown: string, range: TextRange, spans: readonly CodeSpan[]): string {
let result = "";
let cursor = range.start;
for (const span of spans) {
if (span.end <= range.start) continue;
if (span.start >= range.end) break;
const start = Math.max(span.start, range.start);
const end = Math.min(span.end, range.end);
result += markdown.slice(cursor, start) + " ";
cursor = end;
}
return result + markdown.slice(cursor, range.end);
}
/**
* Reports gate-registry facts that `tests/README.md` must state without duplicating their derived
* cardinalities. An id is covered only by an exact Markdown inline-code span, never a substring.
*/
export function registryReadmeIntegrityProblems(
registry: readonly GateReadmeLintGate[],
readme: string,
): string[] {
const problems: string[] = [];
const fences = markdownFencedCodeRanges(readme);
const codeSpans = markdownCodeSpans(readme, fences);
const inlineCode = new Set(codeSpans.map(span => span.text));
for (const { id } of registry)
if (!inlineCode.has(id))
problems.push(`meta-6: registry gate '${id}' has no exact inline-code span in tests/README.md`);
const groups = new Set(
registry.map(g => g.concurrent).filter((group): group is string => group !== undefined && group.length > 0),
);
const paragraphs = markdownProseParagraphs(readme, fences);
const cardinal = /\b(?:\d+|zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion)\b/gi;
for (const group of groups) {
const escaped = group.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const namesGroup = new RegExp(`(^|[^A-Za-z0-9_-])${escaped}(?=$|[^A-Za-z0-9_-])`);
for (const paragraph of paragraphs) {
const original = readme.slice(paragraph.start, paragraph.end);
if (!namesGroup.test(original)) continue;
const prose = withoutCodeSpans(readme, paragraph, codeSpans);
for (const token of prose.matchAll(cardinal))
problems.push(`meta-6: concurrent group '${group}' has authored prose cardinal '${token[0]}' in tests/README.md`);
}
}
return problems;
}
/** Pure synthetic floor for the documentation-integrity half of `--selftest`. */
export function registryReadmeIntegritySelftest(): void {
const missing = registryReadmeIntegrityProblems(
[{ id: "needed_gate" }],
"plain needed_gate and `needed_gate_suffix` are not exact coverage.",
);
if (!missing.some(p => p.includes("registry gate 'needed_gate'")))
throw new Error("README-integrity self-test failed to reject a missing exact gate-id span");
const counted = registryReadmeIntegrityProblems(
[{ id: "named_gate", concurrent: "manual_heavy" }],
"`named_gate`\n\nThe thirteen gates in `manual_heavy` run together.",
);
if (!counted.some(p => p.includes("group 'manual_heavy'") && p.includes("thirteen")))
throw new Error("README-integrity self-test failed to reject a prose concurrent-group count");
const symbolic = registryReadmeIntegrityProblems(
[{ id: "named_gate", concurrent: "manual_heavy" }],
"`named_gate`\n\nThe group `manual_heavy` reports `N gate(s)` at runtime.",
);
if (symbolic.length)
throw new Error(`README-integrity self-test rejected count-free symbolic prose: ${symbolic.join("; ")}`);
const multiline = registryReadmeIntegrityProblems(
[{ id: "named_gate", concurrent: "manual_heavy" }],
"`named_gate`\n\nThe `manual_heavy` group reports `batch: 15\ngates` without a prose count.",
);
if (multiline.length)
throw new Error(`README-integrity self-test failed to strip a multiline code span: ${multiline.join("; ")}`);
const longerDelimiter = registryReadmeIntegrityProblems(
[{ id: "named_gate", concurrent: "manual_heavy" }],
"``named_gate``\n\nThe ``manual_heavy`` group reports ``batch: 15 `gates` `` without a prose count.",
);
if (longerDelimiter.length)
throw new Error(`README-integrity self-test mishandled a longer-delimiter code span: ${longerDelimiter.join("; ")}`);
}
// ---- process helpers -----------------------------------------------------------------------------
function sh(cmd: string[], cwd = ROOT, env?: Record<string, string>): number {
const r = Bun.spawnSync(cmd, {
cwd,
env: env ? { ...process.env, ...env } : process.env,
stdout: "inherit",
stderr: "inherit",
stdin: "inherit",
});
return r.exitCode ?? 1;
}
// ==================================================================================================
// GATE-LEVEL CONCURRENCY
// ==================================================================================================
// Why: measured, the heavy gates leave ~89 % of a 32-core box idle. Each is a single `#[test]`
// looping serially over catalog rows, spawning nested cargo; solo, the tier's second-largest gate
// runs at 16.1 % CPU — 5.1 of 32 cores. A controlled same-session A/B over
// `multifile_matrix_roundtrips` + `wasm_matrix_roundtrips` measured 603 s serial against 338 s
// 2-way parallel — 1.78×, 89 % of the 2× ideal, with only +6 % per-gate inflation.
//
// What bounds the win, and therefore the design: under perfect parallelism a tier's wall is bounded
// by its LONGEST gate, and this set is badly skewed (~636 s down to ~10 s). "Sum ÷ jobs" is not
// achievable, so the pool dispatches LONGEST-FIRST — a short gate scheduled ahead of the longest one
// adds its whole duration to the tail.
//
// What bounds the DEGREE is memory, not cores: concurrent `rustc` is the memory-hungry part and only
// ~6 of 32 GiB is free on the development box in practice. Hence a small default, overridable.
const DEFAULT_JOBS = 4;
/** `CHECK_JOBS=1` restores fully sequential execution — the pool with degree 1 IS the old loop. */
export function parseJobs(raw: string | undefined): { jobs: number; warning?: string } {
if (raw === undefined || raw.trim() === "") return { jobs: DEFAULT_JOBS };
const n = Number(raw.trim());
if (!Number.isInteger(n) || n < 1)
return { jobs: DEFAULT_JOBS, warning: `CHECK_JOBS='${raw}' is not a positive integer — using ${DEFAULT_JOBS}` };
return { jobs: n };
}
// ---- the SECOND factor: cargo's own `-j` inside each batched gate --------------------------------
// `CHECK_JOBS` bounds how many GATES overlap. It does not bound how many `rustc` each one spawns, and
// a nested cargo defaults to `-j $(nproc)`. So the quantity that actually consumes memory —
//
// (gates in flight) × (rustc per gate) × (per-rustc resident set)
//
// — was bounded by nothing, and its second factor scaled with CORE COUNT, which is unrelated to the
// machine's memory. That is the defect this bound closes: a WSL2 box with a 32 GiB cap and 32 cores
// went unresponsive for ~10 minutes under a full tier and had to be power-cycled.
//
// Hence a bound derived from MEMORY, not cores, written into each batched child's `CARGO_BUILD_JOBS`:
//
// product = round(MemTotal × RUSTC_MEM_FRACTION ÷ ASSUMED_PEAK_RUSTC_GIB)
// per-gate -j = max(1, floor(product ÷ gates in flight))
//
// On a 32 GiB machine that is 8 rustc across the whole batch — `-j2` at the default 4 gates in
// flight — so 8 × 2 GiB = 16 GiB worst case against a 32 GiB cap. The half-of-MemTotal fraction is
// the headroom the arithmetic ignores: page cache (a tier writes tens of GiB of scratch), the parent
// `bun`/`cargo test` processes, an editor's rust-analyzer, and a concurrent session's own gates.
// `round`, not `floor`, because MemTotal reads a little under the machine's nominal cap (31.3 GiB on
// a 32 GiB WSL2 box — the kernel reserves the rest) and flooring would make a 32 GiB machine behave
// like a 28 GiB one; the assumed footprint below carries a 4× margin, so a rounding step at the
// boundary is inside the estimate's own error bar.
//
// Measured on this box (32 GiB cap, `nproc` 32, sampled at 1 s over a 4-gate batch of
// `recombination_wasm_crates_check` + `recombination_crates_execute` +
// `identifier_hazard_crates_compile` + `rust_oracle_fingerprint`, one warm regime):
//
// CARGO_BUILD_JOBS | peak rustc | peak Σ rustc RSS | batch wall
// -----------------+-------------+------------------+-----------
// 32 (unbounded) | 27 | 2.25 GiB | 63.6 s
// 2 (the default) | 3 | 0.53 GiB | 60.3 s
// 1 | 2 | 0.37 GiB | 60.0 s
//
// Two things that table settles. The bound is close to FREE at the batch level: these gates are
// internally serial loops over catalog rows, so the win came from overlapping GATES, never from
// cargo's own `-j` — only `rust_oracle_fingerprint`, the one member that is a single crate compile,
// slows (17.4 s → 34.0 s at `-j2`), and it is not the batch's tail. And process count is the wrong
// thing to reason about anyway: the largest single rustc resident set seen anywhere — across this
// batch and a whole `local` tier — was **455 MiB**, so bounding COUNT alone would still admit a very
// different peak if a gate ever compiles one large crate instead of many small ones.
/**
* Assumed worst-case resident set of ONE concurrency slot, in GiB.
*
* Named for `rustc` because that is what `CARGO_BUILD_JOBS` divides, but what it must actually cover
* is the peak of everything one slot holds at once — and compilation is only the first half of that.
* A nested-cargo gate compiles, then RUNS the binaries it built: the full tier's emitted-test crates
* carry thousands of `#[test]` functions each, and a test process's own resident set is not sampled
* anywhere in this file. Treating the constant as "one rustc" measured the cheaper half and budgeted
* as though it were the whole.
*
* Raised from 2 after two whole-machine freezes (~1 h and ~1.5 h, 100% memory and swap, sustained
* thrashing) under full tiers that every memory check passed — the largest single rustc ever observed
* here is 455 MiB, so 8 of those is ~3.6 GiB and cannot explain a 31 GiB box going down. The old
* value was not a small margin over the wrong quantity; it was a comfortable margin over a quantity
* that was never the binding one. Until a slot's true peak is measured (see `tests/testing-roadmap.toml`),
* this is deliberately pessimistic: the cost of being too low is a slower tier, and the cost of being
* too high is a machine that stops responding for an hour and takes every other session with it.
*/
const ASSUMED_PEAK_RUSTC_GIB = 4;
/**
* Fraction of AVAILABLE memory the batch may commit to concurrent `rustc`; the rest is headroom.
*
* Of AVAILABLE, not MemTotal — the distinction is the whole point. MemTotal answers "how big is this
* machine", which is not the question: the batch cannot use memory another process already holds. A
* developer box runs an editor's language servers, other agent sessions, and whatever else, so a
* budget struck against MemTotal silently assumes it owns a half of the machine that is already
* spoken for. Committing that half anyway is how a tier drives the box into swap, and a thrashing
* machine does not fail a gate — it stops responding, so nothing in this runner ever observes it.
* Budgeting against `MemAvailable` makes the bound shrink exactly when the machine is busy, which is
* when it must.
*/
const RUSTC_MEM_FRACTION = 0.5;
/** Product used when MemTotal is unreadable (non-Linux): the value this machine's memory derives. */
const FALLBACK_RUSTC_PRODUCT = 8;
/** MemTotal in GiB, or `undefined` where `/proc/meminfo` does not exist. Injectable for tests. */
export function memTotalGiB(meminfo?: string): number | undefined {
try {
const txt = meminfo ?? readFileSync("/proc/meminfo", "utf8");
const m = txt.match(/^MemTotal:\s+(\d+)\s*kB$/m);
return m ? Number(m[1]) / 1024 / 1024 : undefined;
} catch { return undefined; }
}
/**
* `CARGO_BUILD_JOBS` for one batched gate, given how many gates share the machine with it.
*
* Pure, so the arithmetic is pinned by a test rather than observed only in a full-tier run.
*
* Precedence, and why each rung is where it is:
* 1. `CHECK_CARGO_JOBS` — an explicit operator override wins outright. A 128 GiB box should be able
* to raise this, and a sequential-minded operator should be able to pin it to 1.
* 2. otherwise the memory-derived product above, and then
* 3. **never above an inherited `CARGO_BUILD_JOBS`** — a `min`, not a replace. Someone who exported
* `CARGO_BUILD_JOBS=2` to be gentle to their machine said something the runner must not undo;
* someone who exported `CARGO_BUILD_JOBS=32` did not know about the batch, so the bound applies.
*/
export function cargoJobsForBatch(o: {
gatesInFlight: number;
memTotalGiB?: number;
/**
* `MemAvailable` at the moment this batch starts — the preferred basis, and measured PER BATCH
* rather than once at startup, because a tier's later batches begin under whatever the earlier
* ones (and every other process on the box) left behind.
*/
memAvailGiB?: number;
override?: string; // CHECK_CARGO_JOBS
inherited?: string; // CARGO_BUILD_JOBS already present in the environment
}): { jobs: number; why: string; warning?: string } {
let warning: string | undefined;
const asPositiveInt = (raw: string | undefined, name: string): number | undefined => {
if (raw === undefined || raw.trim() === "") return undefined;
const n = Number(raw.trim());
if (Number.isInteger(n) && n >= 1) return n;
warning = `${name}='${raw}' is not a positive integer — ignoring it`;
return undefined;
};
const override = asPositiveInt(o.override, "CHECK_CARGO_JOBS");
if (override !== undefined)
return { jobs: override, why: `CHECK_CARGO_JOBS=${override} (operator override)`, ...(warning ? { warning } : {}) };
// `MemAvailable` first, `MemTotal` only as a fallback for a machine that cannot report it. Both
// are floored, never rounded: `Math.round` rounds a 3.5-slot budget UP to 4, spending headroom the
// fraction exists to reserve, and it is the wrong direction to be wrong in.
const basisGiB = o.memAvailGiB ?? o.memTotalGiB;
const basisName = o.memAvailGiB !== undefined ? "MemAvailable" : "MemTotal";
const product = basisGiB === undefined
? FALLBACK_RUSTC_PRODUCT
: Math.max(1, Math.floor((basisGiB * RUSTC_MEM_FRACTION) / ASSUMED_PEAK_RUSTC_GIB));
const derived = Math.max(1, Math.floor(product / Math.max(1, o.gatesInFlight)));
const memWhy = basisGiB === undefined
? `memory unreadable — fallback product ${product}`
: `${basisGiB.toFixed(1)} GiB ${basisName} × ${RUSTC_MEM_FRACTION} ÷ ${ASSUMED_PEAK_RUSTC_GIB} GiB/slot = ${product} slot(s)`;
const inherited = asPositiveInt(o.inherited, "CARGO_BUILD_JOBS");
if (inherited !== undefined && inherited < derived)
return {
jobs: inherited,
why: `${memWhy} ÷ ${o.gatesInFlight} gates = -j${derived}, held down to the inherited CARGO_BUILD_JOBS=${inherited}`,
...(warning ? { warning } : {}),
};
return {
jobs: derived,
why: `${memWhy} ÷ ${o.gatesInFlight} gates = -j${derived}`,
...(warning ? { warning } : {}),
};
}
/**
* Runaway-hang guard on the JOIN, not a duration assertion on any gate.
*
* Nothing here may fail a gate on a number — durations are nondeterministic and this delivery adds
* no test that fails on one. What this bounds is the pool's own liveness: if the join ever fails to
* settle after its work is done, the runner must say so loudly and RETURN rather than hang. That
* failure mode is not hypothetical — two ad-hoc probe scripts written while gathering the evidence
* for this feature backgrounded a sampler alongside their gates and then used a bare `wait`, which
* waits for EVERY background job including the never-exiting sampler. Both ran to completion
* internally — every gate exit-0, every verdict on disk — and then hung without emitting a summary;
* one went unnoticed for ~5 hours. The default is ~5× the slowest gate on record, so it cannot fire
* on slowness; if it fires, the runner has a bug and the diagnostic names the gates still in flight.
*/
const DEFAULT_JOIN_TIMEOUT_MS = 3 * 60 * 60 * 1000;
export function parseJoinTimeoutMs(raw: string | undefined): number {
if (raw === undefined || raw.trim() === "") return DEFAULT_JOIN_TIMEOUT_MS;
const n = Number(raw.trim());
return Number.isFinite(n) && n > 0 ? n * 1000 : DEFAULT_JOIN_TIMEOUT_MS;
}
// ---- the THIRD factor: how many nested tool children a gate runs AT ONCE -------------------------
// `CARGO_BUILD_JOBS` bounds how many `rustc` each nested cargo spawns; it says nothing about how
// many nested cargos a gate holds open concurrently. For a `cargo test` gate that count is the
// libtest thread count — `nproc` by default — so the true peak was
// `test threads × CARGO_BUILD_JOBS` compilers plus a spawned test binary per thread, a product the
// slot arithmetic above never modeled. The bound lives in the test helper every nested spawn goes
// through (`tool_cmd`, src/tests/integration_tests.rs), which reads `CDDL_NESTED_TOOL_PERMITS`;
// this helper is the runner's side of it.
//
// The derived value is ONE child per gate, and that is not timidity but the only reading under
// which the slot model is honest: a gate's whole `CARGO_BUILD_JOBS` share is each nested child's
// internal `-j`, so at N children the gate spends N × share slots — permits and jobs multiply, and
// any pair that both track the share overshoots the budget quadratically. One child at `-j share`
// is exactly the "compile then run" slot `ASSUMED_PEAK_RUSTC_GIB` prices. The wall-time this
// leaves on the table is real and deliberately unspent until the sampler below can price it
// (raising permits is safe exactly when measured child peaks say so, and `CHECK_NESTED_PERMITS`
// is the operator's override meanwhile).
export function nestedToolPermitsForGate(o: {
override?: string; // CHECK_NESTED_PERMITS
inherited?: string; // CDDL_NESTED_TOOL_PERMITS already present in the environment
}): { permits: number; why: string; warning?: string } {
let warning: string | undefined;
const asPositiveInt = (raw: string | undefined, name: string): number | undefined => {
if (raw === undefined || raw.trim() === "") return undefined;
const n = Number(raw.trim());
if (Number.isInteger(n) && n >= 1) return n;
warning = `${name}='${raw}' is not a positive integer — ignoring it`;
return undefined;
};
const override = asPositiveInt(o.override, "CHECK_NESTED_PERMITS");
if (override !== undefined)
return { permits: override, why: `CHECK_NESTED_PERMITS=${override} (operator override)`, ...(warning ? { warning } : {}) };
const derived = 1;
const inherited = asPositiveInt(o.inherited, "CDDL_NESTED_TOOL_PERMITS");
if (inherited !== undefined && inherited < derived)
return {
permits: inherited,
why: `1 nested child per gate, held down to the inherited CDDL_NESTED_TOOL_PERMITS=${inherited}`,
...(warning ? { warning } : {}),
};
return { permits: derived, why: "1 nested child per gate (its whole -j share is that child's)", ...(warning ? { warning } : {}) };
}
// ---- per-run memory sampler (report-only, asserted by NOTHING) -----------------------------------
// The slot arithmetic above budgets an ASSUMED per-slot footprint because no measurement of the
// real one existed: no gate ever sampled concurrent `rustc`, Σ RSS of the run's own process tree
// (test processes included), or how low `MemAvailable` actually went. This sampler is that
// measurement. It reports and records; it never fails anything — peaks and floors are
// nondeterministic, and a gate that fails on a number would be flaky by construction. What the
// numbers buy is the ability to replace pessimistic constants (the 4 GiB slot, the one-permit
// nested bound) with measured ones, and to split the NEXT whole-machine incident into "the memory
// bound was wrong again" vs "memory was healthy, something else saturated". OS memory-pressure
// notifications sharpen exactly that split: the kernel knows it is about to reclaim before either
// sampled number moves, so a run whose peaks looked healthy but which the OS was already squeezing
// is attributable to memory rather than left in the "something else saturated" bucket by default.
export interface ProcStat { pid: number; comm: string; ppid: number; rssBytes: number }
/**
* One `/proc/<pid>/stat` line. The comm field is parenthesized and may itself contain spaces or
* parens (`(tokio-runtime-w)`, `(a) weird (name)`), so the split point is the LAST `)` — fields
* after it are whitespace-separated with state at index 0, ppid at 1, rss (pages) at 21.
*/
export function parseProcStat(pid: number, line: string, pageBytes: number): ProcStat | undefined {
const open = line.indexOf("(");
const close = line.lastIndexOf(")");
if (open < 0 || close < open) return undefined;
const rest = line.slice(close + 1).trim().split(/\s+/);
const ppid = Number(rest[1]);
const rssPages = Number(rest[21]);
if (!Number.isInteger(ppid) || !Number.isFinite(rssPages)) return undefined;
return { pid, comm: line.slice(open + 1, close), ppid, rssBytes: rssPages * pageBytes };
}
/** Transitive children of `root` (root itself excluded — the runner measures what it SPAWNED). */
export function descendantsOf(root: number, procs: ProcStat[]): ProcStat[] {
const kids = new Map<number, ProcStat[]>();
for (const p of procs) {
const a = kids.get(p.ppid);
if (a) a.push(p); else kids.set(p.ppid, [p]);
}
const out: ProcStat[] = [];
const stack = [root];
while (stack.length) {
for (const c of kids.get(stack.pop()!) ?? []) {
out.push(c);
stack.push(c.pid);
}
}
return out;
}
export interface MemPeaks {
ticks: number;
readErrors: number;
/** Peak Σ RSS across the run's descendant tree, and the tree's shape at that tick. */
peakTreeGiB: number;
peakTreeProcs: number;
/** Peak count of concurrent `rustc` processes in the tree (its own tick, not the peak-RSS one). */
peakRustc: number;
/** Largest single process ever seen in the tree. */
maxSingleGiB: number;
maxSingleComm: string;
/** Machine-wide MemAvailable floor over the run — the number the budget's basis dips to. */
memAvailFloorGiB?: number;
/**
* OS low-memory notifications seen during the run (`process.on("memoryPressure")`, Bun 1.4+ —
* Linux and Windows deliver "critical"). `pressureEvents` is the TOTAL; `pressure` is a bounded
* prefix, because a reclaim storm can deliver these faster than the 1 s tick and an unbounded
* list would grow the printed block and the ledger row without bound. A Bun without the event
* leaves both at their zero values, which is indistinguishable from a healthy run — deliberately
* so: this is a hint, never a measurement to assert on. `atS` is when the loop DISPATCHED the
* notification, not when the kernel raised it: a sequential gate runs under `spawnSync` and blocks
* the loop for its whole duration (which is also why `ticks` is 1 on such a run), so a squeeze
* mid-gate surfaces at the gate boundary. The COUNT still survives that; only the timing slips.
*/
pressureEvents: number;
pressure: { level: string; atS: number }[];
}
/** How many pressure notifications the sampler keeps VERBATIM; the rest are counted only. */
const MAX_PRESSURE_ROWS = 32;
function startMemSampler(intervalMs = 1000): { stop: () => MemPeaks } {
const peaks: MemPeaks = {
ticks: 0, readErrors: 0, peakTreeGiB: 0, peakTreeProcs: 0, peakRustc: 0,
maxSingleGiB: 0, maxSingleComm: "-", pressureEvents: 0, pressure: [],
};
// The one signal the 1 s tick cannot synthesize, so it is subscribed rather than sampled. Both the
// subscribe and the unsubscribe are guarded: on a Bun that does not emit this event the listener
// is simply never called, and on a runtime whose `process` rejects the name outright the sampler
// carries on with the counts at zero — same failure philosophy as every other read in here.
const startedAt = Date.now();
const onPressure = (level?: unknown): void => {
peaks.pressureEvents++;
if (peaks.pressure.length < MAX_PRESSURE_ROWS)
peaks.pressure.push({
level: typeof level === "string" ? level : level === undefined ? "unknown" : String(level),
atS: Math.round((Date.now() - startedAt) / 100) / 10,
});
};
try { process.on("memoryPressure", onPressure); } catch { /* no such event here — counts stay 0 */ }
// Page size once: x86-64 and most aarch64 kernels use 4096, but 16k/64k-page arm64 kernels
// exist, and rss in /proc is in PAGES.
let pageBytes = 4096;
try {
const out = Bun.spawnSync(["getconf", "PAGESIZE"]).stdout.toString().trim();
const n = Number(out);
if (Number.isInteger(n) && n > 0) pageBytes = n;
} catch { /* keep the default */ }
const tick = (): void => {
try {
const procs: ProcStat[] = [];
for (const entry of readdirSync("/proc")) {
if (!/^\d+$/.test(entry)) continue;
try {
const p = parseProcStat(Number(entry), readFileSync(`/proc/${entry}/stat`, "utf8"), pageBytes);
if (p) procs.push(p);
} catch { /* the process exited between readdir and read — normal churn, not an error */ }
}
const tree = descendantsOf(process.pid, procs);
let sum = 0;
let rustc = 0;
for (const p of tree) {
sum += p.rssBytes;
if (p.comm === "rustc") rustc++;
if (p.rssBytes / 2 ** 30 > peaks.maxSingleGiB) {
peaks.maxSingleGiB = p.rssBytes / 2 ** 30;
peaks.maxSingleComm = p.comm;
}
}
const sumGiB = sum / 2 ** 30;
if (sumGiB > peaks.peakTreeGiB) {
peaks.peakTreeGiB = sumGiB;
peaks.peakTreeProcs = tree.length;
}
if (rustc > peaks.peakRustc) peaks.peakRustc = rustc;
const avail = availGiB("mem");
if (avail !== undefined && (peaks.memAvailFloorGiB === undefined || avail < peaks.memAvailFloorGiB))
peaks.memAvailFloorGiB = avail;
peaks.ticks++;
} catch {
peaks.readErrors++;
}
};
const timer = setInterval(tick, intervalMs);
// Belt to the stop()'s braces: an unref'd timer can never hold the event loop open, so even a
// path that misses stop() cannot recreate the hang-after-success class the join guard exists for.
(timer as unknown as { unref?: () => void }).unref?.();
return {
stop: () => {
clearInterval(timer);
try { process.off("memoryPressure", onPressure); } catch { /* never registered — nothing to undo */ }
tick(); // one final sample, so a run shorter than the interval still reports something
return peaks;
},
};
}
/** The sampler's end-of-run report: a printed block, and a row in the gitignored local ledger. */
function reportMemPeaks(peaks: MemPeaks, tier: Tier): void {
if (peaks.ticks === 0) {
console.log(`\nmemory sampler: no samples (${peaks.readErrors} read error(s) — no /proc on this platform?)` +
`; OS memory-pressure events ${peaks.pressureEvents}`);
return;
}
const gib = (n: number | undefined): string => n === undefined ? "?" : n.toFixed(2);
console.log(
`\nmemory sampler (report-only, 1 s ticks × ${peaks.ticks}): ` +
`peak run Σ RSS ${gib(peaks.peakTreeGiB)} GiB over ${peaks.peakTreeProcs} proc(s); ` +
`peak concurrent rustc ${peaks.peakRustc}; ` +
`largest single process ${gib(peaks.maxSingleGiB)} GiB (${peaks.maxSingleComm}); ` +
`machine MemAvailable floor ${gib(peaks.memAvailFloorGiB)} GiB; ` +
`OS memory-pressure events ${peaks.pressureEvents}` +
(peaks.readErrors ? `; ${peaks.readErrors} read error(s)` : ""),
);
// The zero case is reported above and says something on its own ("the OS never squeezed us"), so
// this second line exists only when there IS a squeeze to place in the run's timeline.
if (peaks.pressureEvents)
console.log(` pressure at (s): ${peaks.pressure.map(p => `${p.atS}s ${p.level}`).join(", ")}` +
(peaks.pressureEvents > peaks.pressure.length ? ` … +${peaks.pressureEvents - peaks.pressure.length} more` : ""));
try {
mkdirSync(join(ROOT, "draft"), { recursive: true });
const ledger = join(ROOT, "draft", "memory-peaks.jsonl");
// Bounded like the other draft ledgers, but self-contained: keep the last N runs at append
// time rather than joining the log-keyed retention pass — peaks rows carry no cross-file keys,
// and 200 runs is months of history at one row per run, plenty to re-derive a constant from.
const rows = existsSync(ledger) ? readFileSync(ledger, "utf8").split("\n").filter(l => l.trim()) : [];
rows.push(JSON.stringify({ stamp: new Date().toISOString(), tier, ...peaks }));
writeFileSync(ledger, rows.slice(-200).join("\n") + "\n");
} catch (e) {
console.log("memory sampler: ledger append failed (non-fatal — peaks are never a gate): " +
(e instanceof Error ? e.message : String(e)));
}
}
/** One unit of parallel work plus the handle the timeout guard would need to reclaim it. */
export interface PoolItem { id: string }
/**
* Bounded-concurrency pool. **The join awaits exactly the worker promises and nothing else.**
*
* `degree` workers pull from one cursor over `items`, so the pool is bounded by construction rather
* than by counting live promises. `stopAfter` stops the pull WITHOUT cancelling anything already in
* flight: that is fail-fast's meaning here — no NEW gate starts after a failure, gates already
* running finish and report their real verdicts, and everything never started is reported as
* never-run. Cancelling in-flight gates was rejected: it throws away minutes of completed work, and
* killing a nested cargo/rustc tree is the operation this repo has already been bitten by
* (a pattern-matched `pkill` took out a concurrent session's live run).
*
* The timeout races the join against a TIMER — a promise that cannot itself fail to settle — and the
* timer is always cleared, because an outstanding `setTimeout` keeps Bun's event loop alive and
* would reproduce the hang-after-success mode through a different door. `onTimeout` gets the items
* still in flight so the caller can reclaim them by their OWN handles (never by name pattern).
*/
export async function runPool<T extends PoolItem, R>(
items: T[],
degree: number,
work: (item: T) => Promise<R>,
o: { timeoutMs?: number; stopAfter?: (r: R, item: T) => boolean; onTimeout?: (inFlight: T[]) => void } = {},
): Promise<Map<string, R>> {
const results = new Map<string, R>();
const inFlight = new Set<T>();
let next = 0;
let stopped = false;
const worker = async (): Promise<void> => {
for (;;) {
if (stopped || next >= items.length) return;
const item = items[next++]!;
inFlight.add(item);
try {
const r = await work(item);
results.set(item.id, r);
if (o.stopAfter?.(r, item)) stopped = true;
} finally {
inFlight.delete(item);
}
}
};
const width = Math.max(1, Math.min(degree, items.length));
const joined = Promise.all(Array.from({ length: width }, () => worker())).then(() => "done" as const);
if (!o.timeoutMs) { await joined; return results; }
let timer: ReturnType<typeof setTimeout> | undefined;
const guard = new Promise<"timeout">(res => { timer = setTimeout(() => res("timeout"), o.timeoutMs); });
try {
if (await Promise.race([joined, guard]) === "timeout") {
stopped = true;
o.onTimeout?.([...inFlight]);
}
} finally {
if (timer !== undefined) clearTimeout(timer);
}
return results;
}
/**