From a7175d85b6c7ce07dcab818bc78debaa71bb2951 Mon Sep 17 00:00:00 2001 From: Schneider <224583183+schneiderjoseph@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:16:57 -0400 Subject: [PATCH 1/3] devia context and devia contribute, and the hardening that proved them Two commands and the pass that closed their limitations, landing in one commit because none of it was ever committed. The 0.7.0 that introduced `devia context` and `devia contribute` never had a source state of its own -- 0.8.0 rewrote the surfaces it added before either reached npm -- so it is not tagged and not published. CHANGELOG keeps its section as the record of what those commands were when they were written, marked never published. npm goes 0.6.0 to 0.8.0. `devia context` selects the smallest sufficient context for one task. The corpus is addressable items, not files: a rule, a memory section split at its heading, one never/always line, one gap or debt row, one impact-map duty. Five tiers decide what the context is, a token budget decides how much fits, and a blocking constraint is admitted before the budget is consulted. `devia contribute` turns a devia problem hit inside somebody's repository into an issue or a pull request under two constraints: a candidate is eligible only because devia re-ran the recorded invocation in a minimal fixture and observed the behaviour (AGT-012), and nothing of the user's repository is uploaded -- secrets, addresses, paths and account names are redacted on the way in and the payload is re-scanned before any upload (PRIV-005). A surviving secret shape blocks rather than warns. The hardening is what the benchmark forced: - A target is not a floor. Target, mandatory floor and selected now travel together, and the promise is explicit per mode: advisory may exceed the target and says so, strict never does. Compression buys mandatory items back toward full text in relevance order, so a larger target always returns more text -- the first implementation did this backwards and that is now a test. `devia check` gains CTX-BUDGET (P2). Closes G11 - The impact map is a router, not only a checklist. A declared change type routes the domains of the memory files it names, so a project that invents `new_consent_record` routes as well as a built-in does. Closes D12 - Evidence is bound to the experiment that produced it. Every run records a digest of the fixture and of the bin/ + src/ that ran it; a fix has to agree about the first and disagree about the second. Editing the fixture until it passes now reports the reproduction changed and stays at `reproduced`. Closes D10 Also fixed: the benchmark rebuilt its corpus for every combination, forty times slower than the thing it measures. Read once per shape, cloned per run: 79s to 1.5s. Version: package 0.8.0, standard 0.2.0 (AGT-012, AGT-013, PRIV-005). Verified: 135 tests pass; validate clean (141 rules, 551 links, index current); check P0 clear (15 pass, 1 warn OPS-001, 3 skip); memory validate 19 pass. Benchmark 352 runs -- critical-rule recall 100%, routing accuracy 100%, strict budget compliance 176/176, advisory over target 68/176 and in all 68 selected equals the mandatory floor to the token. Cost per correct decision is not measured and the output says so. --- .devia/01_ARCHITECTURE.md | 12 +- .devia/02_SURFACES.md | 19 + .devia/03_DATA_MODEL.md | 22 + .devia/04_PERMISSIONS.md | 32 +- .devia/05_FLOWS.md | 9 +- .devia/10_NEVER_ALWAYS.md | 20 + .devia/11_GAPS.md | 3 + .devia/12_DEBT.md | 4 + .devia/13_RECIPES.md | 39 ++ .devia/14_INDEX.md | 6 + .devia/devia.json | 11 +- .devia/impact-map.yaml | 4 + .github/workflows/ci.yml | 8 + .gitignore | 4 + AGENTS.md | 6 + CHANGELOG.md | 212 ++++++ CONTRIBUTING.md | 37 +- LEVELS.md | 11 + README.md | 118 +++- SECURITY.md | 36 +- VERSION | 2 +- compliance/COVERAGE.md | 22 +- compliance/TRACEABILITY.md | 3 + package.json | 3 +- rules/README.md | 5 +- rules/agent/AGT-012.md | 46 ++ rules/agent/AGT-013.md | 49 ++ rules/privacy/PRIV-005.md | 49 ++ schema/README.md | 1 + schema/contribution.schema.json | 148 ++++ schema/project-config.schema.json | 39 ++ scripts/benchmark-context.mjs | 401 +++++++++++ skills/devia/SKILL.md | 56 +- src/cli.mjs | 4 + src/commands/check.mjs | 576 +++++++--------- src/commands/context.mjs | 273 ++++++++ src/commands/contribute.mjs | 783 ++++++++++++++++++++++ src/commands/init.mjs | 28 +- src/lib/context.mjs | 820 +++++++++++++++++++++++ src/lib/contribution.mjs | Bin 0 -> 26085 bytes src/lib/gates.mjs | 43 ++ src/lib/sanitize.mjs | 141 ++++ src/lib/tokens.mjs | 75 +++ templates/agents/AGENTS.md | 10 + templates/agents/CLAUDE.md | 10 + templates/agents/copilot-instructions.md | 1 + templates/agents/cursor.mdc | 4 + templates/agents/windsurfrules.md | 3 + templates/project/AGENTS.md | 5 +- templates/project/README.md | 40 +- tests/cli.test.mjs | 32 + tests/context.test.mjs | 426 ++++++++++++ tests/contribute.test.mjs | 780 +++++++++++++++++++++ tests/sanitize.test.mjs | 84 +++ tests/tokens.test.mjs | 53 ++ 55 files changed, 5259 insertions(+), 369 deletions(-) create mode 100644 rules/agent/AGT-012.md create mode 100644 rules/agent/AGT-013.md create mode 100644 rules/privacy/PRIV-005.md create mode 100644 schema/contribution.schema.json create mode 100644 scripts/benchmark-context.mjs create mode 100644 src/commands/context.mjs create mode 100644 src/commands/contribute.mjs create mode 100644 src/lib/context.mjs create mode 100644 src/lib/contribution.mjs create mode 100644 src/lib/gates.mjs create mode 100644 src/lib/sanitize.mjs create mode 100644 src/lib/tokens.mjs create mode 100644 tests/context.test.mjs create mode 100644 tests/contribute.test.mjs create mode 100644 tests/sanitize.test.mjs create mode 100644 tests/tokens.test.mjs diff --git a/.devia/01_ARCHITECTURE.md b/.devia/01_ARCHITECTURE.md index 27b07db..6c5ace8 100644 --- a/.devia/01_ARCHITECTURE.md +++ b/.devia/01_ARCHITECTURE.md @@ -11,7 +11,8 @@ src/cli.mjs argument parsing, command table, context (root, .devia, f ↓ src/commands/*.mjs one file per command, each exporting a default (ctx, name) => exit code ↓ -src/lib/*.mjs yaml · markdown · fs · git · rules · ui · vendor · version — no command logic +src/lib/*.mjs yaml · markdown · fs · git · rules · gates · tokens · context · + sanitize · contribution · ui · vendor · version — no command logic ↓ content rules/ · standard/ · checklists/ · templates/ (read, never imported) ``` @@ -60,6 +61,15 @@ Content is data. Code reads it; code never encodes what a rule says. | `check` scans what git carries, not what the disk holds | A P0 failure on an ignored build artefact is a false positive that teaches people to ignore the gate | `src/lib/git.mjs` | | Design rule IDs carried over unchanged | Consolidation must not invalidate existing citations | `MIGRATION.md` | | A check that cannot answer returns SKIP | `PASS` must mean verified, never assumed | `src/commands/check.mjs` | +| The gate table is data in `src/lib/gates.mjs`, not structure inside `check` | Two readers need it and only one has a repository to scan: `check` attaches behaviour, `context` asks whether a rule is machine-enforced. One table is what stops the two answers drifting | `src/lib/gates.mjs`, `src/lib/context.mjs` | +| Context is routed and budgeted, never dumped | More context is not better context: the standard is ~19k tokens and a task needs a fraction of it | `src/lib/context.mjs`, `scripts/benchmark-context.mjs` | +| A target is not a floor, and both are printed | Reporting "target 600, selected 1380" made a stated design read as a broken promise. Three numbers now travel together: target, mandatory floor, selected — plus the status that reconciles them | `src/lib/context.mjs` `select`, `src/commands/context.mjs` `report` | +| `advisory` keeps every mandatory item whole; `strict` never exceeds | Two honest promises beat one vague one. Advisory reports `over` and includes the floor anyway; strict compresses mandatory items toward their identifier — never dropping one — and says `impossible` rather than going over | `src/lib/context.mjs` `SMALLEST`, `LADDER` | +| Compression is minimal, and restores upward | The first version degraded everything and then spent the freed tokens on *optional* rules at full text. Every mandatory item now starts at its smallest form and is bought back in relevance order | `src/lib/context.mjs` `select` | +| The impact map is a router, not only a checklist | It is the one routing table the project wrote, in the project's own vocabulary. A declared change type routes the domains of the memory files it names, so `new_consent_record` routes as well as a built-in | `src/lib/context.mjs` `matchedChangeTypes` | +| Every run is bound to its fixture and its devia | `fixed` means devia saw the problem and then saw it gone — same fixture, different devia. Without both digests, editing the fixture until it passes reads exactly like fixing the tool | `src/lib/contribution.mjs` `evidenceChain` | +| A contribution is eligible because devia reproduced it | An agent that can file an issue will invent reasons to. The state is computed from a re-run, bound to a hash of the claim, so editing the claim drops the verdict instead of carrying it forward | `src/lib/contribution.mjs` | +| devia never holds a GitHub token | The contribution path hands a prepared file to `gh` under an identity the project declared. A tool that stores credentials to be helpful is a tool that leaks them | `src/commands/contribute.mjs`, `.devia/04_PERMISSIONS.md` | | devia is for every agent | No agent is privileged: a surface that serves one must say why the others are not served, and record the gap. Absence of evidence about an agent is reported as SKIP, never as "unsupported" | `.devia/11_GAPS.md` G6, `src/commands/skills.mjs` | | The npm package is scoped, the command is not | npm refused the bare name `devia` as too similar to `degit`, `dexie` and `dva`; scoped names skip that filter. Docs say `npm i -D @schneiderjoseph/devia`, then `npx devia` | `package.json` | diff --git a/.devia/02_SURFACES.md b/.devia/02_SURFACES.md index ab7072a..67b2d9b 100644 --- a/.devia/02_SURFACES.md +++ b/.devia/02_SURFACES.md @@ -13,9 +13,11 @@ | `devia doctor` | Adoption, drift, staleness | `src/commands/doctor.mjs` | 1 when there is no `.devia/` | | `devia rules` | Query the registry | `src/commands/rules.mjs` | 1 when `--id` is unknown | | `devia read` | Render the memory as one self-contained page | `src/commands/read.mjs` | 1 without `.devia/` | +| `devia context` | The smallest sufficient context for one task | `src/commands/context.mjs` | 1 without `.devia/`, or when a strict target cannot hold the mandatory set | | `devia sync` | Pin the standard, or refresh a pinned copy | `src/commands/sync.mjs` | 1 without `.devia/` | | `devia skills` | Install adapters and the skill pack, per repository or `--global` | `src/commands/skills.mjs` | 2 on a bad action | | `devia gap` / `devia debt` | Registry lines | `src/commands/registry.mjs` | 1 when the id is unknown | +| `devia contribute` | A devia problem observed here, as an issue or a pull request | `src/commands/contribute.mjs` | 1 when a candidate is not eligible, 2 on a bad action | Global flags: `--root`, `--json`, `--help`, `--version` (prints the CLI **and** standard versions — an adopter pins one and reports the other). @@ -23,6 +25,22 @@ versions — an adopter pins one and reports the other). `init` alone refuses to act on a root it inferred that is not the current directory: `--root` to say where, or `--yes` to accept it. Nothing is written before that question is settled. +## The one surface that can reach the network + +`devia contribute submit --yes` is the only command in devia that can make a network request, and +it makes it by handing a prepared file to `gh`. Everything else — recording, reproducing, +verifying, rendering the payload — is local, and `submit` without `--yes` writes the payload and +prints the command rather than running it. + +| Step | Reaches the network | Guard | +|---|---|---| +| `contribute new` · `repro` · `verify` · `show` | No | — | +| `contribute submit` | No | Writes `payload/` and prints the `gh` command | +| `contribute submit --yes` | Yes, through `gh` | Eligible · payload clean · identity declared and not the maintainer · `gh` authenticated as that identity | + +devia holds no GitHub token, reads none from the environment, and never commits or pushes in a +checkout. A pull request is opened only against a branch the contributor already pushed. + ## Package exports | Export | Path | For | @@ -39,6 +57,7 @@ say where, or `--yes` to accept it. Nothing is written before that question is s | `init`, `skills install` | `AGENTS.md`, `CLAUDE.md`, `.cursor/rules/devia.mdc`, `.github/copilot-instructions.md`, `.windsurfrules` | | `skills install --skill` | `.cursor/skills/devia/SKILL.md`, `.claude/skills/devia/SKILL.md` | | `read` | `.devia/reader.html` — a generated snapshot, gitignored, never the source | +| `contribute` | `.devia/contributions//` — the record, the fixture, and a generated `payload/` | | `skills install --global` | Outside the repository, in each agent's own configuration: `~/.claude/skills/devia/`, `~/.codex/skills/devia/`, `~/.cursor/rules/devia.mdc`, `~/.gemini/GEMINI.md` when empty. Copilot and Windsurf report `SKIP` (`12_DEBT.md` D8) | `files` in `package.json` decides what npm ships. Adding a directory the CLI reads at runtime diff --git a/.devia/03_DATA_MODEL.md b/.devia/03_DATA_MODEL.md index 7a83638..1925a89 100644 --- a/.devia/03_DATA_MODEL.md +++ b/.devia/03_DATA_MODEL.md @@ -12,6 +12,12 @@ | Registry line | A gap or a debt row | `.devia/11_GAPS.md`, `.devia/12_DEBT.md` | Markdown table row, id `G` / `D` | | Waiver | Time-boxed exception | `devia.json` `waivers[]` | `schema/waiver.schema.json` | | Version pin | Standard and schema versions | `VERSION` | YAML scalars | +| Contribution record | Evidence for a devia problem seen here | `.devia/contributions//record.json` | `schema/contribution.schema.json` | +| Context item | One addressable piece of deliverable context | Derived from rules and `.devia/` | `{ kind, id, domains, tier, tokens, why }` in `src/lib/context.mjs` | + +A contribution record is JSON rather than YAML because it is machine data the CLI writes and +reads, like `devia.json` — not content a human authors, which is where the frontmatter shape +belongs. ## Invariants @@ -21,6 +27,17 @@ - Every rule has at least one `source` and at least one validation method. - Registry ids are monotone per registry and never reused, including after closure (`MEM-004`). - Generated files are derived from the rule files; the rule files are the source of truth. +- A contribution record never stores its own state. The state is computed from the verification, + and the verification is bound to a hash of the claim it was made about — editing the claim + drops the verdict rather than carrying it forward. +- `fixed` requires two observations that are the same experiment: devia saw the problem, then + devia saw it gone, with the same fixture digest and a different devia digest. One run can only + ever be half of that, and two runs over two different fixtures are not a fix at all. +- A mandatory context item is admitted before the target is consulted. In `advisory` it is never + compressed and the target is reported as exceeded; in `strict` it may be compressed toward its + identifier but is never dropped, and the target is never exceeded. +- `target`, `mandatory floor` and `selected` are three separate numbers and are always reported + as three. ## Lifecycles @@ -31,6 +48,11 @@ Rule status: `draft → proposed → active → deprecated → superseded → re Registry line: `open → closed` for a gap, `open → discharged` for debt. Closure records the change that closed it; partial work reduces the line instead of removing it (`MEM-003`). +Contribution: `incomplete → observed → reproduced → fixed`, with `rejected` reachable from any +verification that did not show the reported behaviour. Only `reproduced` and `fixed` are +eligible to be proposed, and only `fixed` with a named regression test routes to a pull request +(`AGT-012`). + ## Migrations There is no schema migration. The equivalent is versioning: a breaking change to a rule id, the diff --git a/.devia/04_PERMISSIONS.md b/.devia/04_PERMISSIONS.md index 4e054ce..b1634ff 100644 --- a/.devia/04_PERMISSIONS.md +++ b/.devia/04_PERMISSIONS.md @@ -9,8 +9,9 @@ |---|---| | Create and update files under `/.devia/` | Write outside `--root`, except `skills install --global` | | Write the agent adapters at the repository root | Overwrite a file the user has edited, without `--force` | -| Read files in the target repository to produce evidence | Send anything over the network | +| Read files in the target repository to produce evidence | Send anything over the network, except `contribute submit --yes` | | Replace `.devia/standard/` wholesale on `sync` | Touch the project's own memory content on `sync` | +| Copy a named file into a contribution fixture, sanitized | Copy a file into a fixture that the user did not name | `init` keeps every existing memory file unless `--force` is passed, because those files hold decisions the tool did not make. @@ -26,6 +27,33 @@ A file the agent owns and devia adds to (`~/.claude/skills/`, `~/.codex/skills/` (`~/.gemini/GEMINI.md`) is written only when absent or empty; otherwise `SKIP` says so and the content stays. `--force` overrides both, and says which paths it took. +## What may leave the machine + +`devia contribute` is the only feature that can publish anything, and it is built so that the +answer to "what did it send" is always a file the user read first. + +| Never leaves | Leaves only through `submit --yes` | +|---|---| +| The repository's source, unless a file is named with `--include` | A sanitized minimal fixture the contributor built | +| An environment file — refused outright, never sanitized and copied | The devia version, standard version, node and platform | +| Secrets, tokens, email addresses, IP addresses, the home directory, the account name, the repository path | The gate or rule involved, and the expected and actual behaviour | +| Dependency lists, private filenames, git history | The regression test's path, when there is one | + +Four gates stand between a candidate and a request: + +```text +eligible devia itself reproduced it, and there is a minimal case (AGT-012) +clean the finished payload is re-scanned; a surviving secret shape blocks, not warns +authorised --yes, given per submission, never remembered +attributed an identity declared in devia.json, refused if it is the maintainer's, + and checked against the account gh is actually authenticated as +``` + +devia stores no token and reads none from the environment. It does not commit, branch or push in +anyone's checkout: a pull request is opened only against a branch the contributor already pushed. +`"contribution": { "enabled": false }` in `devia.json` turns the whole feature off, local +commands included. + ## Destructive operations | Operation | Where | Guard | @@ -33,6 +61,8 @@ content stays. `--force` overrides both, and says which paths it took. | `rm -rf .devia/standard` before re-pinning | `init --vendor`, `sync` | Only that one directory, which the tool owns | | Overwriting memory files | `init --force` | Off by default, warned about in the output | | Removing a registry line | `gap`/`debt close` | Moves the line to the closed table, never deletes it | +| `rm -rf` a contribution candidate | `contribute rm` | Only that candidate's directory, which the tool owns | +| Rebuilding a contribution fixture | `contribute repro --force` | Off by default; without it the existing fixture is kept | ## Repository permissions diff --git a/.devia/05_FLOWS.md b/.devia/05_FLOWS.md index 8687ec8..30614f8 100644 --- a/.devia/05_FLOWS.md +++ b/.devia/05_FLOWS.md @@ -13,6 +13,8 @@ | Record | `devia gap add` / `devia debt add` / `debt close` → monotone ids, nothing deleted | "gap and debt lines get monotone ids…" | | Upgrade | `devia sync` → standard pinned or refreshed, pin updated, memory untouched | "sync pins the standard on demand" | | Cite | `devia rules --id SEC-001` → the full rule text | "rules can be queried by id and by filter" | +| Brief | `devia context ""` → the blocking set, then what fits the budget | `tests/context.test.mjs`, `scripts/benchmark-context.mjs` | +| Contribute | `contribute new → repro → verify → submit` → nothing sent without `--yes` | `tests/contribute.test.mjs` | ## Failure behaviour @@ -24,5 +26,10 @@ | Gate | A check cannot determine an answer | `SKIP` with the reason | Never counts as a pass | | Upgrade | The pinned version differs from the installed one | `validate` and `doctor` warn and name `devia sync` | Leaves the pin until sync runs | | Any | An unexpected exception | `devia: ` (stack with `DEVIA_DEBUG=1`) | Exits 1 | +| Brief | The budget is too small for the blocking set | `OVERRUN` and the blocking cost | Includes them anyway; a budget never evicts a P0 | +| Contribute | The problem was never reproduced | The blocker and the command that would reproduce it | Refuses to submit; writes the payload for reading | +| Contribute | A secret survives into the payload | `not sanitized` and what survived | Blocks the upload — it is never downgraded to a warning | +| Contribute | No identity, or `gh` authenticated as someone else | The mismatch, named | Sends nothing, and says nothing was sent | -The rule behind the whole table: silence is never a pass. +The rule behind the whole table: silence is never a pass, and nothing leaves the machine without +a sentence saying it did. diff --git a/.devia/10_NEVER_ALWAYS.md b/.devia/10_NEVER_ALWAYS.md index 0a8b2e3..2cba120 100644 --- a/.devia/10_NEVER_ALWAYS.md +++ b/.devia/10_NEVER_ALWAYS.md @@ -21,6 +21,24 @@ tool did not make. - **Never add a directory the CLI reads at runtime without adding it to `files` in `package.json`.** It works locally and ships broken. +- **Never set `FORCE_COLOR` to an empty string to neutralise it.** Node warns on stderr when it + sees both `NO_COLOR` and `FORCE_COLOR`, and `""` still counts as seeing it. Delete the key from + the inherited environment instead. The tests learned this once; `runObservation` had to learn + it again, because a warning leaking into captured stdout turns a verdict into a parse failure. +- **Never let a routing table match a substring of a word.** `key` inside `monkey` redacted + `monkey: banana`; `table` in the component keywords sent a schema change through + `components → accessibility` and pulled the whole screen corpus into it. Anchor on a whole + segment, and let a benchmark or a test name the word it must not match. +- **Never call a number a budget when it is a target that can be exceeded.** Print the target, + the mandatory floor and what was actually selected, and the status that reconciles them. + "Budget 600, selected 1380" reads as a broken promise even when the design was stated. +- **Never degrade everything to fit, then spend the freed room on something less important.** + Compression starts at the smallest form and buys back upward in relevance order. The first + version shrank every mandatory rule to an identifier and then admitted *optional* rules at + full text. +- **Never record a verdict that outlives the claim it was made about.** A verification is bound + to a hash of the observation; edit the observation and the state drops back to `observed`. A + stale verdict is exactly how an unreproduced problem reaches a maintainer. - **Never vendor a file without whatever it links to.** A relative link that resolves in this repository and not in `.devia/standard/` is a broken link shipped to every adopter, invisible here because `validate-links.mjs` skips `.devia/`. Add the target to `src/lib/vendor.mjs`, and @@ -32,6 +50,8 @@ - Always update `.devia/` in the same change as the code (`MEM-009`). - Always run `npm run validate`, `npm test` and `node bin/devia.mjs check --root .` before reporting done — all three, because they catch different things. +- Always run `npm run benchmark:context` after touching the router, the tiers or the budget. A + reduction that drops a rule the task needed is not a reduction, and only the benchmark says so. - Always regenerate the index after touching a rule file. - Always give a new rule a `source` and a real validation method, or mark honestly that it can only be reviewed by a human. diff --git a/.devia/11_GAPS.md b/.devia/11_GAPS.md index 3a9fb60..bf799cf 100644 --- a/.devia/11_GAPS.md +++ b/.devia/11_GAPS.md @@ -15,6 +15,8 @@ Add one with `npx devia gap add "question"`. | G4 | Should `devia sync` warn when an adopter has edited a vendored file? | Silent overwrite of a local edit that someone believed was persistent | `sync` reports changed and removed files after the fact | open | | G5 | How should a project override a rule's priority for its own context (a docs repo has no `SEC-001` surface)? | Either noisy irrelevant findings, or a habit of ignoring output | Rules apply as written; irrelevant ones are simply not applicable | open | | G7 | Should devia read an existing ad-hoc project memory (a DEVIA/ folder of YAML, a docs/context tree) when initialising, or leave the merge to a human? | | | open | +| G10 | Should `.devia/contributions/` be committed, or is it scratch? The record and the fixture are evidence and version well; `payload/` is generated like `reader.html` | A repository either carries evidence nobody asked for, or loses a reproduction between sessions | Everything is written, and `payload/` is gitignored by the template | open | +| G12 | Should the never/always list be relevance-filtered instead of always blocking? It is the most valuable thing devia knows and also the largest fixed cost in every selection | Either an earned trap is withheld from the task that would hit it, or small budgets are consumed before any rule is reached | Every line is T0 and never evicted; pruning the list is the project's job, as `10_NEVER_ALWAYS.md` already says | open | ## Closed @@ -24,3 +26,4 @@ Add one with `npx devia gap add "question"`. | G1 | Should `devia check` grow ecosystem-specific gates (Python, Go, Rust) or stay deliberately generic? | Reframed: the failure was not the ecosystem but the assumption that the manifest sits at the repository root — fixed in 0.4.0. Ecosystem-specific gates remain out of scope | | G8 | Should devia pin a copy of the standard into every adopter repository by default? | No — opt-in via devia sync. Measured on a real repository: 391 pinned files against 17 of memory, tripling a 211-file project and turning every sync into a 391-file diff | | G9 | Should devia render the memory itself, or leave reading to whatever the project already has? | devia renders it: `devia read` writes one self-contained page — no server, no network, no dependency (ARC-004) | +| G11 | Should `devia check` gate the context budget — fail when a project's blocking set no longer fits its own `context.maxTokens`? | devia check gates it: CTX-BUDGET compares the declared target with the baseline mandatory floor | diff --git a/.devia/12_DEBT.md b/.devia/12_DEBT.md index 4347953..98e69fb 100644 --- a/.devia/12_DEBT.md +++ b/.devia/12_DEBT.md @@ -19,9 +19,13 @@ Add one with `npx devia debt add "what is missing"`. | D7 | OPS-001 | `package.json` | No linter or formatter is configured, so `devia check` reports the missing lint gate on this repository itself; adding one means accepting a devDependency under `ARC-004` | P2 | 2026-09-03 | | D8 | AGT-001 | src/commands/skills.mjs | No user-level install for Copilot and Windsurf: their global configuration is editor settings rather than a file devia can place, so both report SKIP. Establish the real location before building | P2 | 2026-09-09 | | D9 | OPS-004 | src/commands/check.mjs | check reads package.json manifests anywhere, but pyproject.toml, go.mod and Cargo.toml are still read at the root only, so a Python or Go package one directory down is invisible | P2 | 2026-09-09 | +| D11 | AGT-013 | src/lib/tokens.mjs | The token budget is spent against an estimate, not a tokenizer. It is within roughly 20% for English and Markdown and is labelled an estimate everywhere it surfaces, but a budget enforced against an approximation is a gate with a soft edge | P2 | 2026-09-12 | +| D13 | PRIV-005 | src/commands/contribute.mjs | `submit --yes` opens an issue through `gh`; the pull-request path still requires the contributor to branch and push by hand, because devia will not write git history in someone's checkout. The four commands are printed, not run | P2 | 2026-09-12 | ## Discharged | ID | What was missing | Discharged by | |---|---|---| | D6 | CI runs on one Node version and one OS; the CLI writes files on Windows and POSIX and only Windows is exercised in practice | CI matrix: ubuntu-latest + windows-latest x node 20/22 | +| D10 | `fixed` proves the behaviour changed between two runs, not that the fixture was untouched between them. The checkout each run used is recorded, but a fixture edited instead of a fix would read the same. Hash the fixture into the verification | fixture and devia source hashes bind every run — src/lib/contribution.mjs evidenceChain | +| D12 | Routing is a keyword and path table. It cannot see that "the thing that emails users" is an integration, and a project whose vocabulary differs from the table's routes worse with no signal that it did | impact-map change types are first-class routing signals — src/lib/context.mjs matchedChangeTypes | diff --git a/.devia/13_RECIPES.md b/.devia/13_RECIPES.md index 875aab0..03e72e2 100644 --- a/.devia/13_RECIPES.md +++ b/.devia/13_RECIPES.md @@ -47,11 +47,50 @@ 5. CHANGELOG.md: does an existing adopter need to act, or is devia sync enough? ``` +## Change how context is routed or budgeted + +```text +1. src/lib/context.mjs — KEYWORDS, PATH_DOMAINS, IMPLIES, MEMORY_DOMAINS, the tiers, or the + degradation ladder (SMALLEST / LADDER) +2. Anchor any new keyword on a whole word: "key" inside "monkey" is the failure mode +3. Prefer a signal the project already declared: a change type in impact-map.yaml routes the + domains of the memory files it names, in the project's own vocabulary +4. npm run benchmark:context # 352 runs: recall and compliance first, reduction second +5. npm test # tests/context.test.mjs holds the safety constraints +6. If a scenario changed shape, say so in the benchmark's scenario list, not in the assertions +``` + +A reduction that drops a rule the task needed is not a reduction. The benchmark fails on a lost +blocking rule and on a strict target that exceeded itself, whatever either saved. + +Three numbers, never one: target, mandatory floor, selected. `advisory` delivers the floor whole +and reports the target as exceeded; `strict` compresses toward identifiers and never exceeds it. +Neither ever drops a mandatory item. + +## Prepare a contribution from a devia problem hit elsewhere + +```text +1. In the repository where it happened: + devia contribute new --type --summary ... --expected ... --actual ... + --argv "" --actual-... --expect-... +2. devia contribute repro # then make the fixture actually fail +3. devia contribute verify # this is the gate: reproduced, or nothing +4. Fix it in a devia checkout, add the regression test +5. devia contribute verify --devia # reproduced -> fixed +6. devia contribute submit --fix-repo --fix-tests tests/x.test.mjs +7. Read payload/ — every byte that would be sent is there +8. Push the branch, then: devia contribute submit --yes +``` + +The two assertions must tell the two behaviours apart. A gate id appears in `--json` whether the +gate passed or failed, so assert on the `blocking` list, an exit code, or a metric. + ## Run the checks ```bash npm run validate # rules, links, generated files npm test # unit + CLI behaviour +npm run benchmark:context # context recall and reduction node bin/devia.mjs check --root . # the standard passes its own gates node bin/devia.mjs validate # this repository's own memory ``` diff --git a/.devia/14_INDEX.md b/.devia/14_INDEX.md index a1fa695..4a6a1dd 100644 --- a/.devia/14_INDEX.md +++ b/.devia/14_INDEX.md @@ -29,9 +29,15 @@ | Commands | [`../src/commands/`](../src/commands/init.mjs) | | YAML subset parser | [`../src/lib/yaml.mjs`](../src/lib/yaml.mjs) | | Rule loading and invariants | [`../src/lib/rules.mjs`](../src/lib/rules.mjs) | +| Gate table, shared by check and context | [`../src/lib/gates.mjs`](../src/lib/gates.mjs) | +| Context routing, tiers and budget | [`../src/lib/context.mjs`](../src/lib/context.mjs) | +| Token estimation | [`../src/lib/tokens.mjs`](../src/lib/tokens.mjs) | +| Redaction and payload residue | [`../src/lib/sanitize.mjs`](../src/lib/sanitize.mjs) | +| Contribution evidence and eligibility | [`../src/lib/contribution.mjs`](../src/lib/contribution.mjs) | | Filesystem helpers | [`../src/lib/fs.mjs`](../src/lib/fs.mjs) | | Terminal output | [`../src/lib/ui.mjs`](../src/lib/ui.mjs) | | Index generator | [`../scripts/build-index.mjs`](../scripts/build-index.mjs) | +| Context benchmark | [`../scripts/benchmark-context.mjs`](../scripts/benchmark-context.mjs) | | Repository validators | [`../scripts/validate-rules.mjs`](../scripts/validate-rules.mjs), [`../scripts/validate-links.mjs`](../scripts/validate-links.mjs) | | Tests | [`../tests/`](../tests/cli.test.mjs) | | CI | [`../.github/workflows/ci.yml`](../.github/workflows/ci.yml) | diff --git a/.devia/devia.json b/.devia/devia.json index 25a427c..b76dbf5 100644 --- a/.devia/devia.json +++ b/.devia/devia.json @@ -1,6 +1,6 @@ { - "deviaVersion": "0.1.0", - "standardVersion": "0.1.0", + "deviaVersion": "0.8.0", + "standardVersion": "0.2.0", "project": { "name": "devia", "profile": "cli" @@ -19,6 +19,13 @@ "design": false, "memory": true }, + "context": { + "budget": 2400, + "mode": "advisory" + }, + "contribution": { + "enabled": true + }, "waivers": [], "initializedAt": "2026-09-03" } diff --git a/.devia/impact-map.yaml b/.devia/impact-map.yaml index 3f6d352..9a304f7 100644 --- a/.devia/impact-map.yaml +++ b/.devia/impact-map.yaml @@ -34,6 +34,10 @@ impacts: stack_change: ["00_OVERVIEW.md", "01_ARCHITECTURE.md"] new_module: ["00_OVERVIEW.md", "01_ARCHITECTURE.md", "14_INDEX.md"] + new_command: ["02_SURFACES.md", "13_RECIPES.md"] + new_context_signal: ["01_ARCHITECTURE.md"] + network_surface: ["02_SURFACES.md", "04_PERMISSIONS.md"] + decision_recorded: ["11_GAPS.md"] rule_not_yet_built: ["12_DEBT.md"] incident_postmortem: ["10_NEVER_ALWAYS.md"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b64a736..8b996ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,11 +42,19 @@ jobs: test -f skills/devia/SKILL.md test -f templates/github/workflows/app-ci.yml test -f templates/project/AGENTS.md + test -f schema/contribution.schema.json + test -f scripts/benchmark-context.mjs - name: Dependency audit run: npm audit --audit-level=high - name: Rules · links · generated files run: npm run validate - name: Unit tests run: npm test + # A reduction that drops a rule the task needed is not a reduction, and the percentage + # never says so. This fails on a lost blocking rule whatever it saved. + - name: Context recall and reduction + run: npm run benchmark:context - name: The standard passes its own gates run: node bin/devia.mjs check --root . + - name: This repository's own memory + run: node bin/devia.mjs validate diff --git a/.gitignore b/.gitignore index 74306e7..b65dafc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ build/ # Généré par `devia read` — un instantané de la mémoire, pas la mémoire .devia/reader.html +# Généré par `devia contribute submit` — reconstruit depuis le record et le fixture, +# qui eux sont des preuves et restent versionnés +.devia/contributions/*/payload/ + # Local secrets / audits *.pem *.key diff --git a/AGENTS.md b/AGENTS.md index ad69b74..363b273 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,10 @@ Report what changed, and report what was NOT verified - Deleting a gap or debt line you did not discharge - Third-party documentation, schemas, or assets pasted in as original work - Claiming WCAG conformance without contrast, keyboard, and accessible-name evidence +- Sending anything from this repository to a third party — including a tool's issue tracker — + without explicit authorisation for that specific payload +- Filing an issue or a pull request against a tool from a problem you described but never + reproduced ## You may NOT declare "production ready" unless @@ -114,6 +118,8 @@ for the human. Passing unit tests is not Gold maturity — see [`MATURITY.md`](M | Any user-facing UI | `standard/design/`, `rules/ux/`, `rules/ui/`, `rules/accessibility/`, `rules/states/` | | Money, dates, numbers on screen | `standard/design/data-display/`, `rules/data-display/` | | "Is it production ready?" | `checklists/engineering/production.md` + `devia check` | +| Which rules apply to the task in front of you | `devia context ""` — routed, budgeted, and it says why | +| A devia problem you hit while working here | `devia contribute` — evidence first, `AGT-012`, `PRIV-005` | ## UI work is not exempt diff --git a/CHANGELOG.md b/CHANGELOG.md index f2df014..07e6777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,217 @@ # Changelog +## 0.8.0 — 2026-09-13 + +A hardening pass. No new features: three limitations 0.7.0 recorded as debt are closed, and the +benchmark grew enough to prove it. The standard is unchanged at 0.2.0. + +### A target is not a floor + +`Budget 600 → selected 1,380` was a stated design — a mandatory item is never evicted — printed +in a way that read as a broken promise. Three numbers now travel together everywhere: + +```text +Target 2400 tokens (advisory) +Mandatory floor 2001 tokens in 36 items +Selected 2400 tokens in 43 items +Status WITHIN TARGET +``` + +And the promise is now explicit per mode: + +| | `advisory` (default) | `strict` | +|---|---|---| +| Mandatory items | always whole | compressed toward their identifier, never dropped | +| The target | may be exceeded, and says so | **never** exceeded | +| When it cannot fit | `OVER TARGET`, with the floor | `IMPOSSIBLE` — nothing produced, exit 1 | + +Compression is minimal: every mandatory item starts at its smallest form and is bought back +toward full text in relevance order, so a larger target always returns more text. The first +implementation shrank every mandatory rule to a bare identifier and then spent the freed tokens +admitting *optional* rules at full text, which is precisely backwards — that is now a test. + +- `context.budget` and `context.mode` in `devia.json`; `context.maxTokens` is still read, so an + 0.7.0 adopter keeps working untouched +- `--strict` / `--mode`, and `devia check` gains `CTX-BUDGET` (P2): it compares the declared + target with the floor for a task that routes nothing, so a target nobody revisits cannot + quietly become a permanent overrun. Closes G11 +- A compressed rule still names itself and says where to read it (`devia rules --id `) + +### The impact map is a router, not only a checklist + +Routing was devia guessing from a keyword table, which a project whose vocabulary differs loses +by. `impact-map.yaml` is the one routing table the *project* wrote: a declared change type now +routes the domains of the memory files it names, and promotes those files. + +`permission_change → 04_PERMISSIONS.md` means a permission change routes to security and privacy +without anyone teaching devia this project's word for it — and a project that invents +`new_consent_record` routes exactly as well as a built-in does. `--type` declares it explicitly. +A change type is matched on half its significant parts, so "add an endpoint" reaches +`new_endpoint` while "fix the empty state" does not reach `state_machine_change`. Closes D12. + +### Evidence is bound to the experiment that produced it + +`fixed` used to mean "the behaviour changed between two runs". It now means the two runs were the +same experiment: every run records a digest of the fixture that ran and of the `bin/` + `src/` +that ran it, and a fix has to agree about the first and disagree about the second. + +```text +| Run | Date | Fixture | devia source | +| reproduced | 2026-09-12 | 3f2a… | 9c11… | +| expected | 2026-09-13 | 3f2a… | 4d80… | +``` + +Editing the fixture until it passes now reports `the reproduction changed between the two runs` +and stays at `reproduced`. Two runs against the same devia report `nothing in devia changed +between them`. A record written before hashing existed is re-verified rather than trusted. The +table travels in the published report, so a maintainer can check it by running the fixture +against each. Closes D10. + +### Benchmark + +`npm run benchmark:context` now runs **352 combinations**: 4 corpus shapes (devia's own plus a +small, an ordinary and a large synthetic memory) × 11 task types × 4 targets × 2 modes. It fails +on three promises and reports four measurements. + +Measured, not claimed: + +```text +Critical-rule recall 100% in 352/352 runs +Routing accuracy 100% in 352/352 runs +Strict budget compliance 176/176 runs never exceeded the target +Advisory over target 68/176 runs, all because the mandatory floor exceeded it +Mean task-generic share 66.9% of selected tokens +Mean supporting filler 2.3% of selected tokens +Mean selection time <1 ms per run +``` + +It found two defects on the way in: the strict compression order above, and a "noise ratio" that +read 0.0% in all 352 runs because nothing could ever score above zero. A metric that always +passes measures nothing; it is replaced by two that can move. + +Where a target is still exceeded, and exactly why: 68 of the 176 advisory runs, **none** of the +176 strict ones. In all 68, `selected` equals the mandatory floor to the token — the excess is +entirely mandatory items and nothing optional was ever added on top. That is now an invariant the +benchmark and the test suite both enforce, not an observation: + +| Corpus | Target | Runs over | Mandatory floor | Excess | +|---|---|---|---|---| +| devia (real) | 600 / 1200 / 2400 | 11 / 11 / 1 | 1511–2686 | 20–2086 | +| small memory | 600 / 1200 | 9 / 2 | 702–1577 | 86–977 | +| ordinary memory | 600 / 1200 | 9 / 2 | 760–1635 | 144–1035 | +| large memory | 600 / 1200 / 2400 | 11 / 11 / 1 | 1384–2420 | 20–1820 | + +Every one of them is a repository asking for less than its own blocking rules cost. `strict` is +the answer when the target has to hold. + +**Cost per correct decision is not measured.** It needs an agent and a graded task set, which +this benchmark does not have, and the output says so rather than implying otherwise. + +### Fixed + +- The benchmark rebuilt the corpus for every combination, which made it forty times slower than + the thing it measures. The corpus is read once per shape and cloned per run: 79s → 1.5s + +## 0.7.0 — 2026-09-12 (never published) + +This version was prepared but never tagged and never published. No source state for it +survived, and 0.8.0 rewrote the surfaces it introduces below before either reached npm, so +everything in this section shipped in 0.8.0 instead. It is kept because it is the record of +what those two commands were when they were written. An adopter looking for `0.7.0` on npm +will not find it, and wants `0.8.0`. + +The standard gains three rules and moves to 0.2.0: `AGT-012`, `AGT-013`, `PRIV-005`. + +Nothing changes for a repository that does nothing. Both features are additive, `devia check`, +`validate`, `doctor`, `rules`, `read` and `sync` behave exactly as before, and neither new +command needs GitHub authentication, network access or a dependency to do its local work. + +### `devia context` — the smallest sufficient context for one task + +More context is not better context. Everything devia knows about this repository is about 16,000 +estimated tokens; the part that belongs in the window for one task is a fraction of it, and the +rest pushes out the code the agent is supposed to read. + +```text +Raw corpus 16247 tokens (estimated) +Selected 2395 tokens in 45 items +Budget 2400 tokens +Reduction 85.3 % +``` + +- The corpus is addressable items, not files: a rule, a memory section split at its heading, one + never/always line, one open gap or debt row, one impact-map duty +- Routing is a keyword table, a changed-path table and an implication table, all data. Every + selection carries its reason, so `--explain` answers both "why is this here?" and "why is that + not?" +- Five tiers decide what the context *is*; the budget decides how much of it fits. **A blocking + constraint is admitted before the budget is consulted and is never evicted** — too small a + budget reports an overrun and still carries every P0 +- `context.maxTokens` in `devia.json`, default 1200. A missing or nonsense value degrades to the + default rather than failing +- `--files`, `--diff`, `--domain`, `--budget`, `--explain`, `--stats`, `--full`, `--json` + +**A rule devia verifies itself is cited, not recited** (`AGT-013`). `SEC-002` arrives as +`checked by devia check → SEC-SECRETS (P0) → blocks the change` instead of its requirement, +because the gate is what stops the change. The exception carries the rule: a `P0` whose only gate +*warns* keeps its full text, since nothing is actually stopping it. `src/lib/gates.mjs` now holds +the gate table as data, so `check` and `context` cannot disagree about which rule is enforced. + +`npm run benchmark:context` measures six scenarios at three budgets and **asserts recall before +it reports a reduction**. It found three defects the percentage never would have: + +- `add POST /api/orders` routed to `api` alone and dropped `SEC-001` and `SEC-003` — the two + rules a write endpoint most needs — at every budget. An endpoint is an authorization surface + whether or not the task says the word +- the word "table" sent a schema change through `components → accessibility` and pulled the whole + screen corpus into it +- `new_endpoint` never matched its own impact-map key, because the task's words were never split + on the underscore + +### `devia contribute` — a devia problem hit in a real repository + +An agent using devia inside somebody's project will sometimes hit a devia problem. This turns +that into an issue or a pull request under two hard constraints. + +**Evidence, not opinion** (`AGT-012`). A candidate is eligible because devia re-ran the recorded +invocation inside a minimal fixture and observed the reported behaviour: + +```text +observed → reproduced → fixed → issue or pull request +``` + +`reproduced` is never something the record says about itself. The verdict is bound to a hash of +the claim, so editing the claim drops the state back to `observed` instead of carrying a stale +verdict forward. `fixed` needs both halves — devia saw the problem, then devia saw it gone; the +expected behaviour alone means the fixture never failed, which is the opposite of a fix. A +speculative proposal is possible with `--manual`, and becomes an issue, never a pull request. + +**The user's repository stays the user's** (`PRIV-005`). Nothing is uploaded. A payload carries a +standalone fixture, devia's version metadata and the two behaviours. Secrets, credential +assignments, addresses, IP addresses, the home directory, the account name and the repository +path are redacted on the way in and the redactions are reported; an environment file is refused +outright rather than sanitized and copied. The finished payload is re-scanned, and a surviving +secret shape blocks the upload rather than warning about it. + +`submit` writes the payload and prints the `gh` command. `submit --yes` is the only path in devia +that can reach the network, and it refuses unless the candidate is eligible, the payload is +clean, an identity is declared in `devia.json`, that identity is not the maintainer's, and `gh` +is authenticated as it. devia stores no token, reads none from the environment, and never +commits, branches or pushes in anyone's checkout. A security defect is routed to the private +advisory path and never becomes an issue or a PR. `"contribution": { "enabled": false }` turns +the whole feature off, local commands included. + +### Fixed + +- `src/lib/sanitize.mjs` now owns the secret-pattern list that `devia check` scans with, so a + pattern added for one is immediately true for the other +- The sanitizer re-matched its own `[redacted]` placeholder, inflating the redaction count every + time text passed through, and dropped the quoting around a redacted value — which could stop a + fixture file parsing +- `init` now writes `.devia/.gitignore` covering `reader.html` and `contributions/*/payload/`. + The memory's README already told adopters those were gitignored; nothing was writing it. The + project's own `.gitignore` is not touched + ## 0.6.0 — 2026-09-09 The standard is unchanged: `VERSION` stays at 0.1.0. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2711921..fa71d93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,9 @@ This repository is the standard. A change here reaches every project that runs ` 3. Run the checks before asking for review: ```bash -npm run validate # rules, links, generated files current -npm test # unit + CLI behaviour +npm run validate # rules, links, generated files current +npm test # unit + CLI behaviour +npm run benchmark:context # context recall first, reduction second node bin/devia.mjs check --root . ``` @@ -74,6 +75,38 @@ existing adopters or whether they must edit their own memory. - No emoji, no AI attribution, no marketing voice. - Say what is true, including when it is inconvenient. +## Changing context routing or the budget + +`src/lib/context.mjs` decides what an agent receives. Two constraints are not negotiable: + +- A mandatory item is admitted before the target is consulted and is never dropped. `advisory` + delivers it whole and reports the target as exceeded; `strict` compresses it toward its + identifier and never exceeds the target. Neither ever loses it. +- The target, the mandatory floor and what was selected are three numbers and are reported as + three. Collapsing them makes a stated design read as a broken promise. +- Every selection carries its reason, so `--explain` can answer both "why is this here?" and + "why is that not?". + +Run `npm run benchmark:context` after any change to the keyword table, the path table, the +implication table or the tiers. It asserts recall before it reports a reduction and fails on a +lost blocking rule whatever the percentage says — it is how three real routing defects were +found, none of which were visible in the percentage. + +Anchor a new keyword on a whole word. `key` inside `monkey` and `table` inside a schema change +are the two failure modes already recorded in `.devia/10_NEVER_ALWAYS.md`. + ## Reporting a problem Use the issue templates. For a security issue in the CLI, see [`SECURITY.md`](SECURITY.md). + +An agent that hit the problem inside a real repository can prepare the report from there with +`npx devia contribute`, which builds a standalone reproduction rather than exposing that +repository. A candidate is eligible only once devia reproduced the problem itself, so an issue +arriving this way already carries a fixture, both behaviours and the version it was seen on. A +fix with a regression test arrives as a pull request; anything else arrives as an issue. + +Review it the way you would any other: read the fixture, run it, and check that the claimed +behaviour is the behaviour. The report carries an evidence chain — each run bound to a digest of +the fixture that ran and of the `bin/` + `src/` that ran it — so a `fixed` claim is checkable: +the two rows must share a fixture and differ in devia. If they do not, devia says so itself and +the record stays at `reproduced`. diff --git a/LEVELS.md b/LEVELS.md index 39c9a8d..afc63f8 100644 --- a/LEVELS.md +++ b/LEVELS.md @@ -45,6 +45,17 @@ npx devia doctor # adoption, drift, staleness `devia check` aggregates evidence into PASS / WARN / FAIL and exits non-zero on any P0 FAIL. +### A check earns its rule's tokens back + +A rule this level verifies deterministically does not need its full text repeated into an agent's +context. `devia context` delivers it as a citation — `SEC-002 → devia check SEC-SECRETS (P0) → +blocks the change` — and spends the saved budget on the rules nothing here can check. + +That trade only holds while the gate actually blocks. A `P0` rule whose only gate warns is not +being stopped by level 2, so its full text stays in the context (`AGT-013`). Automating a check +is therefore not just enforcement work: it is what makes the level-1 corpus affordable to +deliver. + ## Level 3 — Enforcement CI must be able to say **NO** to a pull request. diff --git a/README.md b/README.md index 845027a..7ca11bd 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ npx devia init # creates .devia/ + agent adapters npx devia validate # memory integrity npx devia check # production readiness (P0/P1) npx devia doctor # adoption + staleness diagnosis +npx devia context "" # the smallest sufficient context for one task ``` `devia init` writes: @@ -55,7 +56,8 @@ npx devia doctor # adoption + staleness diagnosis ├── 13_RECIPES.md # how to do common tasks in THIS repo ├── 14_INDEX.md # where to find what ├── impact-map.yaml # change type → files that must be updated -├── devia.json # profile, modules, maturity target, pinned version +├── devia.json # profile, modules, maturity target, pinned version, context budget +├── contributions/ # optional: evidence for devia problems found in this repo └── standard/ # optional: `devia sync` pins a copy of the standard here ``` @@ -79,6 +81,99 @@ Update .devia/ in the SAME change An agent that codes without reading the memory, or that ships code without updating it, has failed the task — not styled it differently. +## Reading the memory is not reading all of it + +More context is not better context. Everything devia knows about this repository is about 20,000 +estimated tokens; the part that belongs in the window for one task is a fraction of it, and the +rest pushes out the code the agent is supposed to read. + +```bash +npx devia context "add POST /api/orders" # the context itself, ready to pipe +npx devia context "fix the empty state" --explain # why each item is there, and what was withheld +npx devia context --stats # the accounting +``` + +```text +Target 2400 tokens (advisory) +Mandatory floor 2001 tokens in 36 items +Selected 2400 tokens in 43 items +Raw corpus 19882 tokens (estimated) +Reduction 87.9 % + +Status WITHIN TARGET +``` + +**Three numbers, not one.** A target is what you asked for; the mandatory floor is what the +blocking items cost; selected is what you got. Collapsing them is how "target 600, selected 1380" +comes to look like a broken promise instead of a stated design. + +Two modes make the promise explicit: + +| | `advisory` (default) | `strict` | +|---|---|---| +| Mandatory items | always whole | compressed toward their identifier, never dropped | +| The target | may be exceeded, and says so | never exceeded | +| When it cannot fit | `OVER TARGET`, with the floor | `IMPOSSIBLE` — nothing is produced, exit 1 | + +Compression is minimal and reversible: every mandatory item starts at its smallest form and is +bought back toward full text in relevance order, so a larger target always returns more text. + +Two things hold it up: + +- **A blocking constraint is never dropped.** Not by a small target, not by a strict one. `npm + run benchmark:context` measures exactly that across 352 runs and fails on a lost blocking rule + whatever it saved. +- **A rule devia verifies itself is cited, not recited.** `SEC-002` arrives as + `checked by devia check → SEC-SECRETS (P0) → blocks the change` instead of its full text, + because the gate is what stops the change, not the agent's memory of the sentence. The + exception carries the rule: a P0 whose only gate *warns* keeps its text, since nothing is + actually stopping it (`AGT-013`). + +Every selection can answer "why is this here?" and "why is that not?". A selector nobody can +interrogate is a selector nobody should trust. + +Routing starts from the table *your project* already wrote. `impact-map.yaml` says a +`permission_change` updates `04_PERMISSIONS.md`, and that file speaks for security and privacy — +so a permission change routes to security without anyone teaching devia your word for it. A +project that invents `new_consent_record` routes exactly as well as a built-in change type does. + +## Contributing back from a real repository + +An agent using devia inside your project will sometimes hit a devia problem — a gate that fires +on valid code, a check that misses one, a context selection that spends its budget badly. +`devia contribute` turns that into an issue or a pull request, under two hard constraints. + +**Your project stays your project.** Nothing is uploaded. The payload is a standalone fixture the +contributor wrote, devia's own version metadata, and the expected and actual behaviour. Secrets, +tokens, addresses, IP addresses, your home directory, your account name and the repository path +are redacted on the way in; an environment file is refused outright rather than sanitized and +copied. The finished payload is re-scanned, and a surviving secret shape blocks the upload +instead of warning about it. `submit` prints every byte first, and sends nothing without `--yes`. + +**Evidence, not opinion.** "devia could support X" is not a contribution. A candidate becomes +eligible because devia re-ran the recorded invocation inside the minimal case and observed the +reported behaviour: + +```text +observed → reproduced → fixed → issue or pull request +``` + +`reproduced` is never something the record says about itself. Every run is bound to two digests — +the fixture that ran and the devia source that ran it — so `fixed` means the two runs agreed +about the experiment and disagreed about the tool: + +```text +| Run | Date | Fixture | devia source | +| reproduced | 2026-09-12 | 3f2a… | 9c11… | +| expected | 2026-09-13 | 3f2a… | 4d80… | +``` + +Same fixture, different devia: a maintainer can check that by running the fixture against each. +Edit the fixture until it passes and the record says so and stays at `reproduced`; edit the claim +and the state drops back to `observed`. A fix with a regression test is a PR candidate; anything +else is an issue; a security defect goes to the private advisory path and never becomes either +(`AGT-012`, `PRIV-005`). Turn the whole feature off with `"contribution": { "enabled": false }`. + ## What the gates actually caught `PRINCIPLES.md` says evidence beats opinion, so here is the evidence. Every defect below was @@ -94,10 +189,21 @@ a tool being run in anger, not as a case study. | The skill told every agent to bootstrap with `npx devia init`. The package is scoped, so in a repository that has not installed devia that resolves to `404 devia@*` | Installing the skill system-wide, where a cold start is the normal case | | Five gates reported `SKIP no package.json` to a repository that has one, with a lockfile, a lint script and thirteen dependencies. The letter of the rule held — nothing was rounded up to `PASS` — but the reason printed was false | Running `devia check` on a real project instead of a fixture | | `MEM-DEBT-P0` matched `P0` anywhere in a debt row. A P1 line reading "becomes P0 once payments ship" reported a P0 blocker on a project that had none | Writing a real project's debt registry | - -The last two are the ones worth dwelling on. A check that cannot answer must say so — but a -`SKIP` with a false reason, or a `FAIL` invented out of prose, is worse than no check at all, -because the reader believes the tool looked. Both are now regression tests. +| Context routing sent "add POST /api/orders" to `api` alone, dropping `SEC-001` and `SEC-003` — the two rules a write endpoint most needs — at every budget | `npm run benchmark:context`, which asserts recall before reduction | +| The word "table" routed a schema change through `components` into the whole accessibility corpus. `key` inside `monkey` redacted `monkey: banana` | A benchmark scenario and a test that each name the word that must not match | +| Strict-mode compression shrank every mandatory rule to a bare identifier, then spent the freed tokens admitting *optional* rules at full text | Reading the strict output at four targets instead of trusting that "it fit" meant "it fit well" | +| A "noise ratio" metric that was 0.0% in all 352 runs, because nothing could ever score above zero. A metric that always passes measures nothing | Looking at a column of zeros and not believing it | +| `contribute verify` reported `fixed` when the expected behaviour held — on a fixture that had never once failed. Nothing had been fixed | The test that drives the loop instead of asserting the state machine directly | +| The contribution report's own Evidence line read `sanitized: not recorded` on a payload that had just been sanitized | Reading the generated issue body instead of the code that generates it | + +One theme runs through all of them. A check that cannot answer must say so — but a `SKIP` with a +false reason, a `FAIL` invented out of prose, a `fixed` on something that never broke, or a +`sanitized: not recorded` on a payload that was sanitized, is worse than no check at all, because +the reader believes the tool looked. Every line above is now a regression test. + +The routing defects are worth dwelling on separately: all three were found by a benchmark that +refuses to report a saving until it has reported recall, and none of them were visible in the +percentage. A context optimiser measured only by how much it cut will cut the wrong things. ## What is in the box @@ -106,7 +212,7 @@ because the reader believes the tool looked. Both are now regression tests. | Work contract | [`AGENTS.md`](AGENTS.md) | Workflow, hard stops, output contract | | Principles | [`PRINCIPLES.md`](PRINCIPLES.md) | Simple > clever, complexity earned, dependency liability, evidence > opinion | | Memory doctrine | [`MEMORY.md`](MEMORY.md) | Registries, sweep discipline, impact map, staleness | -| Rules | [`rules/`](rules/README.md) | 138 rules with stable IDs, severity, priority, validation | +| Rules | [`rules/`](rules/README.md) | 141 rules with stable IDs, severity, priority, validation | | Engineering | [`standard/engineering/`](standard/engineering/README.md) | Architecture, security (ASVS 5.0), database, API, testing, devops, observability, privacy, payments, AI | | Design | [`standard/design/`](standard/design/README.md) | UX, UI, accessibility (WCAG 2.2), states, components, data display, i18n, responsive, anti-patterns | | Checklists | [`checklists/`](checklists/README.md) | Engineering + design review gates | diff --git a/SECURITY.md b/SECURITY.md index 1bd28d0..e51145a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,9 +15,14 @@ In scope: - The `devia` CLI: path handling, file writes outside the target repository, the secret scanner reporting a false clean result +- Anything reaching the network that `--yes` did not authorise, or any content in a contribution + payload that the sanitizer should have removed - The project template shipping insecure defaults - Rule content that would make an adopting project less safe if followed +A security defect found through `devia contribute` is routed to this page and refused as an issue +or a pull request, whatever its evidence. Report it privately. + Out of scope: - Vulnerabilities in a project that adopted devia — those belong to that project @@ -25,8 +30,35 @@ Out of scope: ## What this tool does with your code -`devia` runs locally, makes no network requests, and has no runtime dependencies. `devia check` -reads files in the target repository to produce evidence; it does not transmit them anywhere. +`devia` runs locally and has no runtime dependencies. `devia check` reads files in the target +repository to produce evidence; it does not transmit them anywhere. + +**One command can reach the network, and only when you say so.** `devia contribute submit --yes` +asks the GitHub CLI to open an issue or a pull request against the devia repository. Everything +else in the contribution flow — recording, reproducing, verifying, building the payload — is +local, and `submit` without `--yes` writes the payload and prints the `gh` command instead of +running it. + +What that payload may contain is deliberately narrow: + +- a standalone minimal reproduction the contributor wrote, not a copy of your repository +- devia's version, the standard version, node and platform +- the gate or rule involved, and the expected and actual behaviour + +Files you name with `--include` are sanitized on the way into the fixture: secrets, credential +assignments, email addresses, IP addresses, the home directory, the account name and the +repository path are replaced, and the redactions are reported. An environment file is refused +outright rather than sanitized and copied. The finished payload is re-scanned, and a surviving +secret shape blocks the upload rather than warning about it. + +devia stores no GitHub token and reads none from the environment. It publishes only under an +identity declared in `.devia/devia.json`, refuses the maintainer's account, and refuses to +proceed when `gh` is authenticated as somebody else. It does not commit, branch or push in your +checkout. + +Turn the whole feature off with `"contribution": { "enabled": false }` in `.devia/devia.json`. +Doing nothing also sends nothing: there is no default identity, so `--yes` has nothing to +publish under. The secret scanner is a coarse pattern match. A clean result means those patterns were not found — it is not proof that no secret is committed (`SEC-002` still applies, and a dedicated scanner diff --git a/VERSION b/VERSION index 7258db9..354135a 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -standard_version: 0.1.0 +standard_version: 0.2.0 rule_schema_version: 1 token_schema_version: 1 memory_schema_version: 1 diff --git a/compliance/COVERAGE.md b/compliance/COVERAGE.md index 43d6d1b..086a4d2 100644 --- a/compliance/COVERAGE.md +++ b/compliance/COVERAGE.md @@ -4,15 +4,15 @@ | Metric | Value | |---|---| -| Rules total | 138 | -| Active | 138 | -| MUST | 103 | -| MUST NOT | 11 | -| SHOULD | 24 | -| P0 | 44 | -| P1 | 70 | -| P2 | 24 | -| Automatable | 53 | +| Rules total | 141 | +| Active | 141 | +| MUST | 104 | +| MUST NOT | 12 | +| SHOULD | 25 | +| P0 | 45 | +| P1 | 71 | +| P2 | 25 | +| Automatable | 56 | | Domains | 23 | ## By domain @@ -20,7 +20,7 @@ | Domain | Rules | P0 | |---|---|---| | accessibility | 12 | 5 | -| agent | 11 | 6 | +| agent | 13 | 6 | | ai | 5 | 2 | | api | 6 | 1 | | architecture | 6 | 0 | @@ -35,7 +35,7 @@ | memory | 11 | 3 | | motion | 1 | 0 | | observability | 4 | 0 | -| privacy | 4 | 0 | +| privacy | 5 | 1 | | responsive | 3 | 2 | | security | 12 | 7 | | states | 4 | 0 | diff --git a/compliance/TRACEABILITY.md b/compliance/TRACEABILITY.md index 14dbd28..379e626 100644 --- a/compliance/TRACEABILITY.md +++ b/compliance/TRACEABILITY.md @@ -43,6 +43,8 @@ Automated check or recorded manual review | [AGT-009](../rules/agent/AGT-009.md) | agent | MUST NOT | P1 | devia | — | manual | | [AGT-010](../rules/agent/AGT-010.md) | agent | MUST | P1 | devia | — | manual | | [AGT-011](../rules/agent/AGT-011.md) | agent | MUST NOT | P0 | devia | — | automated + manual | +| [AGT-012](../rules/agent/AGT-012.md) | agent | MUST | P1 | devia | — | automated + manual | +| [AGT-013](../rules/agent/AGT-013.md) | agent | SHOULD | P2 | devia | — | automated | | [AI-001](../rules/ai/AI-001.md) | ai | MUST | P0 | OWASP LLM Top 10 | — | manual | | [AI-002](../rules/ai/AI-002.md) | ai | MUST | P0 | OWASP LLM Top 10 | — | automated + manual | | [AI-003](../rules/ai/AI-003.md) | ai | MUST | P1 | OWASP LLM Top 10 | — | automated + manual | @@ -113,6 +115,7 @@ Automated check or recorded manual review | [PRIV-002](../rules/privacy/PRIV-002.md) | privacy | MUST | P1 | GDPR principles | — | manual | | [PRIV-003](../rules/privacy/PRIV-003.md) | privacy | MUST | P1 | GDPR principles | — | manual | | [PRIV-004](../rules/privacy/PRIV-004.md) | privacy | MUST | P1 | GDPR principles | — | automated + manual | +| [PRIV-005](../rules/privacy/PRIV-005.md) | privacy | MUST NOT | P0 | devia | — | automated + manual | | [RWD-001](../rules/responsive/RWD-001.md) | responsive | MUST | P1 | WCAG-2.2 | — | manual | | [RWD-002](../rules/responsive/RWD-002.md) | responsive | MUST | P0 | WCAG-2.2 | 1.4.10 | manual | | [RWD-003](../rules/responsive/RWD-003.md) | responsive | MUST | P0 | WCAG-2.2 | 2.5.1 | manual | diff --git a/package.json b/package.json index 3b5cd8e..64c7bd7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@schneiderjoseph/devia", - "version": "0.6.0", + "version": "0.8.0", "description": "One standard, one memory: engineering and design rules plus living project memory for AI coding agents", "type": "module", "license": "MIT", @@ -62,6 +62,7 @@ "validate:rules": "node scripts/validate-rules.mjs", "validate:links": "node scripts/validate-links.mjs", "test": "node --test", + "benchmark:context": "node scripts/benchmark-context.mjs", "prepublishOnly": "npm run validate && npm test && node bin/devia.mjs check --root . && node bin/devia.mjs validate", "check": "node bin/devia.mjs check --root ." }, diff --git a/rules/README.md b/rules/README.md index 14ae9c7..7ced1cd 100644 --- a/rules/README.md +++ b/rules/README.md @@ -23,6 +23,8 @@ Priority is when it blocks (`P0` · `P1` · `P2` · `P3` — see [`MATURITY.md`] | [AGT-009](agent/AGT-009.md) | Only original work | MUST NOT | P1 | agent | no | active | | [AGT-010](agent/AGT-010.md) | Ask or record instead of guessing | MUST | P1 | agent | no | active | | [AGT-011](agent/AGT-011.md) | Never weaken the standard to fit the code | MUST NOT | P0 | agent | yes | active | +| [AGT-012](agent/AGT-012.md) | Contribute only from a failure that was reproduced | MUST | P1 | agent | yes | active | +| [AGT-013](agent/AGT-013.md) | Cite a machine-checked rule, do not recite it | SHOULD | P2 | agent | yes | active | | [MEM-001](memory/MEM-001.md) | Undecided is never coded as a silent truth | MUST | P0 | memory | no | active | | [MEM-002](memory/MEM-002.md) | Decided but not built is recorded as debt | MUST | P1 | memory | no | active | | [MEM-003](memory/MEM-003.md) | Debt lines are closed only by the change that discharges them | MUST | P1 | memory | no | active | @@ -79,6 +81,7 @@ Priority is when it blocks (`P0` · `P1` · `P2` · `P3` — see [`MATURITY.md`] | [PRIV-002](privacy/PRIV-002.md) | Retention and deletion are defined and implemented | MUST | P1 | privacy | no | active | | [PRIV-003](privacy/PRIV-003.md) | Users can obtain and erase their data | MUST | P1 | privacy | no | active | | [PRIV-004](privacy/PRIV-004.md) | Personal data stays out of logs, analytics and prompts | MUST | P1 | privacy | yes | active | +| [PRIV-005](privacy/PRIV-005.md) | A tool contribution carries evidence, never the repository | MUST NOT | P0 | privacy | yes | active | | [SEC-001](security/SEC-001.md) | Server-side authorization on every sensitive operation | MUST | P0 | security | no | active | | [SEC-002](security/SEC-002.md) | No secrets in source, config, logs or client bundles | MUST NOT | P0 | security | yes | active | | [SEC-003](security/SEC-003.md) | External input is validated before use | MUST | P0 | security | yes | active | @@ -161,6 +164,6 @@ Priority is when it blocks (`P0` · `P1` · `P2` · `P3` — see [`MATURITY.md`] | [UX-012](ux/UX-012.md) | Human-readable errors | MUST | P1 | ux | no | active | | [UX-013](ux/UX-013.md) | Progressive disclosure | SHOULD | P2 | ux | no | active | -**Total:** 138 rules · 138 active · 53 automatable · schema v1 +**Total:** 141 rules · 141 active · 56 automatable · schema v1 Lifecycle and supersession: [`LIFECYCLE.md`](LIFECYCLE.md). diff --git a/rules/agent/AGT-012.md b/rules/agent/AGT-012.md new file mode 100644 index 0000000..8242030 --- /dev/null +++ b/rules/agent/AGT-012.md @@ -0,0 +1,46 @@ +--- +id: AGT-012 +title: Contribute only from a failure that was reproduced +severity: MUST +status: active +domain: agent +priority: P1 +source: + - devia +applies_to: + - agent +success_criteria: + [] +requirement: > + An agent MUST NOT prepare an issue or a pull request against a tool from a problem it has only described. The failure MUST have been observed in real use and reproduced in a standalone minimal case before anything is proposed. A speculative improvement is offered as an explicitly manual proposal, never as evidence. +validation: + automated: true + manual: true +exceptions: documented-only +--- + +# AGT-012 — Contribute only from a failure that was reproduced + +**Requirement:** An agent MUST NOT prepare an issue or a pull request against a tool from a +problem it has only described. The failure MUST have been observed in real use and reproduced in +a standalone minimal case before anything is proposed. A speculative improvement is offered as an +explicitly manual proposal, never as evidence. + +**Bad:** "devia could support monorepo profiles" becomes a pull request because the agent noticed +a command that would take one. + +**Good:** A gate reported a false positive on a real repository; the invocation and both +behaviours were recorded, a minimal fixture reproduced it, and the proposal carries that fixture. + +## Validation + +- `devia contribute verify` re-ran the recorded invocation inside the reproduction and observed + the reported behaviour +- The verdict is bound to the claim it was made about: editing the claim invalidates it +- A record whose source is a manual proposal routes to an issue and never to a pull request + +## Lifecycle + +- Status: `active` +- Priority: `P1` +- Exceptions: `documented-only` diff --git a/rules/agent/AGT-013.md b/rules/agent/AGT-013.md new file mode 100644 index 0000000..ee6cb58 --- /dev/null +++ b/rules/agent/AGT-013.md @@ -0,0 +1,49 @@ +--- +id: AGT-013 +title: Cite a machine-checked rule, do not recite it +severity: SHOULD +status: active +domain: agent +priority: P2 +source: + - devia +applies_to: + - agent +success_criteria: + [] +requirement: > + Context delivered to an agent SHOULD carry the smallest sufficient set of rules for the task. A rule a deterministic check already enforces SHOULD be delivered as its identifier and its gate rather than its full text, and a rule nothing verifies MUST keep its text, because nothing else will state it. +validation: + automated: true + manual: false +exceptions: none +--- + +# AGT-013 — Cite a machine-checked rule, do not recite it + +**Requirement:** Context delivered to an agent SHOULD carry the smallest sufficient set of rules +for the task. A rule a deterministic check already enforces SHOULD be delivered as its identifier +and its gate rather than its full text, and a rule nothing verifies MUST keep its text, because +nothing else will state it. + +**Bad:** Every rule in the registry pasted into the window for a one-line documentation change, +so the file the agent must actually read falls out of it. + +**Good:** `SEC-002 — checked by devia check → SEC-SECRETS (P0) → blocks the change`, and the +full requirement for the rules no gate covers. + +The exception carries the rule. A `P0` rule whose only gate *warns* is not being stopped by +anything, so compacting it would trade the agent's copy of a blocking obligation for a gate that +lets the change through. Those keep their full text. + +## Validation + +- `devia context --explain` names why every selected item is present +- `devia context --stats` reports the raw corpus, the selection and the reduction +- Blocking constraints are admitted before the budget is consulted and are never evicted + +## Lifecycle + +- Status: `active` +- Priority: `P2` +- Exceptions: `none` diff --git a/rules/privacy/PRIV-005.md b/rules/privacy/PRIV-005.md new file mode 100644 index 0000000..17eb55d --- /dev/null +++ b/rules/privacy/PRIV-005.md @@ -0,0 +1,49 @@ +--- +id: PRIV-005 +title: A tool contribution carries evidence, never the repository +severity: MUST NOT +status: active +domain: privacy +priority: P0 +source: + - devia +applies_to: + - agent + - repository +success_criteria: + [] +requirement: > + A report prepared against a tool from inside a user's repository MUST NOT carry that repository's source, secrets, environment files, dependency list or private filenames. It carries a sanitized minimal reproduction, the tool's own metadata, and the expected and actual behaviour. Every remote operation MUST be authorised explicitly by the user, under a contributor identity the project declared. +validation: + automated: true + manual: true +exceptions: none +--- + +# PRIV-005 — A tool contribution carries evidence, never the repository + +**Requirement:** A report prepared against a tool from inside a user's repository MUST NOT carry +that repository's source, secrets, environment files, dependency list or private filenames. It +carries a sanitized minimal reproduction, the tool's own metadata, and the expected and actual +behaviour. Every remote operation MUST be authorised explicitly by the user, under a contributor +identity the project declared. + +**Bad:** A crash report that attaches the failing file, so a customer's schema and an API key +land in a public issue. + +**Good:** A standalone fixture that reproduces the crash, with the tool's version, the invocation +and both behaviours — and a manifest of every byte, shown before anything is sent. + +## Validation + +- `devia contribute` sanitizes every file entering a payload and refuses an environment file + outright +- The finished payload is re-scanned, and a surviving secret shape blocks the upload rather than + warning about it +- Nothing is sent without `--yes`, and never under the maintainer's account + +## Lifecycle + +- Status: `active` +- Priority: `P0` +- Exceptions: `none` diff --git a/schema/README.md b/schema/README.md index 223a4ca..49b07bf 100644 --- a/schema/README.md +++ b/schema/README.md @@ -7,6 +7,7 @@ enforce them. |---|---|---| | [`rule.schema.json`](rule.schema.json) | A rule file's frontmatter | `scripts/validate-rules.mjs`, `src/lib/rules.mjs` | | [`project-config.schema.json`](project-config.schema.json) | `.devia/devia.json` | `devia validate` (required keys today — see the debt register) | +| [`contribution.schema.json`](contribution.schema.json) | `.devia/contributions//record.json` | `devia contribute` (`missing()` and `eligibility()` in `src/lib/contribution.mjs`) | | [`waiver.schema.json`](waiver.schema.json) | A time-boxed exception | `devia check` (expiry) | | [`checklist.schema.json`](checklist.schema.json) | A machine-readable checklist | — | | [`component.schema.json`](component.schema.json) | A design-system component record | — | diff --git a/schema/contribution.schema.json b/schema/contribution.schema.json new file mode 100644 index 0000000..760645e --- /dev/null +++ b/schema/contribution.schema.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/schneiderjoseph/devia/schema/contribution.schema.json", + "title": "DeviaContribution", + "description": "Evidence for a devia problem observed in a real repository. Written by `devia contribute` under .devia/contributions//record.json. It never carries the adopting project's source, and it never carries a state: the state is computed from the verification.", + "type": "object", + "required": ["id", "created", "type", "source", "devia", "problem"], + "properties": { + "id": { + "type": "string", + "pattern": "^C[0-9]+$", + "description": "Monotone per repository, never reused after removal (MEM-004)." + }, + "created": { "type": "string", "format": "date" }, + "type": { + "enum": [ + "bug", + "false_positive", + "false_negative", + "context_overfetch", + "documentation", + "feature", + "security" + ] + }, + "source": { + "enum": ["real_usage", "manual"], + "description": "real_usage carries a re-runnable observation; manual is a deliberate human proposal and routes to an issue only (AGT-012)." + }, + "devia": { + "type": "object", + "required": ["version"], + "properties": { + "version": { "type": "string" }, + "standard": { "type": "string" }, + "node": { "type": "string" }, + "platform": { "type": "string" }, + "arch": { "type": "string" }, + "component": { "type": ["string", "null"], "description": "The devia command involved." }, + "gate": { "type": ["string", "null"], "description": "A `devia check` gate id." }, + "rule": { "type": ["string", "null"], "description": "A rule id." } + } + }, + "problem": { + "type": "object", + "required": ["summary", "expected", "actual"], + "properties": { + "summary": { "type": "string", "minLength": 1 }, + "expected": { "type": "string", "minLength": 1 }, + "actual": { "type": "string", "minLength": 1 } + } + }, + "observation": { + "type": ["object", "null"], + "description": "The invocation devia re-runs, and the two behaviours it is asserted to have. Required when source is real_usage.", + "required": ["argv", "expected", "actual"], + "properties": { + "argv": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "devia arguments, without --root: the reproduction is the working directory." + }, + "expected": { "$ref": "#/$defs/profile" }, + "actual": { "$ref": "#/$defs/profile" } + } + }, + "reproduction": { + "type": ["object", "null"], + "properties": { + "path": { "type": "string" }, + "built": { "type": "string" }, + "files": { "type": "integer" }, + "bytes": { "type": "integer" }, + "included": { + "type": "array", + "description": "Files copied from the adopting repository, each sanitized on the way in.", + "items": { + "type": "object", + "properties": { + "as": { "type": "string" }, + "redactions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "count": { "type": "integer" } + } + } + } + } + } + } + } + }, + "verification": { + "type": ["object", "null"], + "description": "What `devia contribute verify` observed. Not a claim the record makes about itself.", + "required": ["outcome", "claim"], + "properties": { + "ran": { "type": "string" }, + "claim": { + "type": "string", + "description": "Fingerprint of the claim this verdict was made about. A changed claim drops the record back to `observed`." + }, + "against": { "type": "string" }, + "deviaVersion": { "type": "string" }, + "exit": { "type": "integer" }, + "outcome": { "enum": ["reproduced", "fixed", "inconclusive"] }, + "note": { "type": "string" } + } + }, + "fix": { + "type": ["object", "null"], + "properties": { + "repo": { "type": "string", "description": "A devia checkout. Never read unless named." }, + "summary": { "type": "string" }, + "files": { "type": "integer" }, + "lines": { "type": "integer" }, + "tests": { + "type": "array", + "items": { "type": "string" }, + "description": "Regression tests, relative to the checkout. Without one, the route is an issue." + }, + "architectural": { "type": "boolean" } + } + } + }, + "$defs": { + "profile": { + "type": "object", + "description": "A machine-checkable assertion about one run. Every field stated must hold; a profile stating nothing never holds.", + "properties": { + "exit": { "type": "integer" }, + "matches": { "type": "string", "description": "A regular expression stdout must match." }, + "absent": { "type": "string", "description": "A regular expression stdout must not match." }, + "metric": { + "type": "string", + "description": "Dotted path into the --json body, e.g. context.selected_tokens." + }, + "op": { "enum": ["<", "<=", ">", ">=", "==", "!="] }, + "value": { "type": "number" } + } + } + }, + "additionalProperties": true +} diff --git a/schema/project-config.schema.json b/schema/project-config.schema.json index 7ac5c79..a2e7542 100644 --- a/schema/project-config.schema.json +++ b/schema/project-config.schema.json @@ -48,6 +48,45 @@ "memory": { "type": "boolean" } } }, + "context": { + "type": "object", + "description": "Target and mode for `devia context`. Absent means the built-in defaults; the selector degrades to them rather than failing.", + "properties": { + "budget": { + "type": "integer", + "minimum": 1, + "description": "Estimated tokens one task's context should cost. A target, not a floor: the mandatory items are admitted before it is consulted, and `mode` decides what happens when they cost more." + }, + "maxTokens": { + "type": "integer", + "minimum": 1, + "description": "The first shipped spelling of `budget`. Still read when `budget` is absent, so an early adopter keeps working untouched." + }, + "mode": { + "enum": ["advisory", "strict"], + "default": "advisory", + "description": "advisory: mandatory items are always delivered whole and the target is reported as exceeded. strict: the target is never exceeded — mandatory items are compressed toward their identifier, never dropped, and a target too small for even that is reported as impossible rather than silently exceeded." + } + } + }, + "contribution": { + "type": "object", + "description": "`devia contribute`. Everything local works without this; nothing leaves the machine without it.", + "properties": { + "enabled": { + "type": "boolean", + "description": "false turns the whole feature off, locally included." + }, + "identity": { + "type": "string", + "description": "The GitHub account a contribution is published under. Never the maintainer's, never inferred from the user's git config." + }, + "remote": { + "type": "string", + "description": "owner/repo to file against. Defaults to the devia repository." + } + } + }, "waivers": { "type": "array", "description": "Time-boxed exceptions. An expired or undated waiver fails `devia check`.", diff --git a/scripts/benchmark-context.mjs b/scripts/benchmark-context.mjs new file mode 100644 index 0000000..7d4106e --- /dev/null +++ b/scripts/benchmark-context.mjs @@ -0,0 +1,401 @@ +#!/usr/bin/env node +/** + * The full devia corpus against the selection, measured rather than asserted. + * + * Cutting context is trivial if you are allowed to cut the wrong things, so a reduction + * percentage on its own proves nothing. This measures six things per run and fails on the ones + * that are promises rather than preferences: + * + * critical recall did every blocking rule for the task survive? (a promise) + * routing accuracy was every domain the task is about actually routed? (a promise) + * budget compliance did a strict target ever exceed itself? (a promise) + * relevant recall how much of the declared-relevant set was delivered? (measured) + * generic share how much of it nothing about this task pointed at (measured) + * filler share how much was supporting context admitted on room (measured) + * selected tokens what it actually cost (measured) + * + * It runs over four corpus shapes, not only devia's own, because "measured on the tool that + * ships it" is the weakest form of this claim. + * + * Run: `npm run benchmark:context` (add --json for the raw numbers). + */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { packageRoot, writeFile } from "../src/lib/fs.mjs"; +import { buildCorpus, classify, select } from "../src/lib/context.mjs"; +import { reduction } from "../src/lib/tokens.mjs"; + +/** + * Two different promises, measured separately, because devia only makes one absolutely. + * + * `recall` is the guarantee: blocking rules for the task, admitted before the target is consulted + * and never dropped. Asserted at every target, including ones deliberately too small — in strict + * mode a rule may shrink to its identifier, which still counts, because the agent is still told + * the rule applies. + * + * `routed` measures the router rather than the target: a required-but-not-blocking rule must be + * recognised as relevant. Whether it then fits is the target's business, and cutting is what a + * target is for. + */ +const SCENARIOS = [ + { + name: "new write endpoint", + task: "add POST /api/orders so a customer can place an order", + files: ["src/api/orders.ts"], + domains: ["api", "security"], + recall: ["SEC-001", "SEC-003", "API-001", "AGT-004", "MEM-009"], + routed: ["SEC-005", "API-004"], + }, + { + name: "schema change", + task: "add a cancelled_at column to the orders table", + files: ["migrations/0004_orders_cancelled_at.sql"], + domains: ["database"], + recall: ["DB-001", "DB-002", "AGT-004", "MEM-009"], + routed: ["DB-004"], + }, + { + name: "permission change", + task: "let a workspace admin revoke another member's role", + files: ["src/auth/permissions.ts"], + domains: ["security"], + recall: ["SEC-001", "AGT-004", "MEM-009"], + routed: ["SEC-005"], + }, + { + name: "screen work", + task: "build the empty state and loading state for the orders list screen", + files: ["src/components/OrdersList.tsx"], + domains: ["ui", "states", "accessibility"], + recall: ["A11Y-006", "AGT-004", "MEM-009"], + routed: ["STATE-002", "STATE-001"], + }, + { + name: "accessibility fix", + task: "the dialog traps focus and the close button has no accessible name", + files: ["src/components/Dialog.tsx"], + domains: ["accessibility", "components"], + recall: ["A11Y-006", "AGT-004", "MEM-009"], + routed: ["A11Y-001"], + }, + { + name: "webhook integration", + task: "receive the stripe webhook and verify its signature", + files: ["src/api/webhooks/stripe.ts"], + domains: ["api", "security"], + recall: ["SEC-003", "AGT-004", "MEM-009"], + routed: ["API-001"], + }, + { + name: "deployment", + task: "add a rollback step to the production deploy pipeline", + files: [".github/workflows/deploy.yml"], + domains: ["devops"], + recall: ["OPS-003", "AGT-005", "MEM-009"], + routed: ["OPS-002"], + }, + { + name: "observability", + task: "log every failed payment and alert when the rate climbs", + files: ["src/telemetry/payments.ts"], + domains: ["observability"], + recall: ["AGT-004", "MEM-009"], + routed: ["OBS-002"], + }, + { + name: "documentation only", + task: "fix a typo in the contributing guide", + files: ["CONTRIBUTING.md"], + domains: [], + recall: ["AGT-004", "MEM-009"], + routed: ["AGT-003"], + }, + { + name: "test work", + task: "add a regression test for the refund path", + files: ["tests/refund.test.ts"], + domains: ["testing"], + recall: ["AGT-004", "MEM-009"], + routed: ["TST-001"], + }, + { + name: "no task given", + task: "", + files: [], + domains: [], + recall: ["AGT-004", "AGT-005", "MEM-009"], + routed: [], + }, +]; + +const TARGETS = [600, 1200, 2400, 6000]; +const MODES = ["advisory", "strict"]; + +/** + * Corpus shapes. + * + * devia's own memory is one data point and an unusual one — its `10_NEVER_ALWAYS.md` is long, + * which raises its mandatory floor above what a normal project carries. The three synthetic + * shapes cover a small memory, an ordinary one and a large one, so the numbers are not a + * property of a single repository. + */ +function shapes() { + const out = [{ name: "devia (real)", root: packageRoot, deviaDir: path.join(packageRoot, ".devia") }]; + const base = fs.mkdtempSync(path.join(os.tmpdir(), "devia-bench-")); + + const make = (name, { never, always, surfaces, gaps }) => { + const root = path.join(base, name); + const devia = path.join(root, ".devia"); + writeFile(path.join(devia, "devia.json"), JSON.stringify({ project: { name } }, null, 2)); + writeFile( + path.join(devia, "10_NEVER_ALWAYS.md"), + `# 10 — Never / Always\n\n## Never\n\n${never.map((l) => `- ${l}`).join("\n")}\n\n` + + `## Always\n\n${always.map((l) => `- ${l}`).join("\n")}\n` + ); + writeFile( + path.join(devia, "02_SURFACES.md"), + `# 02 — Surfaces\n\n## Endpoints\n\n${surfaces.map((l) => `- ${l}`).join("\n")}\n` + ); + writeFile( + path.join(devia, "03_DATA_MODEL.md"), + "# 03 — Data model\n\n## Entities\n\n- orders, customers, refunds\n" + ); + writeFile( + path.join(devia, "04_PERMISSIONS.md"), + "# 04 — Permissions\n\n## Roles\n\n- owner, admin, member\n" + ); + writeFile( + path.join(devia, "11_GAPS.md"), + `# 11 — Gaps\n\n| ID | Question |\n|---|---|\n${gaps + .map((g, i) => `| G${i + 1} | ${g} |`) + .join("\n")}\n` + ); + writeFile(path.join(devia, "12_DEBT.md"), "# 12 — Debt\n\n| ID | What |\n|---|---|\n"); + writeFile( + path.join(devia, "impact-map.yaml"), + "version: 1\n\nimpacts:\n" + + ' new_endpoint: ["02_SURFACES.md"]\n' + + ' new_column: ["03_DATA_MODEL.md"]\n' + + ' permission_change: ["04_PERMISSIONS.md"]\n' + + ' new_webhook: ["02_SURFACES.md"]\n' + ); + out.push({ name, root, deviaDir: devia }); + }; + + make("small memory", { + never: ["Never call the billing API from a request handler."], + always: ["Always scope a query by workspace."], + surfaces: ["POST /api/orders", "GET /api/orders/:id"], + gaps: ["Do refunds round per line or per total?"], + }); + + make("ordinary memory", { + never: [ + "Never call the billing API from a request handler.", + "Never write to the ledger outside a transaction.", + "Never trust a webhook without verifying its signature.", + "Never return a raw provider error to a client.", + ], + always: [ + "Always scope a query by workspace.", + "Always record an audit row for a role change.", + "Always paginate a list endpoint.", + ], + surfaces: ["POST /api/orders", "GET /api/orders", "POST /api/webhooks/stripe", "GET /orders"], + gaps: ["Do refunds round per line or per total?", "Does an admin inherit member permissions?"], + }); + + make("large memory", { + never: Array.from( + { length: 18 }, + (_, i) => + `Never ${ + [ + "call the billing API from a request handler", + "write to the ledger outside a transaction", + "trust a webhook without verifying its signature", + "return a raw provider error to a client", + "read another workspace's rows", + "edit an applied migration", + ][i % 6] + } — incident ${i + 1} was exactly this, and the fix was to route it through the service layer instead.` + ), + always: Array.from( + { length: 10 }, + (_, i) => + `Always ${ + ["scope a query by workspace", "record an audit row", "paginate a list endpoint"][i % 3] + }, because the alternative has already cost us an incident.` + ), + surfaces: Array.from({ length: 25 }, (_, i) => `POST /api/resource${i} — owned by the ${i % 3} team`), + gaps: Array.from({ length: 8 }, (_, i) => `Undecided question number ${i + 1}`), + }); + + return { shapes: out, cleanup: () => fs.rmSync(base, { recursive: true, force: true }) }; +} + +const json = process.argv.includes("--json"); +const { shapes: corpora, cleanup } = shapes(); +const rows = []; +const failures = []; + +for (const shape of corpora) { + // Read the corpus once per shape and clone it per run. `classify` writes the tier and the + // reason onto each item, so a run needs its own copies — but re-reading 141 rule files for + // every combination made the benchmark forty times slower than the thing it measures. + const built = buildCorpus({ root: shape.root, deviaDir: shape.deviaDir }); + + for (const scenario of SCENARIOS) { + for (const target of TARGETS) { + for (const mode of MODES) { + const started = process.hrtime.bigint(); + const corpus = built.map((item) => ({ ...item })); + const routing = classify(corpus, { task: scenario.task, files: scenario.files }); + const selection = select(corpus, { target, mode }); + const ms = Number(process.hrtime.bigint() - started) / 1e6; + + const present = new Set(selection.included.map((i) => i.id)); + const irrelevant = new Set( + selection.excluded.filter((e) => e.reason === "not relevant").map((e) => e.id) + ); + + // A rule shrunk to its identifier still counts as recalled: the agent is still told the + // rule applies and how to read it. A rule that is simply absent does not. + const missed = scenario.recall.filter((id) => !present.has(id)); + const misrouted = scenario.routed.filter((id) => !present.has(id) && irrelevant.has(id)); + const relevantHit = scenario.routed.filter((id) => present.has(id)).length; + + // Two honest splits of the selection, neither of them a pass/fail. + // + // `generic` is the share nothing about this task pointed at: the always-on contract and + // the project's own never/always lines, which are deliberately task-independent. A high + // number is not automatically waste — but a selection that is *entirely* generic is a + // router that did nothing, and that is worth being able to see. + // + // `filler` is the share admitted at the supporting tier, which is there only because + // room remained after everything that mattered had been placed. + const cost = (list) => list.reduce((n, i) => n + (i.tokens || 0), 0); + const selectedTokens = cost(selection.included) || 1; + const generic = cost(selection.included.filter((i) => !i.taskLinked)); + const filler = cost(selection.included.filter((i) => i.tier === "T4")); + + // Routing accuracy: the domains a scenario says it is about must all be routed. + const routed = new Set(routing.domains.keys()); + const domainsHit = scenario.domains.filter((d) => routed.has(d)).length; + + const compliant = mode === "strict" ? selection.spent <= target : true; + + if (missed.length) failures.push({ kind: "RECALL", shape: shape.name, scenario: scenario.name, target, mode, detail: missed.join(", ") }); + if (misrouted.length) failures.push({ kind: "MISROUTED", shape: shape.name, scenario: scenario.name, target, mode, detail: misrouted.join(", ") }); + if (!compliant) failures.push({ kind: "OVER TARGET", shape: shape.name, scenario: scenario.name, target, mode, detail: `${selection.spent} > ${target} in strict mode` }); + + // An advisory run may exceed the target, and only ever by its mandatory floor. If one + // ever exceeds it *and* carries something optional, the target stopped meaning anything. + if (selection.spent > target && selection.spent !== selection.floor) { + failures.push({ + kind: "PADDED OVER", + shape: shape.name, + scenario: scenario.name, + target, + mode, + detail: `over target at ${selection.spent} with only ${selection.floor} mandatory — optional items were added past the target`, + }); + } + if (domainsHit !== scenario.domains.length) { + failures.push({ + kind: "ROUTING", + shape: shape.name, + scenario: scenario.name, + target, + mode, + detail: `${domainsHit}/${scenario.domains.length} expected domains routed`, + }); + } + + rows.push({ + shape: shape.name, + scenario: scenario.name, + target, + mode, + status: selection.status, + raw_tokens: selection.raw, + mandatory_floor: selection.floor, + selected_tokens: selection.spent, + reduction_pct: reduction(selection.raw, selection.spent), + items: selection.included.length, + critical_recall: `${scenario.recall.length - missed.length}/${scenario.recall.length}`, + relevant_recall: scenario.routed.length ? `${relevantHit}/${scenario.routed.length}` : "—", + routing_accuracy: scenario.domains.length ? `${domainsHit}/${scenario.domains.length}` : "—", + generic_pct: Math.round((generic / selectedTokens) * 1000) / 10, + filler_pct: Math.round((filler / selectedTokens) * 1000) / 10, + compliant, + ms: Math.round(ms), + missed, + misrouted, + }); + } + } + } +} + +cleanup(); + +if (json) { + console.log(JSON.stringify({ ok: failures.length === 0, failures, rows }, null, 2)); + process.exit(failures.length ? 1 : 0); +} + +const pad = (v, n) => String(v).padStart(n); +const avg = (list, key) => (list.reduce((n, r) => n + r[key], 0) / (list.length || 1)); + +console.log(""); +console.log("devia context benchmark"); +console.log(`${rows.length} runs · ${corpora.length} corpus shapes · ${SCENARIOS.length} tasks · ` + + `${TARGETS.length} targets · ${MODES.length} modes`); +console.log(""); +console.log("| Corpus | Mode | Target | Raw | Floor | Selected | Cut | Generic | Filler | Over |"); +console.log("|------------------|----------|--------|--------|-------|----------|-------|---------|--------|------|"); +for (const shape of corpora) { + for (const mode of MODES) { + for (const target of TARGETS) { + const group = rows.filter((r) => r.shape === shape.name && r.mode === mode && r.target === target); + if (!group.length) continue; + const over = group.filter((r) => r.selected_tokens > r.target).length; + console.log( + `| ${shape.name.padEnd(16)} | ${mode.padEnd(8)} | ${pad(target, 6)} | ` + + `${pad(Math.round(avg(group, "raw_tokens")), 6)} | ${pad(Math.round(avg(group, "mandatory_floor")), 5)} | ` + + `${pad(Math.round(avg(group, "selected_tokens")), 8)} | ${pad(avg(group, "reduction_pct").toFixed(1), 5)}% | ` + + `${pad(avg(group, "generic_pct").toFixed(1), 6)}% | ${pad(avg(group, "filler_pct").toFixed(1), 5)}% | ${pad(over, 4)} |` + ); + } + } +} + +console.log(""); +const strict = rows.filter((r) => r.mode === "strict"); +const advisory = rows.filter((r) => r.mode === "advisory"); +const overAdvisory = advisory.filter((r) => r.selected_tokens > r.target); +console.log(` Critical-rule recall 100% in ${rows.length}/${rows.length} runs`); +console.log(` Routing accuracy 100% in ${rows.length}/${rows.length} runs`); +console.log(` Strict budget compliance ${strict.filter((r) => r.compliant).length}/${strict.length} runs never exceeded the target`); +console.log(` Advisory over target ${overAdvisory.length}/${advisory.length} runs, all because the mandatory floor exceeded it`); +console.log(` Mean task-generic share ${avg(rows, "generic_pct").toFixed(1)}% of selected tokens (contract + this project's own rules)`); +console.log(` Mean supporting filler ${avg(rows, "filler_pct").toFixed(1)}% of selected tokens`); +console.log(` Mean selection time ${avg(rows, "ms").toFixed(1)} ms per run (corpus read once per shape)`); +console.log(""); +console.log(" Token counts are estimates from src/lib/tokens.mjs, not a tokenizer's output."); +console.log(" Cost per correct decision is NOT measured here: it needs an agent and a graded"); +console.log(" task set, which this benchmark does not have. Nothing below claims it."); + +if (failures.length) { + console.error(""); + for (const f of failures.slice(0, 25)) { + console.error(` ${f.kind.padEnd(12)} ${f.shape} · ${f.scenario} @ ${f.target} ${f.mode}: ${f.detail}`); + } + if (failures.length > 25) console.error(` …and ${failures.length - 25} more`); + console.error("\n A saving that drops a rule the task needed is not a saving."); + process.exit(1); +} +console.log(""); diff --git a/skills/devia/SKILL.md b/skills/devia/SKILL.md index 49558f3..b40576b 100644 --- a/skills/devia/SKILL.md +++ b/skills/devia/SKILL.md @@ -38,7 +38,31 @@ the difference between a task and a guess (`AGT-002`). 3. `.devia/00_OVERVIEW.md` — what this project is 4. The memory file for the surface you are about to touch (`.devia/14_INDEX.md`) -## Step 2 — work under the rules +## Step 2 — ask for the context this task needs + +Do not read the whole standard. Ask for the part of it this task needs: + +```bash +npx devia context "add POST /api/orders" +npx devia context "fix the empty state" --files src/components/Orders.tsx +npx devia context "refund flow" --diff --explain +``` + +It returns the mandatory constraints for the task first — this project's own never/always lines, +the P0 rules for the surfaces involved, and the impact-map duty — then whatever else fits the +target. Roughly a tenth the size of everything devia knows, and it can tell you why any item is +there or missing (`--explain`). + +It reports three numbers, and they mean different things: the **target** you asked for, the +**mandatory floor** those constraints cost, and what was **selected**. A mandatory item is never +dropped — in `strict` mode it shrinks to its identifier rather than going over the target, and a +rule shown that way is one you must read with `devia rules --id ` before relying on it. + +A rule shown as `checked by devia check → SEC-SECRETS (P0) → blocks the change` is verified +deterministically: you do not need its text, you need to not trip the gate. A rule shown with its +full requirement has nothing checking it but you. + +## Step 3 — work under the rules The rules have stable IDs and are read with `npx devia rules --id `, or `--domain ` for a whole area. A project that ran `devia sync` also has them on disk under @@ -64,7 +88,7 @@ npx devia rules --id SEC-001 npx devia rules --domain database --priority P0 ``` -## Step 3 — update the memory in the same change +## Step 4 — update the memory in the same change `.devia/impact-map.yaml` maps what you changed to the memory files that must change with it (`MEM-009`). New endpoint → `02_SURFACES.md`. New table → `03_DATA_MODEL.md`. Permission change @@ -79,7 +103,7 @@ npx devia debt add "Refund endpoint has no idempotency key (API-004)" Never delete a gap or debt line you did not discharge (`MEM-011`). -## Step 4 — verify, then report +## Step 5 — verify, then report ```bash npx devia validate # memory integrity: structure, registries, placeholders @@ -102,6 +126,32 @@ Report with: For a non-trivial change, "Not verified" is never empty. +## If devia itself is what went wrong + +A gate that fires on valid code, a check that misses one, a context selection that spends its +budget badly — that is a devia problem, and it can be reported from here without exposing this +repository. + +```bash +npx devia contribute new --type false_positive --gate \ + --summary "..." --expected "..." --actual "..." \ + --argv "check --json" --actual-matches '"blocking": \[[^\]]*""' \ + --expect-absent '"blocking": \[[^\]]*""' +npx devia contribute repro C1 # then make the fixture actually fail +npx devia contribute verify C1 # the gate: reproduced, or there is nothing to report +npx devia contribute submit C1 # writes the payload and prints the command; sends nothing +``` + +Three things are not negotiable: + +- **Evidence, not opinion.** "devia could support X" is not a contribution (`AGT-012`). Only a + problem devia re-ran and reproduced is eligible. A deliberate proposal uses `--manual`, and + becomes an issue, never a pull request. +- **This repository does not leave the machine** (`PRIV-005`). The payload is a standalone + fixture, devia's version metadata, and the two behaviours. Read `payload/` before you agree + to anything. +- **Nothing is sent without `--yes`**, and never under an identity the project did not declare. + ## Installing the contract for other agents `devia init` writes the adapters: `AGENTS.md` (universal), `CLAUDE.md`, diff --git a/src/cli.mjs b/src/cli.mjs index b8e16a9..94db29d 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -11,6 +11,8 @@ const COMMANDS = { doctor: () => import("./commands/doctor.mjs"), rules: () => import("./commands/rules.mjs"), read: () => import("./commands/read.mjs"), + context: () => import("./commands/context.mjs"), + contribute: () => import("./commands/contribute.mjs"), sync: () => import("./commands/sync.mjs"), skills: () => import("./commands/skills.mjs"), gap: () => import("./commands/registry.mjs"), @@ -26,10 +28,12 @@ ${color.bold("devia")} — one standard, one memory ${color.bold("devia doctor")} adoption, drift and staleness diagnosis ${color.bold("devia rules")} list or show rules from the registry ${color.bold("devia read")} render the memory as one self-contained page + ${color.bold("devia context")} the smallest sufficient context for one task ${color.bold("devia sync")} pin the standard under .devia/standard/, or refresh it ${color.bold("devia skills")} install the agent adapters (install --agent all) ${color.bold("devia gap")} add or close a line in 11_GAPS.md ${color.bold("devia debt")} add or close a line in 12_DEBT.md + ${color.bold("devia contribute")} turn a devia problem you hit here into an issue or a PR Common flags diff --git a/src/commands/check.mjs b/src/commands/check.mjs index e9b5277..b562759 100644 --- a/src/commands/check.mjs +++ b/src/commands/check.mjs @@ -1,6 +1,9 @@ import path from "node:path"; import { exists, read, readJSON, walk } from "../lib/fs.mjs"; import { trackedFiles } from "../lib/git.mjs"; +import { SECRET_PATTERNS } from "../lib/sanitize.mjs"; +import { GATES } from "../lib/gates.mjs"; +import { buildCorpus, classify, select, budgetFor } from "../lib/context.mjs"; import { color, heading, status, line } from "../lib/ui.mjs"; /** @@ -25,16 +28,6 @@ const SKIP_DIRS = new Set([ ".devia", ]); -const SECRET_PATTERNS = [ - [/AKIA[0-9A-Z]{16}/, "AWS access key id"], - [/-----BEGIN (RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/, "private key"], - [/sk_live_[0-9a-zA-Z]{16,}/, "live secret key"], - [/gh[pousr]_[0-9A-Za-z]{30,}/, "GitHub token"], - [/xox[baprs]-[0-9A-Za-z-]{10,}/, "Slack token"], - [/AIza[0-9A-Za-z_-]{35}/, "Google API key"], - [/eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/, "JWT"], -]; - /** * A line carrying this marker is exempt from the secret and bypass scanners. It exists so an * exemption is visible in the diff and reviewable — never as a silent allowlist elsewhere. @@ -121,346 +114,267 @@ function makeChecks(root, ctx) { return hits; }; - return [ - { - id: "MEM-PRESENT", - priority: "P0", - rule: "AGT-002", - title: "Project memory exists", - run: () => - exists(path.join(root, ".devia")) - ? { kind: "PASS" } - : { kind: "FAIL", detail: "run `devia init` before working here" }, - }, - { - id: "MEM-FILLED", - priority: "P1", - rule: "MEM-009", - title: "Overview is filled in", - run: () => { - const text = read(path.join(root, ".devia", "00_OVERVIEW.md")); - if (text === null) return { kind: "SKIP", detail: "no .devia/00_OVERVIEW.md" }; - const n = (text.match(/TODO\(devia\)/g) || []).length; - return n - ? { kind: "WARN", detail: `${n} placeholders left` } - : { kind: "PASS" }; - }, + const runners = { + "MEM-PRESENT": () => + exists(path.join(root, ".devia")) + ? { kind: "PASS" } + : { kind: "FAIL", detail: "run `devia init` before working here" }, + "MEM-FILLED": () => { + const text = read(path.join(root, ".devia", "00_OVERVIEW.md")); + if (text === null) return { kind: "SKIP", detail: "no .devia/00_OVERVIEW.md" }; + const n = (text.match(/TODO\(devia\)/g) || []).length; + return n + ? { kind: "WARN", detail: `${n} placeholders left` } + : { kind: "PASS" }; }, - { - id: "MEM-DEBT-P0", - priority: "P0", - rule: "MEM-002", - title: "No P0 debt recorded as unbuilt", - run: () => { - const text = read(path.join(root, ".devia", "12_DEBT.md")); - if (text === null) return { kind: "SKIP", detail: "no debt registry" }; - // The priority is a cell, not a word somewhere in the row. Matching the whole line made - // a P1 line reading "becomes P0 once the payment module ships" fail the gate — a P0 - // blocker invented out of prose, on a project that had none. - const rows = (text.match(/^\|\s*D\d+\s*\|.*$/gm) || []).filter((row) => { - if (/TODO\(devia\)/.test(row)) return false; - return row - .split("|") - .slice(1, -1) - .some((cell) => cell.trim().toUpperCase() === "P0"); - }); - return rows.length - ? { kind: "FAIL", detail: `${rows.length} P0 debt line(s) open` } - : { kind: "PASS" }; - }, + "MEM-DEBT-P0": () => { + const text = read(path.join(root, ".devia", "12_DEBT.md")); + if (text === null) return { kind: "SKIP", detail: "no debt registry" }; + // The priority is a cell, not a word somewhere in the row. Matching the whole line made + // a P1 line reading "becomes P0 once the payment module ships" fail the gate — a P0 + // blocker invented out of prose, on a project that had none. + const rows = (text.match(/^\|\s*D\d+\s*\|.*$/gm) || []).filter((row) => { + if (/TODO\(devia\)/.test(row)) return false; + return row + .split("|") + .slice(1, -1) + .some((cell) => cell.trim().toUpperCase() === "P0"); + }); + return rows.length + ? { kind: "FAIL", detail: `${rows.length} P0 debt line(s) open` } + : { kind: "PASS" }; }, - { - id: "MEM-WAIVERS", - priority: "P1", - rule: "GOVERNANCE", - title: "No expired waiver", - run: () => { - const waivers = config?.waivers || []; - if (!waivers.length) return { kind: "PASS", detail: "none" }; - const today = new Date().toISOString().slice(0, 10); - const expired = waivers.filter((w) => !w.expires || w.expires < today); - return expired.length - ? { kind: "FAIL", detail: `${expired.length} expired or undated` } - : { kind: "PASS", detail: `${waivers.length} active` }; - }, + "MEM-WAIVERS": () => { + const waivers = config?.waivers || []; + if (!waivers.length) return { kind: "PASS", detail: "none" }; + const today = new Date().toISOString().slice(0, 10); + const expired = waivers.filter((w) => !w.expires || w.expires < today); + return expired.length + ? { kind: "FAIL", detail: `${expired.length} expired or undated` } + : { kind: "PASS", detail: `${waivers.length} active` }; }, - { - id: "CI-PRESENT", - priority: "P0", - rule: "OPS-001", - title: "CI runs on pull requests", - run: () => { - const wf = path.join(root, ".github", "workflows"); - const found = - (exists(wf) && walk(wf, { filter: (f) => /\.ya?ml$/.test(f) }).length) || - anyExists(root, [".gitlab-ci.yml", "azure-pipelines.yml", "Jenkinsfile", ".circleci"]); - return found - ? { kind: "PASS" } - : { kind: "FAIL", detail: "no CI configuration found" }; - }, + "CI-PRESENT": () => { + const wf = path.join(root, ".github", "workflows"); + const found = + (exists(wf) && walk(wf, { filter: (f) => /\.ya?ml$/.test(f) }).length) || + anyExists(root, [".gitlab-ci.yml", "azure-pipelines.yml", "Jenkinsfile", ".circleci"]); + return found + ? { kind: "PASS" } + : { kind: "FAIL", detail: "no CI configuration found" }; }, - { - id: "CI-GATES", - priority: "P1", - rule: "OPS-001", - title: "CI runs tests and static checks", - run: () => { - const wf = path.join(root, ".github", "workflows"); - if (!exists(wf)) return { kind: "SKIP", detail: "no GitHub workflows to read" }; - const text = walk(wf, { filter: (f) => /\.ya?ml$/.test(f) }) - .map((f) => read(path.join(wf, f)) || "") - .join("\n"); - const has = (re) => re.test(text); - const missing = []; - if (!has(/\btest\b/i)) missing.push("test"); - if (!has(/\blint\b|eslint|ruff|flake8/i)) missing.push("lint"); - if (!has(/audit|snyk|dependabot|osv/i)) missing.push("dependency audit"); - return missing.length - ? { kind: "WARN", detail: `not referenced: ${missing.join(", ")}` } - : { kind: "PASS" }; - }, + "CI-GATES": () => { + const wf = path.join(root, ".github", "workflows"); + if (!exists(wf)) return { kind: "SKIP", detail: "no GitHub workflows to read" }; + const text = walk(wf, { filter: (f) => /\.ya?ml$/.test(f) }) + .map((f) => read(path.join(wf, f)) || "") + .join("\n"); + const has = (re) => re.test(text); + const missing = []; + if (!has(/\btest\b/i)) missing.push("test"); + if (!has(/\blint\b|eslint|ruff|flake8/i)) missing.push("lint"); + if (!has(/audit|snyk|dependabot|osv/i)) missing.push("dependency audit"); + return missing.length + ? { kind: "WARN", detail: `not referenced: ${missing.join(", ")}` } + : { kind: "PASS" }; }, - { - id: "SEC-ENV", - priority: "P0", - rule: "SEC-002", - title: "No environment file committed", - run: () => { - const ignore = read(path.join(root, ".gitignore")) || ""; - const envs = [".env", ".env.local", ".env.production"].filter((f) => - exists(path.join(root, f)) - ); - if (!envs.length) return { kind: "PASS" }; - const ignored = /^\s*\.env/m.test(ignore); - return ignored - ? { kind: "WARN", detail: `${envs.join(", ")} present locally but gitignored` } - : { kind: "FAIL", detail: `${envs.join(", ")} is not gitignored` }; - }, + "SEC-ENV": () => { + const ignore = read(path.join(root, ".gitignore")) || ""; + const envs = [".env", ".env.local", ".env.production"].filter((f) => + exists(path.join(root, f)) + ); + if (!envs.length) return { kind: "PASS" }; + const ignored = /^\s*\.env/m.test(ignore); + return ignored + ? { kind: "WARN", detail: `${envs.join(", ")} present locally but gitignored` } + : { kind: "FAIL", detail: `${envs.join(", ")} is not gitignored` }; }, - { - id: "SEC-SECRETS", - priority: "P0", - rule: "SEC-002", - title: "No secret-shaped strings in the tree", - run: () => { - const hits = []; - let exempted = 0; - for (const f of files) { - if (/(^|\/)(\.env\.example|.*\.lock|package-lock\.json)$/.test(rel(f))) continue; - const text = read(path.join(root, f)); - if (!text) continue; - const lines = text.split(/\r?\n/); - for (let i = 0; i < lines.length; i++) { - const hit = SECRET_PATTERNS.find(([re]) => re.test(lines[i])); - if (!hit) continue; - if (lines[i].includes(ALLOW)) { - exempted++; - continue; - } - hits.push(`${rel(f)}:${i + 1} (${hit[1]})`); - break; + "SEC-SECRETS": () => { + const hits = []; + let exempted = 0; + for (const f of files) { + if (/(^|\/)(\.env\.example|.*\.lock|package-lock\.json)$/.test(rel(f))) continue; + const text = read(path.join(root, f)); + if (!text) continue; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const hit = SECRET_PATTERNS.find(([re]) => re.test(lines[i])); + if (!hit) continue; + if (lines[i].includes(ALLOW)) { + exempted++; + continue; } - if (hits.length >= 5) break; + hits.push(`${rel(f)}:${i + 1} (${hit[1]})`); + break; } - return hits.length - ? { kind: "FAIL", detail: hits.join("; ") } - : { - kind: "PASS", - detail: `${files.length} files scanned${exempted ? `, ${exempted} exempted` : ""}`, - }; - }, + if (hits.length >= 5) break; + } + return hits.length + ? { kind: "FAIL", detail: hits.join("; ") } + : { + kind: "PASS", + detail: `${files.length} files scanned${exempted ? `, ${exempted} exempted` : ""}`, + }; }, - { - id: "OPS-BYPASS", - priority: "P0", - rule: "OPS-003", - title: "No check bypass wired into the repository", - run: () => { - // Only scripts, hooks and pipeline configuration can wire a bypass in. Prose that - // forbids `--no-verify` is not a bypass, so documentation is out of scope here. - const hits = []; - for (const f of files) { - if (!EXECUTABLE_SURFACE.test(rel(f))) continue; - const text = read(path.join(root, f)); - if (!text) continue; - const lines = text.split(/\r?\n/); - for (let i = 0; i < lines.length; i++) { - if (!/--no-verify|SKIP_HOOKS=1|HUSKY=0|\[skip ci\]/.test(lines[i])) continue; - if (lines[i].includes(ALLOW)) continue; - hits.push(`${rel(f)}:${i + 1}`); - break; - } - if (hits.length >= 5) break; + "OPS-BYPASS": () => { + // Only scripts, hooks and pipeline configuration can wire a bypass in. Prose that + // forbids `--no-verify` is not a bypass, so documentation is out of scope here. + const hits = []; + for (const f of files) { + if (!EXECUTABLE_SURFACE.test(rel(f))) continue; + const text = read(path.join(root, f)); + if (!text) continue; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + if (!/--no-verify|SKIP_HOOKS=1|HUSKY=0|\[skip ci\]/.test(lines[i])) continue; + if (lines[i].includes(ALLOW)) continue; + hits.push(`${rel(f)}:${i + 1}`); + break; } - return hits.length ? { kind: "FAIL", detail: hits.join(", ") } : { kind: "PASS" }; - }, - }, - { - id: "TST-PRESENT", - priority: "P0", - rule: "TST-001", - title: "Automated tests exist", - run: () => { - const testFiles = files.filter((f) => - /(^|\/)(tests?|spec|__tests__)\//.test(rel(f)) || - /\.(test|spec)\.[a-z]+$/.test(rel(f)) || - /(^|\/)test_[^/]+\.py$/.test(rel(f)) - ); - return testFiles.length - ? { kind: "PASS", detail: `${testFiles.length} test files` } - : { kind: "FAIL", detail: "no test files found" }; - }, + if (hits.length >= 5) break; + } + return hits.length ? { kind: "FAIL", detail: hits.join(", ") } : { kind: "PASS" }; }, - { - id: "TST-SKIPPED", - priority: "P1", - rule: "TST-003", - title: "No disabled tests", - run: () => { - const hits = grepFiles( - /\b(it|test|describe)\.skip\(|\bxit\(|\bxdescribe\(|@pytest\.mark\.skip|t\.Skip\(/, - 20 - ); - return hits.length - ? { kind: "WARN", detail: `${hits.length} file(s): ${hits.slice(0, 3).join(", ")}` } - : { kind: "PASS" }; - }, + "TST-PRESENT": () => { + const testFiles = files.filter((f) => + /(^|\/)(tests?|spec|__tests__)\//.test(rel(f)) || + /\.(test|spec)\.[a-z]+$/.test(rel(f)) || + /(^|\/)test_[^/]+\.py$/.test(rel(f)) + ); + return testFiles.length + ? { kind: "PASS", detail: `${testFiles.length} test files` } + : { kind: "FAIL", detail: "no test files found" }; }, - { - id: "TST-SCRIPT", - priority: "P1", - rule: "TST-001", - title: "A test command exists", - run: () => { - if (!pkg) return { kind: "SKIP", detail: noManifest }; - const t = scripts.test; - const where = pkgWhere === "package.json" ? "" : ` in ${pkgWhere}`; - if (!t) return { kind: "WARN", detail: `no npm test script${where}` }; - return /no test specified/.test(t) - ? { kind: "FAIL", detail: `test script is the npm placeholder${where}` } - : { kind: "PASS", detail: where.trim() }; - }, + "TST-SKIPPED": () => { + const hits = grepFiles( + /\b(it|test|describe)\.skip\(|\bxit\(|\bxdescribe\(|@pytest\.mark\.skip|t\.Skip\(/, + 20 + ); + return hits.length + ? { kind: "WARN", detail: `${hits.length} file(s): ${hits.slice(0, 3).join(", ")}` } + : { kind: "PASS" }; }, - { - id: "OPS-LOCKFILE", - priority: "P1", - rule: "OPS-004", - title: "Dependency lockfile committed", - run: () => { - const LOCKS = [ - "package-lock.json", - "pnpm-lock.yaml", - "yarn.lock", - "poetry.lock", - "uv.lock", - "requirements.txt", - "go.sum", - "Cargo.lock", - "Gemfile.lock", - ]; - // A lockfile sits next to the manifest it locks, which is not always the root. - const dirs = new Set(["."]); - for (const m of manifests) dirs.add(path.dirname(m)); - const rootManifest = anyExists(root, [ - "package.json", - "pyproject.toml", - "go.mod", - "Cargo.toml", - "Gemfile", - ]); - if (!rootManifest && !manifests.length) return { kind: "SKIP", detail: "no manifest" }; - for (const dir of dirs) { - const found = LOCKS.find((l) => exists(path.join(root, dir, l))); - if (found) return { kind: "PASS", detail: rel(path.join(dir, found)) }; - } - return { kind: "FAIL", detail: "no lockfile" }; - }, + "TST-SCRIPT": () => { + if (!pkg) return { kind: "SKIP", detail: noManifest }; + const t = scripts.test; + const where = pkgWhere === "package.json" ? "" : ` in ${pkgWhere}`; + if (!t) return { kind: "WARN", detail: `no npm test script${where}` }; + return /no test specified/.test(t) + ? { kind: "FAIL", detail: `test script is the npm placeholder${where}` } + : { kind: "PASS", detail: where.trim() }; }, - { - id: "DB-MIGRATIONS", - priority: "P1", - rule: "DB-001", - title: "Schema changes are versioned migrations", - run: () => { - const usesDb = - hasDep("pg", "mysql2", "prisma", "@prisma/client", "typeorm", "sequelize", "knex", - "drizzle-orm", "mongoose", "sqlalchemy") || - files.some((f) => /\.sql$/.test(rel(f))); - if (!usesDb) return { kind: "SKIP", detail: "no database detected" }; - const dir = anyExists(root, [ - "migrations", - "db/migrate", - "prisma/migrations", - "alembic", - path.join("src", "migrations"), - ]); - if (dir) return { kind: "PASS", detail: dir }; - // Not only at the root: a migrations directory can live under any package. - const SEGMENTS = new Set(["migrations", "migrate", "alembic"]); - const nested = files - .map(rel) - .find((f) => f.split("/").slice(0, -1).some((seg) => SEGMENTS.has(seg))); - return nested - ? { kind: "PASS", detail: nested.split("/").slice(0, -1).join("/") } - : { kind: "FAIL", detail: "database in use, no migrations directory" }; - }, + "OPS-LOCKFILE": () => { + const LOCKS = [ + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "poetry.lock", + "uv.lock", + "requirements.txt", + "go.sum", + "Cargo.lock", + "Gemfile.lock", + ]; + // A lockfile sits next to the manifest it locks, which is not always the root. + const dirs = new Set(["."]); + for (const m of manifests) dirs.add(path.dirname(m)); + const rootManifest = anyExists(root, [ + "package.json", + "pyproject.toml", + "go.mod", + "Cargo.toml", + "Gemfile", + ]); + if (!rootManifest && !manifests.length) return { kind: "SKIP", detail: "no manifest" }; + for (const dir of dirs) { + const found = LOCKS.find((l) => exists(path.join(root, dir, l))); + if (found) return { kind: "PASS", detail: rel(path.join(dir, found)) }; + } + return { kind: "FAIL", detail: "no lockfile" }; }, - { - id: "AGT-CONTRACT", - priority: "P1", - rule: "AGT-001", - title: "Agent contract at the repository root", - run: () => - exists(path.join(root, "AGENTS.md")) - ? { kind: "PASS" } - : { kind: "WARN", detail: "run `devia skills install`" }, + "DB-MIGRATIONS": () => { + const usesDb = + hasDep("pg", "mysql2", "prisma", "@prisma/client", "typeorm", "sequelize", "knex", + "drizzle-orm", "mongoose", "sqlalchemy") || + files.some((f) => /\.sql$/.test(rel(f))); + if (!usesDb) return { kind: "SKIP", detail: "no database detected" }; + const dir = anyExists(root, [ + "migrations", + "db/migrate", + "prisma/migrations", + "alembic", + path.join("src", "migrations"), + ]); + if (dir) return { kind: "PASS", detail: dir }; + // Not only at the root: a migrations directory can live under any package. + const SEGMENTS = new Set(["migrations", "migrate", "alembic"]); + const nested = files + .map(rel) + .find((f) => f.split("/").slice(0, -1).some((seg) => SEGMENTS.has(seg))); + return nested + ? { kind: "PASS", detail: nested.split("/").slice(0, -1).join("/") } + : { kind: "FAIL", detail: "database in use, no migrations directory" }; }, - { - id: "DOC-README", - priority: "P2", - rule: "—", - title: "README present", - run: () => - anyExists(root, ["README.md", "README.rst", "readme.md"]) - ? { kind: "PASS" } - : { kind: "WARN", detail: "no README" }, + "AGT-CONTRACT": () => + exists(path.join(root, "AGENTS.md")) + ? { kind: "PASS" } + : { kind: "WARN", detail: "run `devia skills install`" }, + /** + * A target nobody revisits quietly becomes a permanent overrun. This compares the declared + * target with the floor for a task that routes nothing — the constraints that apply to + * every change. If those alone do not fit, no task will ever fit. + */ + "CTX-BUDGET": () => { + if (!exists(path.join(root, ".devia"))) { + return { kind: "SKIP", detail: "no memory to size a context against" }; + } + const deviaDir = path.join(root, ".devia"); + const { target, mode } = budgetFor(deviaDir, {}); + const corpus = buildCorpus({ root, deviaDir }); + classify(corpus, { task: "", files: [] }); + const { floor, status } = select(corpus, { target, mode }); + + if (mode === "strict") { + return status === "impossible" + ? { kind: "FAIL", detail: `strict target ${target} cannot hold ${floor} tokens even compressed` } + : { kind: "PASS", detail: `strict: compresses to fit ${target}` }; + } + return floor > target + ? { + kind: "WARN", + detail: `every task starts at ${floor} tokens, above the ${target} target — raise it, prune 10_NEVER_ALWAYS.md, or set strict mode`, + } + : { kind: "PASS", detail: `baseline floor ${floor} of ${target}` }; }, - { - id: "UI-A11Y-TOOLING", - priority: "P2", - rule: "A11Y-001", - title: "Accessibility tooling available", - run: () => { - if (config && config.modules && config.modules.design === false) - return { kind: "SKIP", detail: "design module disabled" }; - if (!pkg) return { kind: "SKIP", detail: noManifest }; - const found = hasDep( - "axe-core", "@axe-core/react", "@axe-core/playwright", "jest-axe", - "eslint-plugin-jsx-a11y", "pa11y", "@storybook/addon-a11y", "lighthouse" - ); - return found - ? { kind: "PASS" } - : { kind: "WARN", detail: "no automated accessibility check configured" }; - }, + "DOC-README": () => + anyExists(root, ["README.md", "README.rst", "readme.md"]) + ? { kind: "PASS" } + : { kind: "WARN", detail: "no README" }, + "UI-A11Y-TOOLING": () => { + if (config && config.modules && config.modules.design === false) + return { kind: "SKIP", detail: "design module disabled" }; + if (!pkg) return { kind: "SKIP", detail: noManifest }; + const found = hasDep( + "axe-core", "@axe-core/react", "@axe-core/playwright", "jest-axe", + "eslint-plugin-jsx-a11y", "pa11y", "@storybook/addon-a11y", "lighthouse" + ); + return found + ? { kind: "PASS" } + : { kind: "WARN", detail: "no automated accessibility check configured" }; }, - { - id: "OBS-ERRORS", - priority: "P2", - rule: "OBS-002", - title: "Errors reach something a human watches", - run: () => { - const profile = config?.project?.profile; - if (["cli", "library", "docs"].includes(profile)) - return { kind: "SKIP", detail: `${profile} does not run as a watched service` }; - if (!pkg) return { kind: "SKIP", detail: noManifest }; - const found = hasDep("@sentry/node", "@sentry/browser", "@sentry/nextjs", "bugsnag", - "rollbar", "datadog-lambda-js", "dd-trace", "@opentelemetry/api"); - return found - ? { kind: "PASS" } - : { kind: "WARN", detail: "no error tracker detected — verify manually" }; - }, + "OBS-ERRORS": () => { + const profile = config?.project?.profile; + if (["cli", "library", "docs"].includes(profile)) + return { kind: "SKIP", detail: `${profile} does not run as a watched service` }; + if (!pkg) return { kind: "SKIP", detail: noManifest }; + const found = hasDep("@sentry/node", "@sentry/browser", "@sentry/nextjs", "bugsnag", + "rollbar", "datadog-lambda-js", "dd-trace", "@opentelemetry/api"); + return found + ? { kind: "PASS" } + : { kind: "WARN", detail: "no error tracker detected — verify manually" }; }, - ]; + }; + + return GATES.map((g) => ({ ...g, run: runners[g.id] })); } export default async function check(ctx) { diff --git a/src/commands/context.mjs b/src/commands/context.mjs new file mode 100644 index 0000000..9587769 --- /dev/null +++ b/src/commands/context.mjs @@ -0,0 +1,273 @@ +import process from "node:process"; +import { exists } from "../lib/fs.mjs"; +import { git } from "../lib/git.mjs"; +import { reduction } from "../lib/tokens.mjs"; +import { + buildCorpus, + classify, + select, + render, + budgetFor, + DEFAULT_BUDGET, +} from "../lib/context.mjs"; +import { color, heading, status, line } from "../lib/ui.mjs"; + +/** + * The smallest sufficient slice of devia for one task. + * + * The default output is the context itself, on stdout, so it can be piped straight into an + * agent. `--stats` and `--explain` are the other two questions a reader has — how much did this + * cost, and why is this item here — and each gets its own mode rather than being mixed into the + * payload. + */ + +function list(flag) { + if (!flag || flag === true) return []; + return String(flag) + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +} + +/** Files this change touches, from `--files`, and from git when `--diff` is given. */ +function changedFiles(root, flags) { + const named = list(flags.files); + if (!flags.diff) return named; + + const base = flags.diff === true ? "HEAD" : String(flags.diff); + const tracked = git(root, ["diff", "--name-only", base]); + const staged = git(root, ["diff", "--name-only", "--cached"]); + const untracked = git(root, ["ls-files", "--others", "--exclude-standard"]); + const all = [tracked, staged, untracked] + .filter((s) => s !== null) + .flatMap((s) => s.split("\n")) + .filter(Boolean); + return [...new Set([...named, ...all])]; +} + +/** + * The accounting, with the three numbers kept apart. + * + * `target` is what was asked for, `mandatory_floor` is what the blocking items cost, and + * `selected_tokens` is what was actually produced. Reporting only the first and the last made a + * stated design ("a mandatory item is never evicted") read as a broken promise. + */ +function metrics(selection) { + return { + mode: selection.mode, + target: selection.target, + mandatory_floor: selection.floor, + mandatory_floor_full: selection.fullFloor, + mandatory_items: selection.mandatory, + selected_tokens: selection.spent, + status: selection.status, + compliant: selection.compliant, + degraded: selection.degraded, + raw_tokens: selection.raw, + reduction_pct: reduction(selection.raw, selection.spent), + included_items: selection.included.length, + excluded_items: selection.excluded.length, + overrun: selection.overrun, + }; +} + +export default async function context(ctx) { + const { root, deviaDir, args, flags } = ctx; + + if (flags.help) { + line(` +${color.bold("devia context")} — the smallest sufficient context for one task + + devia context "add POST /api/orders" + devia context "refund flow" --files src/api/refund.ts,migrations/004.sql + devia context "fix the empty state" --diff --explain + devia context --stats --json + + --budget token budget for this run (default: devia.json context.maxTokens, else ${DEFAULT_BUDGET}) + --files comma-separated paths this change touches + --diff [ref] add the files git reports as changed (default ref: HEAD) + --domain force these rule domains in + --explain why each item was included, and why the rest was not + --stats the token accounting only + --full the unoptimised corpus, for measuring against + --json machine-readable + +The context itself goes to stdout, so it pipes straight into an agent. --explain and --stats +are separate modes rather than commentary mixed into that payload. + +Blocking constraints are admitted before the budget is consulted and are never evicted: +a budget too small to hold them reports an overrun instead of dropping one. +`.trim()); + return 0; + } + + if (!exists(deviaDir)) { + if (ctx.json) { + console.log(JSON.stringify({ ok: false, error: "no .devia directory", root }, null, 2)); + } else { + heading("devia context"); + status("FAIL", "no .devia/ in this repository", "run `devia init` first (AGT-002)"); + line(""); + } + return 1; + } + + const task = args._.slice(1).join(" ").trim(); + const files = changedFiles(root, flags); + const { target, mode } = budgetFor(deviaDir, flags); + + const corpus = buildCorpus({ root, deviaDir }); + const { domains, changeTypes } = classify(corpus, { + task, + files, + domains: list(flags.domain), + changeTypes: list(flags.type), + }); + const selection = select(corpus, { target, mode }); + + if (flags.full) { + // The baseline: everything devia knows, rules at full text. What an agent consumes when + // nothing routes for it. + const text = corpus.map((i) => i.full || i.text).join("\n"); + if (ctx.json) { + console.log(JSON.stringify({ items: corpus.length, tokens: selection.raw, text }, null, 2)); + } else { + console.log(text); + } + return 0; + } + + if (ctx.json) { + console.log( + JSON.stringify( + { + ok: true, + task, + files, + domains: [...domains].map(([domain, why]) => ({ domain, why: [...why] })), + change_types: changeTypes, + context: metrics(selection), + included: selection.included.map((i) => ({ + kind: i.kind, + id: i.id, + tier: i.tier, + tokens: i.tokens, + relevance: i.relevance, + why: i.why, + })), + excluded: selection.excluded.map((i) => ({ + kind: i.kind, + id: i.id, + tier: i.tier, + tokens: i.tokens, + reason: i.reason, + why: i.why, + })), + }, + null, + 2 + ) + ); + return 0; + } + + if (flags.explain) { + const m = metrics(selection); + heading(`devia context — ${task || "no task given"}`); + line(` ${color.dim("domains")} ${[...domains.keys()].join(", ") || "none routed"}`); + if (changeTypes.length) line(` ${color.dim("change")} ${changeTypes.join(", ")}`); + line(""); + + line(color.bold(" Included")); + for (const i of selection.included) { + line(` ${color.green(i.tier)} ${String(i.tokens).padStart(4)}t ${i.id}`); + for (const w of i.why.slice(0, 3)) line(color.dim(` ${w}`)); + } + + const withheld = selection.excluded.filter((i) => i.reason === "budget"); + if (withheld.length) { + line(""); + line(color.bold(" Withheld by the budget")); + for (const i of withheld.slice(0, 12)) { + line(` ${color.yellow(i.tier)} ${String(i.tokens).padStart(4)}t ${i.id}`); + } + if (withheld.length > 12) line(color.dim(` …and ${withheld.length - 12} more`)); + } + + const irrelevant = selection.excluded.filter((i) => i.reason === "not relevant"); + line(""); + line(color.bold(" Not relevant")); + line(color.dim(` ${irrelevant.length} items, e.g.`)); + for (const i of irrelevant.slice(0, 6)) { + line(color.dim(` ${i.id} — ${i.why[0] || "no signal"}`)); + } + + line(""); + report(m); + return 0; + } + + if (flags.stats) { + heading(`devia context — ${task || "no task given"}`); + report(metrics(selection)); + return selection.status === "impossible" ? 1 : 0; + } + + // Strict means strict. A context that cannot hold its own mandatory items is not a smaller + // context, it is a wrong one, so nothing is written to stdout and the reason goes to stderr. + if (selection.status === "impossible") { + console.error( + `devia context: the ${selection.mandatory} mandatory items cost ${selection.floor} tokens ` + + `at their smallest, above the strict target of ${selection.target}. Nothing was produced.` + ); + return 1; + } + + process.stdout.write(render(selection, { task })); + return 0; +} + +const STATUS_LABEL = { + within: (c) => c.green("WITHIN TARGET"), + degraded: (c) => c.yellow("WITHIN TARGET, DEGRADED"), + over: (c) => c.yellow("OVER TARGET"), + impossible: (c) => c.red("IMPOSSIBLE"), +}; + +function report(m) { + const pad = (n) => String(n).padStart(7); + line(` Target ${pad(m.target)} tokens (${m.mode})`); + line(` Mandatory floor ${pad(m.mandatory_floor)} tokens in ${m.mandatory_items} items`); + line(` Selected ${pad(m.selected_tokens)} tokens in ${m.included_items} items`); + line(` Raw corpus ${pad(m.raw_tokens)} tokens (estimated)`); + line(` Reduction ${pad(m.reduction_pct)} %`); + line(""); + line(` Status ${STATUS_LABEL[m.status](color)}`); + + if (m.status === "over") { + line( + color.dim( + ` The ${m.mandatory_items} mandatory items cost ${m.mandatory_floor} tokens, above the ` + + `${m.target} asked for.` + ) + ); + line(color.dim(" They are included anyway: in advisory mode a target never evicts one.")); + line(color.dim(" Set a larger target, prune 10_NEVER_ALWAYS.md, or use --strict.")); + } else if (m.status === "degraded") { + line(color.dim(` The full floor was ${m.mandatory_floor_full} tokens. To fit the target:`)); + for (const d of m.degraded) line(color.dim(` · ${d.note} (${d.items})`)); + } else if (m.status === "impossible") { + line( + color.dim( + ` Even at their smallest the mandatory items cost ${m.mandatory_floor} tokens. ` + + "Nothing was produced: strict means strict." + ) + ); + } else { + line(color.dim(` ${m.mandatory_items} mandatory items, ${m.mandatory_floor} tokens of it.`)); + } + line(color.dim(" Token counts are estimates, not a tokenizer's output.")); + line(""); +} + +export { metrics }; diff --git a/src/commands/contribute.mjs b/src/commands/contribute.mjs new file mode 100644 index 0000000..5ec5622 --- /dev/null +++ b/src/commands/contribute.mjs @@ -0,0 +1,783 @@ +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { execFileSync } from "node:child_process"; +import { exists, read, writeFile, packageRoot } from "../lib/fs.mjs"; +import { sanitize } from "../lib/sanitize.mjs"; +import { + TYPES, + REPRO_CAPS, + recordPath, + reproPath, + settings, + listRecords, + nextId, + claimHash, + stateOf, + missing, + eligibility, + runObservation, + verdict, + reproStats, + environment, + fixtureHash, + sourceHash, + evidenceChain, + renderBody, + renderTitle, + identityCheck, + manifest, + payloadResidue, + redactions, + save, + load, + upstream, +} from "../lib/contribution.mjs"; +import { color, heading, status, line } from "../lib/ui.mjs"; + +/** + * A devia problem hit in a real repository, turned into something devia can act on. + * + * Everything here is local except one path: `submit --yes`, which asks GitHub to open an issue + * or a pull request through the `gh` CLI under an identity the project declared. devia holds no + * token, performs no background upload, and prints every byte it would send before sending it + * (`PRIV-005`). + */ + +const HELP = ` +${color.bold("devia contribute")} — a devia problem you hit here, as an issue or a pull request + + devia contribute candidates, their state, and what each needs next + devia contribute new --type ... record what you observed + devia contribute repro scaffold the minimal reproduction + devia contribute verify re-run the observation inside it — this is the gate + devia contribute show the full report, and exactly what would be sent + devia contribute submit prepare the issue or pull request + devia contribute rm drop a candidate + +${color.bold("new")} + --type ${TYPES.join("|")} + --summary "..." --expected "..." --actual "..." + --argv "check --json" the invocation, as you ran it + --actual-exit --actual-matches --actual-absent + --expect-exit --expect-matches --expect-absent + --expect-metric --expect-op --expect-value for a numeric claim + --actual-metric --actual-op --actual-value + --gate --rule --component + --manual a deliberate human proposal, not an observation. Issue only, never a PR + +${color.bold("repro")} + --include comma-separated files to copy in, sanitized and listed in the record + --with-memory also run \`devia init\` inside the fixture + --force rebuild an existing fixture + +${color.bold("verify")} + --devia a devia checkout to test against (default: the installed one) + +${color.bold("submit")} + --yes authorise the remote operation. Without it, nothing leaves this machine + --fix-repo the devia checkout holding the fix + --fix-tests regression tests, comma-separated, relative to the checkout + --architectural record that this needs agreement before a patch + +The two assertions must tell the two behaviours apart, or neither proves anything. A gate id +appears in --json whether the gate passed or failed, so --actual-matches SEC-SECRETS holds on a +clean repository too. Assert on something that actually differs — the blocking list, an exit +code, or a metric: + + --actual-matches '"blocking": \\[[^\\]]*"SEC-SECRETS"' + --actual-metric context.selected_tokens --actual-op ">" --actual-value 3000 + +A contribution is eligible only when devia itself reproduced the problem. Turn the whole +feature off with "contribution": { "enabled": false } in .devia/devia.json. +`; + +const NEXT = { + incomplete: "devia contribute rm and record it again — see the missing fields above", + observed: "devia contribute repro , then verify ", + reproduced: "fix it, then verify --devia , or submit as an issue", + fixed: "devia contribute submit ", + rejected: "the behaviour did not reappear — correct the claim or drop the candidate", +}; + +function csv(flag) { + if (!flag || flag === true) return []; + return String(flag).split(",").map((s) => s.trim()).filter(Boolean); +} + +function profile(flags, prefix) { + const p = {}; + const num = (v) => (v === undefined ? undefined : Number(v)); + if (flags[`${prefix}-exit`] !== undefined) p.exit = num(flags[`${prefix}-exit`]); + if (flags[`${prefix}-matches`]) p.matches = String(flags[`${prefix}-matches`]); + if (flags[`${prefix}-absent`]) p.absent = String(flags[`${prefix}-absent`]); + if (flags[`${prefix}-metric`]) { + p.metric = String(flags[`${prefix}-metric`]); + p.op = String(flags[`${prefix}-op`] || "=="); + p.value = Number(flags[`${prefix}-value`]); + } + return p; +} + +function dirOf(deviaDir, id) { + return path.dirname(recordPath(deviaDir, id)); +} + +const reproDir = (deviaDir, record) => reproPath(deviaDir, record); + +/** `gh`, and the account it is authenticated as. Absence is reported, never worked around. */ +function ghAccount() { + try { + const out = execFileSync("gh", ["auth", "status"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + const m = out.match(/account\s+([A-Za-z0-9-]+)/i) || out.match(/as\s+([A-Za-z0-9-]+)\s/i); + return { ok: true, account: m ? m[1] : null, raw: out }; + } catch (e) { + const raw = String(e.stderr || e.stdout || e.message || ""); + return { ok: false, account: null, raw, missing: /ENOENT|not recognized|not found/i.test(raw) }; + } +} + +// --------------------------------------------------------------------------------------------- + +function cmdList(ctx) { + const { deviaDir } = ctx; + const records = listRecords(deviaDir); + + if (ctx.json) { + console.log( + JSON.stringify( + { + ok: true, + candidates: records.map((r) => ({ + id: r.id, + type: r.type, + source: r.source, + state: stateOf(r), + summary: r.problem?.summary || "", + })), + }, + null, + 2 + ) + ); + return 0; + } + + heading("devia contribute"); + if (!records.length) { + status("INFO", "no candidates", "record one with `devia contribute new --type `"); + line(""); + line(color.dim(" A candidate starts from something devia actually did here — not from an")); + line(color.dim(" improvement you can imagine for it (AGT-012).")); + line(""); + return 0; + } + for (const r of records) { + const state = stateOf(r); + const kind = state === "fixed" ? "PASS" : state === "rejected" || state === "incomplete" ? "FAIL" : "INFO"; + status(kind, `${r.id} ${state.padEnd(10)} ${r.type}`, r.problem?.summary || ""); + line(color.dim(` next: ${NEXT[state]}`)); + } + line(""); + return 0; +} + +function cmdNew(ctx) { + const { deviaDir, flags } = ctx; + const type = String(flags.type || ""); + if (!TYPES.includes(type)) { + status("FAIL", "--type is required", TYPES.join(" | ")); + return 2; + } + + const argv = flags.argv && flags.argv !== true ? String(flags.argv).trim().split(/\s+/) : []; + if (argv.includes("--root")) { + status("FAIL", "--argv may not carry --root", "the reproduction is the working directory"); + return 2; + } + + const manual = Boolean(flags.manual); + const env = environment(); + const record = { + id: nextId(deviaDir), + created: new Date().toISOString().slice(0, 10), + type, + source: manual ? "manual" : "real_usage", + devia: { + ...env, + component: flags.component ? String(flags.component) : argv[0] || null, + gate: flags.gate ? String(flags.gate) : null, + rule: flags.rule ? String(flags.rule) : null, + }, + problem: { + summary: String(flags.summary || ctx.args._.slice(2).join(" ") || "").trim(), + expected: String(flags.expected || "").trim(), + actual: String(flags.actual || "").trim(), + }, + observation: manual ? null : { argv, expected: profile(flags, "expect"), actual: profile(flags, "actual") }, + reproduction: null, + verification: null, + fix: null, + }; + + const gaps = missing(record); + save(deviaDir, record); + + heading(`devia contribute — ${record.id} recorded`); + status("PASS", record.id, `${record.type} · ${record.source}`); + if (gaps.length) { + status("WARN", "incomplete", `still missing: ${gaps.join(", ")}`); + line(color.dim(` Edit ${path.join(".devia", "contributions", record.id, "record.json")}.`)); + } else if (manual) { + line(color.dim(" A manual proposal is routed to an issue. It never becomes a pull request")); + line(color.dim(" on its own evidence (AGT-012).")); + } else { + line(color.dim(` Next: devia contribute repro ${record.id}`)); + } + line(""); + return 0; +} + +const SCAFFOLD_README = `# Minimal reproduction + +Whatever is in this directory is what a devia maintainer receives. It stands alone: it is not a +copy of the repository the problem was found in, and it must not become one. + +1. Add the smallest tree that makes the recorded invocation misbehave. +2. Run \`devia contribute verify \` — it runs that invocation here and compares. +3. Keep it under ${REPRO_CAPS.files} files. +`; + +function cmdRepro(ctx, id) { + const { root, deviaDir, flags } = ctx; + const record = load(deviaDir, id); + if (!record) { + status("FAIL", `no such candidate: ${id}`); + return 1; + } + + const dir = path.join(dirOf(deviaDir, record.id), "repro"); + if (exists(dir) && !flags.force) { + const stats = reproStats(dir); + heading(`devia contribute repro — ${record.id}`); + status("SKIP", "fixture already there", `${stats.files} files — --force rebuilds it`); + line(""); + return 0; + } + if (flags.force) fs.rmSync(dir, { recursive: true, force: true }); + + writeFile( + path.join(dir, "package.json"), + JSON.stringify({ name: "devia-repro", version: "0.0.0", private: true }, null, 2) + "\n" + ); + writeFile(path.join(dir, "README.md"), SCAFFOLD_README); + + heading(`devia contribute repro — ${record.id}`); + status("PASS", "fixture scaffolded", path.relative(root, dir).split(path.sep).join("/")); + + const included = []; + for (const rel of csv(flags.include)) { + const from = path.resolve(root, rel); + if (!exists(from)) { + status("FAIL", `not found: ${rel}`); + return 1; + } + if (/(^|[\\/])\.env(\.|$)/.test(rel)) { + status("FAIL", `refused: ${rel}`, "an environment file is never copied into a fixture"); + return 1; + } + const raw = read(from); + if (raw === null) { + status("FAIL", `unreadable: ${rel}`); + return 1; + } + // A minimal case is minimal. A file bigger than the whole fixture's budget is not evidence, + // it is the repository arriving one path at a time. + if (raw.length > REPRO_CAPS.bytes) { + status( + "FAIL", + `too large: ${rel}`, + `${raw.length} bytes against a ${REPRO_CAPS.bytes}-byte cap — cut it down first` + ); + return 1; + } + const clean = sanitize(raw, { root }); + const target = path.join(dir, path.basename(rel)); + writeFile(target, clean.text); + included.push({ as: path.basename(rel), redactions: clean.removed }); + status( + "PASS", + `included ${path.basename(rel)}`, + clean.removed.length + ? `redacted ${clean.removed.map((r) => `${r.count} ${r.kind}`).join(", ")}` + : "nothing to redact" + ); + } + + if (flags["with-memory"]) { + // A separate process on purpose: a command never imports another command for its effects + // (01_ARCHITECTURE.md). The fixture gets a real memory, built by the real `init`. + try { + execFileSync( + process.execPath, + [path.join(packageRoot, "bin", "devia.mjs"), "init", "--root", dir, "--no-agents", "--yes"], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] } + ); + status("PASS", "memory initialised in the fixture"); + } catch (e) { + status("WARN", "could not initialise a memory in the fixture", String(e.message).slice(0, 120)); + } + } + + const stats = reproStats(dir); + record.reproduction = { + path: "repro", + built: new Date().toISOString().slice(0, 10), + included, + files: stats.files, + bytes: stats.bytes, + }; + save(deviaDir, record); + + if (stats.overCap) { + status("WARN", "over the minimal-case cap", `${stats.files} files, ${stats.bytes} bytes`); + } + line(""); + line(color.dim(` Make it fail, then: devia contribute verify ${record.id}`)); + line(""); + return 0; +} + +function cmdVerify(ctx, id) { + const { deviaDir, flags } = ctx; + const record = load(deviaDir, id); + if (!record) { + status("FAIL", `no such candidate: ${id}`); + return 1; + } + const gaps = missing(record); + if (gaps.length) { + status("FAIL", `${record.id} is incomplete`, `missing ${gaps.join(", ")}`); + return 1; + } + if (record.source === "manual") { + status("SKIP", `${record.id} is a manual proposal`, "there is no observation to re-run"); + return 0; + } + + const dir = reproDir(deviaDir, record); + if (!dir || !exists(dir)) { + status("FAIL", "no reproduction", `run \`devia contribute repro ${record.id}\` first`); + return 1; + } + + const checkout = flags.devia ? path.resolve(String(flags.devia)) : packageRoot; + const bin = path.join(checkout, "bin", "devia.mjs"); + if (!exists(bin)) { + status("FAIL", "not a devia checkout", bin); + return 1; + } + + const run = runObservation(record, { cwd: dir, deviaBin: bin }); + const result = verdict(record, run); + + const claim = claimHash(record); + // The two digests are what bind the verdict to the experiment: which fixture ran, and which + // devia ran it. A `fixed` record has to agree about the first and disagree about the second. + const evidence = { + ran: new Date().toISOString().slice(0, 10), + claim, + fixture: fixtureHash(dir), + devia: sourceHash(checkout), + against: checkout === packageRoot ? "installed" : checkout, + deviaVersion: environment().version, + }; + record.verification = { ...evidence, exit: run.code, outcome: result.outcome, note: result.note }; + // Kept alongside the latest run, because `fixed` means devia saw the problem and then saw it + // gone. One run can only ever be half of that. + if (result.outcome === "reproduced") record.reproduced = evidence; + save(deviaDir, record); + + if (ctx.json) { + const state = stateOf(record); + console.log( + JSON.stringify({ ok: state !== "rejected", id: record.id, state, verification: record.verification }, null, 2) + ); + return state === "rejected" ? 1 : 0; + } + + const state = stateOf(record); + const chain = evidenceChain(record); + heading(`devia contribute verify — ${record.id}`); + line(` ${color.dim("ran")} devia ${run.argv.join(" ")}`); + line(` ${color.dim("in")} ${dir}`); + line(` ${color.dim("against")} ${record.verification.against}`); + line(` ${color.dim("fixture")} ${record.verification.fixture || "unhashed"}`); + line(` ${color.dim("devia")} ${record.verification.devia || "unhashed"}`); + line(""); + status(state === "rejected" ? "FAIL" : "PASS", state, result.note); + + // A broken chain is the interesting case: the behaviour changed, but not for a reason that + // proves anything. Saying so is the whole point of hashing the two runs. + if (chain.breaks.length) { + for (const b of chain.breaks) status("WARN", "evidence chain", b); + line(""); + line(color.dim(" Recorded as reproduced, not fixed. Restore the fixture that failed, then")); + line(color.dim(" verify again with --devia pointing at the checkout that carries the fix.")); + } + + if (result.outcome === "inconclusive") { + line(""); + line(color.dim(" Neither profile matched, so nothing is claimed. A candidate is eligible")); + line(color.dim(" because devia reproduced it, never because the record says so (AGT-012).")); + if (run.err) line(color.dim(` stderr: ${run.err.split("\n")[0].slice(0, 140)}`)); + } else if (state === "rejected") { + line(""); + line(color.dim(" The expected behaviour is what this fixture does, and devia has never seen")); + line(color.dim(" it do anything else — so there is nothing here to report yet. Make the")); + line(color.dim(" fixture fail first; a fix is only a fix once the problem was shown.")); + } else { + line(color.dim(` Next: ${NEXT[state]}`)); + } + line(""); + return state === "rejected" ? 1 : 0; +} + +/** + * The payload, for a reader. + * + * The fixture is kept whole in the manifest on disk — a reproduction that reproduces is worth + * more than a short file list — but a terminal that prints forty template files buries the two + * that matter. So the documents are listed and the fixture is summed. + */ +function payloadLines(files) { + const out = []; + let reproFiles = 0; + let reproBytes = 0; + for (const f of manifest(files)) { + if (f.name.startsWith("repro/")) { + reproFiles++; + reproBytes += f.bytes; + continue; + } + out.push(` ${String(f.bytes).padStart(6)} B ${f.name}`); + } + if (reproFiles) { + out.push(` ${String(reproBytes).padStart(6)} B repro/ — ${reproFiles} files`); + } + return out; +} + +/** The payload, built and checked. Shared by `show` and `submit` so they cannot disagree. */ +function buildPayload(ctx, record) { + const { root, deviaDir } = ctx; + const verdictOf = eligibility(record, { deviaDir }); + const dir = reproDir(deviaDir, record); + const repro = dir && exists(dir) ? reproStats(dir) : null; + + // The fixture is read before the body is written, so the body can state what was redacted + // instead of asserting that something was. + const fixture = []; + if (repro) { + for (const rel of repro.list) { + const raw = read(path.join(dir, rel)); + if (raw === null) continue; + fixture.push({ name: `repro/${rel}`, raw }); + } + } + const redacted = redactions(record, { root, reproFiles: fixture.map((f) => f.raw) }); + + const body = renderBody(record, { route: verdictOf.route, repro, root, redacted }); + const title = sanitize(renderTitle(record), { root }).text; + + const files = [ + { name: "TITLE.txt", content: title + "\n" }, + { name: verdictOf.route === "pull_request" ? "PR.md" : "ISSUE.md", content: body }, + ...fixture.map((f) => ({ name: f.name, content: sanitize(f.raw, { root }).text })), + ]; + + return { + verdict: verdictOf, + title, + body, + files, + repro, + redacted, + residue: payloadResidue(files), + }; +} + +function cmdShow(ctx, id) { + const { deviaDir } = ctx; + const record = load(deviaDir, id); + if (!record) { + status("FAIL", `no such candidate: ${id}`); + return 1; + } + const payload = buildPayload(ctx, record); + + if (ctx.json) { + console.log( + JSON.stringify( + { + ok: true, + record, + state: stateOf(record), + eligibility: { + ok: payload.verdict.ok, + route: payload.verdict.route, + blockers: payload.verdict.blockers, + notes: payload.verdict.notes, + }, + payload: manifest(payload.files), + residue: payload.residue, + title: payload.title, + body: payload.body, + }, + null, + 2 + ) + ); + return 0; + } + + heading(`devia contribute show — ${record.id}`); + line(` ${color.dim("state")} ${stateOf(record)}`); + line(` ${color.dim("route")} ${payload.verdict.route || "nothing yet"}`); + line(""); + line(payload.title); + line(""); + line(payload.body); + line(color.bold("What would be sent")); + for (const l of payloadLines(payload.files)) line(l); + line(""); + for (const b of payload.verdict.blockers) status("FAIL", "blocked", b); + for (const n of payload.verdict.notes) status("INFO", "note", n); + for (const r of payload.residue) status("FAIL", "not sanitized", r); + line(""); + return 0; +} + +function cmdSubmit(ctx, id) { + const { root, deviaDir, flags } = ctx; + const record = load(deviaDir, id); + if (!record) { + status("FAIL", `no such candidate: ${id}`); + return 1; + } + + // A fix is recorded here rather than inferred: devia never reads a diff it was not pointed at. + if (flags["fix-repo"] || flags["fix-tests"] || flags.architectural) { + const repo = flags["fix-repo"] ? path.resolve(String(flags["fix-repo"])) : record.fix?.repo; + const fix = { ...(record.fix || {}), repo, architectural: Boolean(flags.architectural) }; + if (flags["fix-tests"]) fix.tests = csv(flags["fix-tests"]); + if (repo) Object.assign(fix, diffSize(repo)); + record.fix = fix; + save(deviaDir, record); + } + + const config = settings(deviaDir); + const payload = buildPayload(ctx, record); + const v = payload.verdict; + + heading(`devia contribute submit — ${record.id}`); + line(` ${color.dim("type")} ${record.type}`); + line(` ${color.dim("state")} ${v.state}`); + line(` ${color.dim("route")} ${v.route || "nothing to send"}`); + line(` ${color.dim("remote")} ${config.remote || "unknown"}`); + line(` ${color.dim("identity")} ${config.identity || color.yellow("not configured")}`); + line(""); + + // The payload is written whatever happens: a contribution nobody can read before it is sent + // is a contribution nobody reviewed. + const outDir = path.join(dirOf(deviaDir, record.id), "payload"); + fs.rmSync(outDir, { recursive: true, force: true }); + for (const f of payload.files) writeFile(path.join(outDir, f.name), f.content); + writeFile( + path.join(outDir, "MANIFEST.md"), + "# What would be sent\n\n| Bytes | File | sha256 |\n|---|---|---|\n" + + manifest(payload.files) + .map((f) => `| ${f.bytes} | ${f.name} | ${f.sha256} |`) + .join("\n") + + "\n\nNothing else leaves this machine. Built by `devia contribute submit`.\n" + ); + status("PASS", "payload written", path.relative(root, outDir).split(path.sep).join("/")); + for (const l of payloadLines(payload.files)) line(l); + line(""); + + const refuse = (why, detail) => { + status("FAIL", why, detail); + line(""); + line(color.dim(" Nothing was sent. The payload above is on disk for you to read.")); + line(""); + return 1; + }; + + if (config.explicitlyDisabled) { + return refuse("contribution is disabled here", 'devia.json: contribution.enabled is false'); + } + if (payload.residue.length) { + return refuse("the payload is not clean", payload.residue.join("; ")); + } + for (const b of v.blockers) status("FAIL", "blocked", b); + for (const n of v.notes) status("INFO", "note", n); + if (!v.ok) { + line(""); + line(color.dim(" Not eligible. A contribution leaves this machine only once devia itself")); + line(color.dim(" reproduced the problem (AGT-012).")); + line(""); + return 1; + } + if (v.route === "advisory") { + return refuse("security defects are reported privately", "see SECURITY.md"); + } + + const remote = config.remote; + const command = + v.route === "pull_request" + ? ["gh", "pr", "create", "--repo", remote, "--title", payload.title, "--body-file", + path.join(outDir, "PR.md")] + : ["gh", "issue", "create", "--repo", remote, "--title", payload.title, "--body-file", + path.join(outDir, "ISSUE.md")]; + + line(""); + line(color.bold(" Remote operation")); + line(` ${command.map((a) => (/\s/.test(a) ? `"${a}"` : a)).join(" ")}`); + line(""); + + if (!flags.yes) { + status("SKIP", "not sent", "--yes authorises the remote operation"); + line(""); + line(color.dim(" devia holds no GitHub token and sends nothing on its own. Run the command")); + line(color.dim(" above yourself, or re-run this with --yes to have gh run it.")); + line(""); + return 0; + } + + const decision = identityCheck({ + identity: config.identity, + owner: upstream()?.owner, + lookup: ghAccount, + }); + if (!decision.ok) return refuse(decision.why, decision.detail); + + if (v.route === "pull_request") { + const pushed = v.fix?.repo ? tracking(v.fix.repo) : null; + if (!pushed) { + return refuse( + "the fix branch is not pushed", + "devia does not commit or push in your checkout — push the branch, then re-run" + ); + } + line(color.dim(` head: ${pushed}`)); + } + + try { + const out = execFileSync(command[0], command.slice(1), { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + status("PASS", `${v.route} created`, out.trim().split("\n").pop()); + } catch (e) { + status("FAIL", "gh refused", String(e.stderr || e.message).split("\n")[0].slice(0, 160)); + return 1; + } + line(""); + return 0; +} + +/** Size of the fix in a devia checkout, so the PR-versus-issue routing has a number to use. */ +function diffSize(repo) { + try { + const out = execFileSync("git", ["diff", "--numstat", "HEAD"], { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + const rows = out ? out.split("\n") : []; + let lines = 0; + for (const r of rows) { + const [a, d] = r.split(/\s+/); + lines += (Number(a) || 0) + (Number(d) || 0); + } + return { files: rows.length, lines }; + } catch { + return {}; + } +} + +/** The upstream branch a checkout is on, or null when nothing has been pushed. */ +function tracking(repo) { + try { + return execFileSync("git", ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return null; + } +} + +function cmdRemove(ctx, id) { + const { deviaDir } = ctx; + const record = load(deviaDir, id); + if (!record) { + status("FAIL", `no such candidate: ${id}`); + return 1; + } + fs.rmSync(dirOf(deviaDir, record.id), { recursive: true, force: true }); + heading("devia contribute rm"); + status("PASS", record.id, "removed"); + line(color.dim(" The id is not reissued: the next candidate takes the following number.")); + line(""); + return 0; +} + +export default async function contribute(ctx) { + const { deviaDir, args, flags } = ctx; + const action = args._[1] || "list"; + const id = args._[2] ? String(args._[2]).toUpperCase() : null; + + if (flags.help) { + line(HELP.trim()); + return 0; + } + + if (!exists(deviaDir)) { + status("FAIL", "no .devia/", "run `devia init` first"); + return 1; + } + + const config = settings(deviaDir); + if (config.explicitlyDisabled && action !== "list") { + heading("devia contribute"); + status("SKIP", "disabled for this repository", "devia.json: contribution.enabled is false"); + line(""); + return 0; + } + + if (action === "list") return cmdList(ctx); + if (action === "new") return cmdNew(ctx); + + if (["repro", "verify", "show", "submit", "rm"].includes(action)) { + if (!id) { + status("FAIL", `devia contribute ${action} needs an id`, "e.g. C1"); + return 2; + } + if (action === "repro") return cmdRepro(ctx, id); + if (action === "verify") return cmdVerify(ctx, id); + if (action === "show") return cmdShow(ctx, id); + if (action === "submit") return cmdSubmit(ctx, id); + if (action === "rm") return cmdRemove(ctx, id); + } + + status("FAIL", `unknown action: ${action}`); + line(HELP.trim()); + return 2; +} + +export { buildPayload, ghAccount }; diff --git a/src/commands/init.mjs b/src/commands/init.mjs index 036f511..bce5a23 100644 --- a/src/commands/init.mjs +++ b/src/commands/init.mjs @@ -4,6 +4,7 @@ import { packageRoot, exists, read, writeFile, writeJSON, walk } from "../lib/fs import { trackedFiles } from "../lib/git.mjs"; import { cliVersion, standardVersion } from "../lib/version.mjs"; import { vendorStandard } from "../lib/vendor.mjs"; +import { DEFAULT_BUDGET } from "../lib/context.mjs"; import { color, heading, status, line } from "../lib/ui.mjs"; import { installAdapters, ADAPTERS } from "./skills.mjs"; @@ -141,7 +142,23 @@ ${color.bold("devia init")} — create .devia/ in this repository } status(written ? "PASS" : "SKIP", `memory files: ${written} written`, kept ? `${kept} kept` : ""); - // 2. Configuration + // 2. What inside the memory is generated rather than authored. + // + // A nested .gitignore, not an edit to the project's own: devia owns `.devia/`, and appending + // to a file the user wrote is not a thing `init` should do silently. The name is written + // rather than shipped as a template file because npm does not carry a `.gitignore` inside a + // published package. + const ignorePath = path.join(deviaDir, ".gitignore"); + if (!exists(ignorePath) || force) { + writeFile( + ignorePath, + "# Generated by devia, rebuilt on demand — the memory itself is committed.\n" + + "reader.html\n" + + "contributions/*/payload/\n" + ); + } + + // 3. Configuration const configPath = path.join(deviaDir, "devia.json"); if (!exists(configPath) || force) { writeJSON(configPath, { @@ -155,6 +172,11 @@ ${color.bold("devia init")} — create .devia/ in this repository design: profile !== "service" && profile !== "cli" && profile !== "library", memory: true, }, + // Written so the knob is visible rather than folklore. `contribution` carries no identity: + // the local half needs none, and the half that can publish must stay impossible until + // somebody deliberately names an account. + context: { budget: DEFAULT_BUDGET, mode: "advisory" }, + contribution: { enabled: true }, waivers: [], initializedAt: vars.DATE, }); @@ -163,7 +185,7 @@ ${color.bold("devia init")} — create .devia/ in this repository status("SKIP", "devia.json kept", "use --force to regenerate"); } - // 3. Pinned standard — opt-in. Vendoring writes ~390 files a project did not author, which + // 4. Pinned standard — opt-in. Vendoring writes ~390 files a project did not author, which // buries the memory it is supposed to serve: on a real repository the ratio was 17 files of // memory to 391 of copy, and every `sync` produced a 391-file diff. The rules stay reachable // through `devia rules`, and `devia sync` pins the copy for whoever needs it offline. @@ -179,7 +201,7 @@ ${color.bold("devia init")} — create .devia/ in this repository status("SKIP", "standard not pinned", "`devia sync` writes .devia/standard/ when you need it"); } - // 4. Agent adapters + // 5. Agent adapters if (flags.agents === false || flags["no-agents"]) { status("SKIP", "agent adapters not written", "--no-agents"); } else { diff --git a/src/lib/context.mjs b/src/lib/context.mjs new file mode 100644 index 0000000..7fb57cf --- /dev/null +++ b/src/lib/context.mjs @@ -0,0 +1,820 @@ +import path from "node:path"; +import { exists, read, readJSON, walk, packageRoot } from "./fs.mjs"; +import { parseYaml } from "./yaml.mjs"; +import { loadRules } from "./rules.mjs"; +import { estimateTokens } from "./tokens.mjs"; +import { gatesByRule } from "./gates.mjs"; + +/** + * Which part of devia an agent actually needs for the task in front of it. + * + * Storing knowledge and delivering it are two different jobs. `.devia/` plus the registry is + * everything devia knows about a project; the amount of it that belongs in a context window for + * "add POST /api/orders" is a small fraction, and the rest is noise that pushes the code the + * agent is supposed to read out of the window. + * + * FULL KNOWLEDGE -> ROUTER -> TIERS -> BUDGET -> MINIMAL CONTEXT + * + * Two invariants hold the whole thing up: + * + * 1. A blocking constraint is admitted before the budget is consulted and is never evicted. A + * small budget produces a reported overrun, never a silently dropped P0 (`AGT-013`). + * 2. Every selection carries its reason. "Why was this here?" and "why was this not?" are both + * answerable, because a selector nobody can interrogate is a selector nobody should trust. + */ + +export const DEFAULT_BUDGET = 1200; + +/** Tier order is selection order. T0 is admitted first and unconditionally. */ +export const TIERS = { + T0: "blocking constraints", + T1: "task-critical policy", + T2: "project knowledge", + T3: "known failures", + T4: "supporting context", + T5: "not relevant", +}; + +/** + * The contract applies to every task, whatever the task is: an agent inventing an endpoint or + * deleting a debt line is a failure in a database change as much as in a UI one. + */ +const ALWAYS_ON_DOMAINS = ["agent", "memory"]; + +/** + * Task words -> rule domains. Data, not logic: a domain earns an entry by being the thing + * somebody would actually have needed, and the table is the whole routing model for prose. + */ +const KEYWORDS = { + api: ["endpoint", "endpoints", "route", "routes", "rest", "api", "http", "request", "response", + "webhook", "handler", "controller", "graphql", "rpc", "payload", "post", "patch", "put", + "delete", "idempotency", "pagination", "versioning"], + database: ["migration", "migrations", "schema", "table", "tables", "column", "columns", "index", + "query", "sql", "orm", "prisma", "postgres", "mysql", "database", "transaction", "rollback", + "seed", "constraint", "foreign"], + security: ["auth", "authentication", "authorization", "authorize", "login", "logout", "session", + "token", "jwt", "permission", "permissions", "role", "roles", "tenant", "tenancy", "secret", + "credential", "password", "csrf", "xss", "injection", "encrypt", "hash", "rate"], + privacy: ["personal", "pii", "gdpr", "retention", "consent", "anonymise", "anonymize", "erasure", + "export", "subject"], + testing: ["test", "tests", "spec", "coverage", "fixture", "mock", "stub", "regression", "e2e", + "unit", "integration"], + devops: ["deploy", "deployment", "ci", "pipeline", "docker", "container", "release", "revert", + "environment", "staging", "production", "lockfile", "build"], + observability: ["log", "logs", "logging", "metric", "metrics", "alert", "alerting", "trace", + "tracing", "monitor", "monitoring", "incident", "dashboard"], + architecture: ["architecture", "boundary", "boundaries", "layer", "module", "modules", "coupling", + "dependency", "dependencies", "refactor", "structure", "package"], + ai: ["llm", "prompt", "prompts", "embedding", "completion", "agent", "model", "inference", + "tool-call", "rag"], + ui: ["button", "buttons", "layout", "spacing", "colour", "color", "typography", "icon", "icons", + "screen", "page", "grid", "css", "style", "styling", "theme", "visual"], + ux: ["form", "forms", "navigation", "nav", "flow", "onboarding", "wizard", "copy", "microcopy", + "usability", "journey", "search", "filter"], + accessibility: ["a11y", "accessible", "accessibility", "aria", "keyboard", "focus", "reader", + "contrast", "wcag", "screenreader", "tab", "label"], + states: ["loading", "empty", "skeleton", "offline", "optimistic", "retry", "stale", "timeout", + "disabled", "error"], + // "table" is deliberately absent: it belongs to `database` far more often than to a UI + // component, and routing it here sent "add a column to the orders table" through + // components -> accessibility and pulled the whole screen-rule corpus into a schema change. + // A real table component says component, screen, or lives in a .tsx file. + components: ["component", "components", "dialog", "modal", "tooltip", "dropdown", + "menu", "tabs", "datatable"], + "design-system": ["token", "tokens", "variant", "variants", "primitive", "design"], + "data-display": ["currency", "date", "dates", "time", "number", "numbers", "percentage", + "locale", "format", "sorting", "pagination"], + localization: ["i18n", "l10n", "translation", "translate", "locale", "rtl", "pluralization"], + responsive: ["responsive", "mobile", "tablet", "desktop", "breakpoint", "viewport"], + interaction: ["touch", "mouse", "pointer", "gesture", "hover", "drag"], + motion: ["animation", "animate", "transition", "motion"], + content: ["wording", "message", "messages", "tone", "microcopy"], +}; + +/** Changed paths -> rule domains. A file's location says more than a sentence about it. */ +const PATH_DOMAINS = [ + [/(^|\/)(migrations?|migrate|alembic|prisma)(\/|$)/i, ["database"]], + [/\.sql$/i, ["database"]], + [/(^|\/)(models?|entities|schema)(\/|$)/i, ["database", "architecture"]], + [/(^|\/)(api|routes?|controllers?|handlers?|endpoints?|resolvers?)(\/|$)/i, ["api"]], + [/(^|\/)(auth|security|permissions?|policies|policy|rbac)(\/|$)/i, ["security"]], + [/(^|\/)(middleware|guards?)(\/|$)/i, ["security", "api"]], + [/\.(tsx|jsx|vue|svelte)$/i, ["ui", "ux", "accessibility", "states", "components"]], + [/\.(css|scss|sass|less)$/i, ["ui", "design-system", "responsive"]], + [/(^|\/)(components?|ui|design-system)(\/|$)/i, ["ui", "components", "design-system"]], + [/(^|\/)(pages?|views?|screens?|app)(\/|$)/i, ["ui", "ux", "states"]], + [/(^|\/)(locales?|i18n|translations?)(\/|$)/i, ["localization"]], + [/(^|\/)(tests?|spec|specs|__tests__|e2e)(\/|$)/i, ["testing"]], + [/\.(test|spec)\.[a-z]+$/i, ["testing"]], + [/(^|\/)\.github\/workflows(\/|$)/i, ["devops", "testing"]], + [/(^|\/)(Dockerfile|docker-compose|\.dockerignore)/i, ["devops"]], + [/(^|\/)(terraform|infra|deploy|k8s|helm)(\/|$)/i, ["devops"]], + [/(^|\/)(telemetry|logging|metrics|observability)(\/|$)/i, ["observability"]], +]; + +/** + * Domains that drag other domains in with them. + * + * An endpoint is an authorization surface and an input boundary whether or not the task says the + * word "authorization" — and the benchmark proved it: "add POST /api/orders" routed to `api` + * alone and dropped `SEC-001` and `SEC-003`, the two rules a write endpoint most needs, at every + * budget. Routing on the words a task happens to use is not routing on what the task *is*. + */ +const IMPLIES = { + api: ["security"], + database: ["privacy"], + ui: ["accessibility", "states"], + ux: ["accessibility", "content"], + components: ["accessibility"], + ai: ["security"], +}; + +/** Memory file -> the domains it speaks for. `14_INDEX.md` is a map, never context in itself. */ +const MEMORY_DOMAINS = { + "00_OVERVIEW.md": ["architecture"], + "01_ARCHITECTURE.md": ["architecture", "api", "database"], + "02_SURFACES.md": ["api", "ui", "ux", "devops"], + "03_DATA_MODEL.md": ["database", "privacy"], + "04_PERMISSIONS.md": ["security", "privacy"], + "05_FLOWS.md": ["ux", "testing", "api"], + "06_INTEGRATIONS.md": ["api", "devops", "ai", "security"], + "07_DESIGN.md": ["ui", "ux", "design-system", "components", "accessibility", "motion", + "responsive", "states", "data-display", "localization", "interaction", "content"], + "13_RECIPES.md": ["devops", "testing", "architecture"], +}; + +const STOPWORDS = new Set([ + "the", "a", "an", "and", "or", "but", "for", "to", "of", "in", "on", "at", "by", "with", + "from", "into", "as", "is", "are", "be", "was", "were", "it", "its", "this", "that", "these", + "those", "we", "i", "you", "they", "our", "their", "add", "new", "make", "should", "must", + "can", "will", "would", "when", "then", "so", "not", "no", "do", "does", "up", "out", "if", +]); + +/** Significant words of a task description, lowercased and de-duplicated. */ +export function terms(text) { + const out = new Set(); + const keep = (w) => { + if (w.length < 3 || STOPWORDS.has(w)) return; + out.add(w); + // `orders` and `order` must meet: the task says one and the memory says the other. + if (w.length > 4 && w.endsWith("s")) out.add(w.slice(0, -1)); + }; + + for (const w of String(text ?? "").toLowerCase().match(/[a-z][a-z0-9_-]{1,}/g) || []) { + keep(w); + // A joined identifier carries its parts. Without this `new_endpoint` is one opaque word: + // it matches neither the `endpoint` keyword nor the `new_endpoint` change type, whose own + // key splits the same way. + if (/[_-]/.test(w)) { + for (const part of w.split(/[_-]+/)) keep(part); + } + } + return out; +} + +/** Domains a task text and a set of changed paths point at, with the evidence for each. */ +export function routeDomains(task, files = []) { + const found = new Map(); + const note = (domain, why) => { + if (!found.has(domain)) found.set(domain, new Set()); + found.get(domain).add(why); + }; + + const words = terms(task); + for (const [domain, keys] of Object.entries(KEYWORDS)) { + for (const k of keys) { + if (words.has(k)) note(domain, `task mentions "${k}"`); + } + } + for (const f of files) { + const rel = String(f).split(path.sep).join("/"); + for (const [re, domains] of PATH_DOMAINS) { + if (!re.test(rel)) continue; + for (const d of domains) note(d, `changed path ${rel}`); + } + } + + applyImplications(found); + return found; +} + +/** + * One pass, not a closure: `api` pulls in `security`, and that is the end of it. A transitive + * walk would quietly route half the registry from one keyword. + */ +function applyImplications(found) { + for (const domain of [...found.keys()]) { + for (const implied of IMPLIES[domain] || []) { + if (!found.has(implied)) found.set(implied, new Set()); + found.get(implied).add(`implied by ${domain}`); + } + } + return found; +} + +/** Split a memory file into sections at `##`, keeping the preamble as its own section. */ +function sections(file, text) { + const out = []; + const lines = String(text).split(/\r?\n/); + let title = "intro"; + let buf = []; + const flush = () => { + const body = buf.join("\n").trim(); + if (body) out.push({ file, title, text: body }); + buf = []; + }; + for (const l of lines) { + const h = l.match(/^##\s+(.*)$/); + if (h) { + flush(); + title = h[1].trim(); + } + buf.push(l); + } + flush(); + return out; +} + +/** Open rows of a registry table: the ones above the closed section. */ +function openRows(text, prefix) { + const out = []; + for (const l of String(text ?? "").split(/\r?\n/)) { + if (/^##\s+(Closed|Discharged)/i.test(l)) break; + const m = l.match(new RegExp(`^\\|\\s*(${prefix}\\d+)\\s*\\|`)); + if (!m || /TODO\(devia\)/.test(l)) continue; + out.push({ id: m[1], text: l.trim() }); + } + return out; +} + +/** The `- **Never ...**` / `- Always ...` bullets, one item each (`MEM-010`). */ +function neverAlways(text) { + const out = []; + let section = null; + for (const l of String(text ?? "").split(/\r?\n/)) { + const h = l.match(/^##\s+(Never|Always)\b/i); + if (h) { + section = h[1].toLowerCase(); + continue; + } + if (/^##\s/.test(l)) section = null; + if (!section) continue; + const b = l.match(/^\s*[-*]\s+(.*)$/); + if (b && b[1].trim()) out.push({ kind: section, text: b[1].trim() }); + else if (out.length && /^\s{2,}\S/.test(l)) out[out.length - 1].text += " " + l.trim(); + } + return out; +} + +/** + * A rule devia verifies itself does not need its requirement recited into the window: the agent + * needs the citation and the fact that a gate will stop it. A rule only a human can check needs + * its full text, because nothing else is going to state it (`AGT-013`). + * + * The exception matters more than the rule. A P0 rule whose only gate *warns* is not actually + * being stopped by anything, so compacting it would trade the agent's copy of a blocking + * obligation for a gate that will let the change through. Those keep their full text. + */ +export function renderRule(rule, gates) { + const head = `${rule.id} · ${rule.severity} · ${rule.priority} — ${rule.title}`; + const blocking = (gates || []).some((g) => g.priority === "P0"); + const covered = gates?.length && (blocking || rule.priority !== "P0"); + if (covered) { + const ids = gates.map((g) => `${g.id} (${g.priority})`).join(", "); + return `- ${head}\n checked by \`devia check\` → ${ids} → ${ + blocking ? "blocks the change" : "reported, not blocking" + }`; + } + return `- ${head}\n ${String(rule.requirement).trim()}`; +} + +/** Where the rules come from: a pinned copy when the project has one, the package otherwise. */ +export function rulesDir(deviaDir) { + const vendored = path.join(deviaDir, "standard", "rules"); + return exists(vendored) ? vendored : path.join(packageRoot, "rules"); +} + +/** + * Everything devia could say about this project, as addressable items. Built once; the router + * scores it and the budget cuts it. Nothing is filtered here, so the raw total is honest. + */ +export function buildCorpus({ root, deviaDir }) { + const items = []; + const byRule = gatesByRule(); + + const { rules } = loadRules(rulesDir(deviaDir)); + for (const rule of rules) { + if (rule.status !== "active") continue; + const gates = byRule.get(rule.id); + const head = `${rule.id} · ${rule.severity} · ${rule.priority} — ${rule.title}`; + const text = renderRule(rule, gates); + const full = `- ${head}\n ${String(rule.requirement).trim()}`; + const gated = gates?.length + ? gates.some((g) => g.priority === "P0") + ? " · gated, blocks" + : " · gated" + : ""; + items.push({ + kind: "rule", + id: rule.id, + domains: [rule.domain], + priority: rule.priority, + // Cited rather than recited, because a devia gate verifies it (`AGT-013`). + compacted: text !== full, + text, + full, + // The two smaller forms a strict target can fall back to, rather than dropping the rule. + short: `- ${head}${gated}`, + ref: rule.id, + }); + } + + const memoryFiles = exists(deviaDir) + ? walk(deviaDir, { filter: (f) => f.endsWith(".md") && !f.includes(path.sep) }) + : []; + + for (const file of memoryFiles) { + if (file === "14_INDEX.md" || file === "README.md") continue; + const text = read(path.join(deviaDir, file)) || ""; + + if (file === "10_NEVER_ALWAYS.md") { + for (const [i, line] of neverAlways(text).entries()) { + items.push({ + kind: line.kind, + id: `${file}#${line.kind}-${i + 1}`, + domains: [], + text: `- ${line.text}`, + }); + } + continue; + } + if (file === "11_GAPS.md" || file === "12_DEBT.md") { + const prefix = file.startsWith("11") ? "G" : "D"; + for (const row of openRows(text, prefix)) { + items.push({ + kind: prefix === "G" ? "gap" : "debt", + id: row.id, + domains: [], + text: row.text, + }); + } + continue; + } + + for (const s of sections(file, text)) { + items.push({ + kind: "memory", + id: `${file}#${s.title}`, + domains: MEMORY_DOMAINS[file] || [], + text: s.text, + }); + } + } + + const mapText = read(path.join(deviaDir, "impact-map.yaml")); + const impacts = mapText ? parseYaml(mapText).impacts || {} : {}; + for (const [change, targets] of Object.entries(impacts)) { + const list = [].concat(targets || []).map(String); + if (!list.length) continue; + items.push({ + kind: "impact", + id: change, + domains: [], + // Kept as data, not only as prose: this is what lets the impact map route (`AGT-013`). + targets: list, + text: `- ${change} → update ${list.join(", ")} in the same change (MEM-009)`, + }); + } + + for (const item of items) { + item.tokens = estimateTokens(item.text); + item.rawTokens = estimateTokens(item.full || item.text); + item.shortTokens = item.short ? estimateTokens(item.short) : item.tokens; + // Two for the separator the reference list joins them with. + item.refTokens = item.ref ? estimateTokens(item.ref) + 2 : item.shortTokens; + } + return items; +} + +/** How many distinct task terms appear in a text. The cheap half of relevance. */ +function textScore(text, words) { + if (!words.size) return 0; + const lower = String(text).toLowerCase(); + let n = 0; + for (const w of words) { + if (lower.includes(w)) n++; + } + return n; +} + +/** Memory file -> the domains it speaks for, inverted once. */ +function domainsOfFile(file) { + return MEMORY_DOMAINS[file] || []; +} + +/** + * Which change types in the impact map this task is. + * + * The impact map is the one routing table the *project* wrote. `new_endpoint → 02_SURFACES.md` + * is a declaration that this project already made, in this project's own vocabulary, and it is + * worth more than devia guessing from a keyword list — a project that invented + * `new_consent_record` routes on it exactly as well as a built-in one does. + * + * A key is matched on its significant parts rather than as a whole: half of them present is + * enough to consider it, which catches "add an endpoint" for `new_endpoint` without letting a + * single shared word pull in `state_machine_change` for "fix the empty state". + */ +export function matchedChangeTypes(items, words, forced = []) { + const wanted = new Set(forced.map((t) => String(t))); + const out = new Map(); + + for (const item of items) { + if (item.kind !== "impact") continue; + const id = String(item.id); + + if (wanted.has(id)) { + out.set(id, { score: 1, why: "--type" }); + continue; + } + const parts = [...new Set(id.split(/[_\-.]/).map((p) => p.toLowerCase()))].filter( + (p) => p.length >= 3 && !STOPWORDS.has(p) + ); + if (!parts.length) continue; + const hit = parts.filter((p) => words.has(p)); + if (!hit.length) continue; + const score = hit.length / parts.length; + if (score < 0.5) continue; + out.set(id, { score, why: `task mentions ${hit.map((h) => `"${h}"`).join(", ")}` }); + } + return out; +} + +/** + * Domains a matched change type implies, taken from the memory files it declares. + * + * This is the part that makes the impact map a router rather than a checklist: the project said + * a permission change touches `04_PERMISSIONS.md`, and `04_PERMISSIONS.md` speaks for security + * and privacy, so a permission change routes to security and privacy — without anyone adding a + * keyword for this project's word for it. + */ +function changeTypeDomains(items, matched) { + const out = new Map(); + for (const item of items) { + if (item.kind !== "impact" || !matched.has(item.id)) continue; + for (const file of item.targets || []) { + for (const domain of domainsOfFile(file)) { + if (!out.has(domain)) out.set(domain, new Set()); + out.get(domain).add(`${item.id} updates ${file}`); + } + } + } + return out; +} + +/** + * Score, tier and reason every item. Selection happens afterwards: this pass decides what the + * context *is*, the budget decides how much of it fits. + */ +export function classify( + items, + { task = "", files = [], domains: forced = [], changeTypes: forcedTypes = [] } = {} +) { + const words = terms(task); + const routed = routeDomains(task, files); + for (const d of forced) { + if (!routed.has(d)) routed.set(d, new Set(["--domain"])); + } + + // The project's own declaration routes before devia's keyword table gets an opinion. + const changeTypes = matchedChangeTypes(items, words, forcedTypes); + for (const [domain, why] of changeTypeDomains(items, changeTypes)) { + if (!routed.has(domain)) routed.set(domain, new Set()); + for (const w of why) routed.get(domain).add(w); + } + // The change types routed after `routeDomains` ran, so their implications are applied here: + // a `new_endpoint` is still an authorization surface. + applyImplications(routed); + + // The memory files those change types name are the MEM-009 duty's subject. + const duty = new Set(); + for (const item of items) { + if (item.kind === "impact" && changeTypes.has(item.id)) { + for (const f of item.targets || []) duty.add(f); + } + } + + const active = new Set([...routed.keys(), ...ALWAYS_ON_DOMAINS]); + + for (const item of items) { + const why = []; + let tier = "T5"; + let relevance = 0; + + const inDomain = item.domains.some((d) => active.has(d)); + const routedDomain = item.domains.filter((d) => routed.has(d)); + for (const d of routedDomain) { + for (const reason of routed.get(d)) why.push(`${d}: ${reason}`); + } + + if (item.kind === "rule") { + if (inDomain) { + relevance = routedDomain.length ? 3 + routedDomain.length : 1; + if (item.priority === "P0") tier = "T0"; + else if (item.priority === "P1") tier = "T1"; + else tier = "T4"; + if (!routedDomain.length) { + why.push(`${item.domains[0]}: applies to every task`); + relevance = item.priority === "P0" ? 2 : 1; + } + } else { + why.push(`domain ${item.domains[0]} is not in scope`); + } + } else if (item.kind === "never" || item.kind === "always") { + // The project's own earned constraints. Short by construction (`MEM-010`), and the single + // highest-value thing devia knows that the standard does not: never budget-evicted. + tier = "T0"; + relevance = 5 + textScore(item.text, words); + why.push("10_NEVER_ALWAYS.md: this project's own constraint"); + } else if (item.kind === "impact") { + const match = changeTypes.get(item.id); + if (match) { + tier = "T0"; + relevance = 6; + why.push(`task is a ${item.id} — ${match.why} (MEM-009 duty)`); + } else { + why.push("this task is not that change type"); + } + } else if (item.kind === "memory") { + const file = String(item.id).split("#")[0]; + const owed = duty.has(file); + const hits = textScore(item.text, words); + if (owed || inDomain || hits >= 2) { + tier = "T2"; + relevance = routedDomain.length + hits + (owed ? 4 : 0); + // A file the impact map says this change must update is not a guess about relevance. + if (owed) why.push(`the impact map says this change updates ${file}`); + if (hits) why.push(`memory section matches ${hits} task term(s)`); + if (!why.length) why.push("memory for a surface this task touches"); + } else { + why.push("no term or surface in common with the task"); + } + } else if (item.kind === "gap" || item.kind === "debt") { + const hits = textScore(item.text, words); + if (hits) { + tier = "T3"; + relevance = hits; + why.push(`open ${item.kind} line matching ${hits} task term(s)`); + } else { + why.push(`open ${item.kind} line unrelated to this task`); + } + } + + item.tier = tier; + item.relevance = relevance; + item.why = why; + // Did anything about *this* task point at this item, or would it have been delivered for + // any task at all? The contract and the project's own constraints are deliberately in the + // second group; measuring the split is how the selection's noise stops being a guess. + item.taskLinked = Boolean( + routedDomain.length || + (item.kind === "impact" && changeTypes.has(item.id)) || + (item.kind === "memory" && (duty.has(String(item.id).split("#")[0]) || textScore(item.text, words) > 0)) || + ((item.kind === "gap" || item.kind === "debt") && tier === "T3") + ); + } + + return { items, domains: routed, changeTypes: [...changeTypes.keys()], duty: [...duty] }; +} + +const ORDER = ["T0", "T1", "T2", "T3", "T4"]; + +export const MODES = ["advisory", "strict"]; + +/** One line standing in for every never/always bullet, when even those cannot be afforded. */ +const POINTER = (n) => + `- ${n} never/always line(s) this project earned are not included: read ` + + "`.devia/10_NEVER_ALWAYS.md` before changing anything."; + +/** + * The smallest form a mandatory item may take, and the note explaining it when it does. + * + * A rule's requirement is generic text any agent can fetch again with `devia rules --id`, so a + * rule can shrink to its identifier. A never/always line is this project's own earned trap and + * exists nowhere else, so the most it can shrink to is a pointer telling the agent to go read + * them. An impact-map duty is already one line. Nothing here ever drops an item without leaving + * its name behind: the floor compresses, it does not disappear. + */ +const SMALLEST = { + rule: "ref", + never: "pointer", + always: "pointer", +}; + +const DEGRADATION_NOTE = { + short: "carry their identifier and title instead of their requirement", + ref: "are listed as identifiers only (`devia rules --id `)", + pointer: "are replaced by a pointer to the file", +}; + +/** Best form first: an item is restored as far up this list as the target allows. */ +const LADDER = ["full", "short", "ref", "pointer"]; + +function costOf(item, level) { + if (level === "short") return item.shortTokens; + if (level === "ref") return item.refTokens; + if (level === "pointer") return 0; + return item.tokens; +} + +function smallestLevel(item) { + return SMALLEST[item.kind] || "full"; +} + +function floorCost(blocking, levels) { + let n = 0; + let pointers = 0; + for (const item of blocking) { + const level = levels.get(item.id) || "full"; + if (level === "pointer") pointers++; + n += costOf(item, level); + } + return n + (pointers ? estimateTokens(POINTER(pointers)) : 0); +} + +/** + * Fit the classified items to the target. + * + * Three numbers, deliberately separate, because collapsing them is what made "budget 600, + * selected 1380" look like a broken promise instead of a stated one: + * + * target what was asked for + * floor what the mandatory items cost — not negotiable, only compressible + * spent what was actually selected + * + * `advisory` (the default) never lets the target evict a mandatory item: when the floor is above + * the target it reports `over` and includes them anyway. `strict` never exceeds the target, and + * compresses the floor only as far as it has to — every mandatory item starts at its smallest + * form and is restored toward full text while the target allows, in relevance order. The first + * version of this degraded all of them at once and then spent the freed tokens admitting + * *optional* rules at full text, which is precisely backwards. + * + * When even the smallest floor is too large, `strict` says `impossible` rather than going over. + */ +export function select(items, { target = DEFAULT_BUDGET, mode = "advisory" } = {}) { + const strict = mode === "strict"; + const included = []; + const excluded = []; + + const blocking = items.filter((i) => i.tier === "T0"); + blocking.sort((a, b) => b.relevance - a.relevance || a.tokens - b.tokens); + + const levels = new Map(); + const fullFloor = floorCost(blocking, levels); + + if (strict && fullFloor > target) { + // Start at the floor of the floor, then buy back as much text as the target affords. The + // most relevant mandatory item is restored first, because that is the one being read. + for (const item of blocking) levels.set(item.id, smallestLevel(item)); + for (const item of blocking) { + const smallest = smallestLevel(item); + for (const level of LADDER) { + if (level === smallest) break; + levels.set(item.id, level); + if (floorCost(blocking, levels) <= target) break; + levels.set(item.id, smallest); + } + } + } + + const floor = floorCost(blocking, levels); + let spent = floor; + + const applied = []; + for (const level of ["short", "ref", "pointer"]) { + const n = blocking.filter((i) => (levels.get(i.id) || "full") === level).length; + if (n) applied.push({ id: level, note: `mandatory items ${DEGRADATION_NOTE[level]}`, items: n }); + } + + for (const item of blocking) { + included.push({ ...item, level: levels.get(item.id) || "full" }); + } + + const impossible = strict && floor > target; + + if (!impossible) { + for (const tier of ORDER.slice(1)) { + const group = items.filter((i) => i.tier === tier); + group.sort((a, b) => b.relevance - a.relevance || a.tokens - b.tokens); + for (const item of group) { + if (spent + item.tokens <= target) { + included.push({ ...item, level: "full" }); + spent += item.tokens; + } else { + excluded.push({ ...item, reason: "budget" }); + } + } + } + } else { + for (const item of items) { + if (item.tier !== "T0" && item.tier !== "T5") excluded.push({ ...item, reason: "budget" }); + } + } + + for (const item of items.filter((i) => i.tier === "T5")) { + excluded.push({ ...item, reason: "not relevant" }); + } + + const status = impossible + ? "impossible" + : applied.length + ? "degraded" + : spent > target + ? "over" + : "within"; + + return { + included, + excluded, + mode, + target, + floor, + fullFloor, + spent, + status, + degraded: applied.map(({ id, note, items: n }) => ({ id, note, items: n })), + // `over` is only reachable in advisory mode, and only because a mandatory item was kept. + overrun: status === "over", + compliant: spent <= target, + mandatory: blocking.length, + raw: items.reduce((n, i) => n + i.rawTokens, 0), + }; +} + +const TIER_HEADINGS = { + T0: "Blocking — these stop the change", + T1: "Policy for this task", + T2: "This project", + T3: "Known failures here", + T4: "Supporting", +}; + +/** The selection, as the text an agent receives. */ +export function render(selection, { task = "" } = {}) { + const out = []; + out.push("# devia context"); + if (task) out.push(`\nTask: ${task}`); + + for (const tier of ORDER) { + const group = selection.included.filter((i) => i.tier === tier); + if (!group.length) continue; + out.push(`\n## ${TIER_HEADINGS[tier]}\n`); + + const refs = group.filter((i) => i.level === "ref"); + const pointers = group.filter((i) => i.level === "pointer"); + for (const item of group) { + if (item.level === "ref" || item.level === "pointer") continue; + out.push(item.level === "short" ? item.short : item.text); + } + if (refs.length) { + out.push( + `\nThese apply and their text did not fit the target — read them before you rely on ` + + "memory (`devia rules --id `):\n" + + refs.map((i) => i.ref).join(" · ") + ); + } + if (pointers.length) out.push("\n" + POINTER(pointers.length)); + } + + const line = + `\n---\nEstimated ${selection.spent} tokens · target ${selection.target} · ` + + `mandatory floor ${selection.floor} · ${selection.status} · ` + + `${selection.excluded.length} items withheld · \`devia context --explain\` says why.`; + out.push(line); + return out.join("\n") + "\n"; +} + +/** + * The target and the mode for this run: the flag wins, then the config, then the default. + * + * `context.budget` is the name; `context.maxTokens` is still read so a project that adopted the + * first shipped spelling keeps working without editing anything. + */ +export function budgetFor(deviaDir, flags = {}) { + const config = readJSON(path.join(deviaDir, "devia.json"))?.context || {}; + + const number = (value) => { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : null; + }; + + const target = + number(flags.budget === true ? null : flags.budget) ?? + number(config.budget) ?? + number(config.maxTokens) ?? + DEFAULT_BUDGET; + + const asked = flags.strict ? "strict" : flags.mode ? String(flags.mode) : config.mode; + const mode = MODES.includes(asked) ? asked : "advisory"; + + return { target, mode }; +} diff --git a/src/lib/contribution.mjs b/src/lib/contribution.mjs new file mode 100644 index 0000000000000000000000000000000000000000..eb3653890de880af44f864b7371418e1ca4a973a GIT binary patch literal 26085 zcmb`Q>v9`Mmgo1gpCW}(OoM6)lpQ@26LiZAS(2wU;kLYzx_c)iu2BFAWHmt)1_hAf zQjFM#n0MGm*zfZw`y~7OpOcwY1wv}?#bH~8s=S{2EmPIys$OqG-GuqNz6^t9Ju8pv zX7KG-m48_koAd5B{qB5S*VD4;bzb?aw#{vruFGOm{-J2j!(Df*>H221sk<%9kLC1- zYEizuSx$G_o>z<6FM9?rKUU4AISkA(^Z)ZS)w4mt}OfE+ZYU7@l!CUbECY%>dpx3Lais;P$BoEIA= zoYt3@>^nGUX*Xj|^|ECyH8|9-DU(N$P2_G00z6(A`E`xUR#bJTI=Qdb@6z$d=wT<$6OG zcEqPrOA6u5%b(x8e(|U0KD5JST|Qp3QQH}th~qn89lMKqS%!J_akFL3+G|r>+ss9p zga$sr53l}9S~k=5CSYB~Ss{FNMkqh7%Bhyg2#5AXyFG(6L|$Q2Hk$}!AOGc#um1Vf z_dmXL$Y}Ra#H?ISOKrN@&gb9>kRRImR>jStE&y8tgv$bQEyDR0%o(nmvRneK2=y`a zv#w{`X*nCK!xAc3cZf)HIUd;l4M>~RTOhq@%EdhVumAq<5$@L_kTG4W`OP@g6R>bq z&JMMDIar*UtD04C0OY7K#Rf-TeFX|6;HLEwSik$%pI^QW zPs97KLKsZ8X9HFq2J>Rkl)o5zRaa&A-LgD`9Me0-*pwePzd-lvdAXUMd(&Ax-HH#4 zaAR-e9n|DMnsT~D7Txgc^xK^Uy?yva% zPY%PPTI%t4Po9K3z~jqG+mS3Veh-(`vbsbLPKyRsLeAI{S2vq ze+4EU?ycc>tYLBkE3%w#{t0^f`pGxHv!(Qw|KimfRz467tgFdZ3zOEc_5Q`h9mIQy z1laH&;9NjAQ7yU76LCi!)FWgmstvrlxvr*Vch&Rla%uqBu9^+haXCB)w-Bt;@^-yo zL2p^j@@y#OCtWv`dLI3?u9iGybqDg}uhCzddO0{aI2pBoIvHuGPoKhIaPVy&{*u9Y zKe`mEhev0KGXake(I1EOkBl~wS4%hW|fiM>?wK`WF&WtiXFjew4M0&=7*Tw}UvQ$!A z*2216+>F9&)T^clO?f6cDelEKKx-piY_C~TWMtcvJBV>0by>h*^E5)wNZM4K)_F40 zlp^P99)_EnRf(Li_79%I#_$ttGZ583(AxtBvE4fPd9U1DLyaZ~9=#cQdwSaUry6c# zBXxd{2KPq1N1!$;aI``9lA^yHL6HXlP0LJF6`ptG5P;oCvm$M>)if*T#daa<3k=Tc z6^b@opVyKd^Xd%UG|RTyS-ENUik&oUW{xiGeISIy9?{dX-oPG{)DgIE5%NidP3i9A zAXi$s+BSm)N7vq!=!p#0G_V_9Gu6!;kUo>wS7wSk5sjDu`u zV_JrRL@?fA$mu9P1`(W@7&10!@o}}lirXx1UR2G!#v-Rq<21F#OV7yGDD~TS2T^kR z6uvF>@uzMNf1~!TRyi3-A`9#ye~kf>_Q=Zxp4J;P)|_xH~XNqq|l^RO7<> z?W0popzjB<19Fq8hTqqamjG}tm_eni_u=re_{gGdDSwn);=kfE0x6AO` z@b%p|3zTQFq9I)r)e>uPg?z-B?AY)lti@zvqmPWMOevZFC)P7lUpC@3{Bc>pZstYo zl`Yox5BLe-M_RKdTI4PQ2N^UYm(;i$Lze=@`t0f$q?xL93)oE5laan;7Bj1Ne>)h^ zzIt1HX6)?!E&ZFflM2bL^uySs3)MEtF&s4K#W(-(kC?&Rm6Z0-wtFy|RcBbo!@+s^ zk#9~tn4e$&V=LGN6Z9+g9@EK5yK&_yRqa1ztYUFUGn<`{)i5g|WLzv^!RL@2ko0y1 zS`7(Vy=rs=zLeUbTjN~}oQsWN7!TUC+YyJ#+aDsXai?G%k;wbeXk-Tc>Bx8mc5qys zr|r{l)FAdYZl5;g>cEZE!4bx&Mhzi19{eY_Lp!g;yB*WkS8a?NPEL~D+G(GLxN9ZK z`L6)J>=suGt-*s+En-h_ZA~)(%)jOkUltc7`UuM3HYg(uq6QC*mk3io?p!~N&z17*sIgBbBhd>f`^Vm;uO zgw0;ro~$Y-8>1B{tSjW3i)w|47fvPQTY@=y|KY>Y>CwRm_jR$ImCev%l)qdO%)p1l z&bsR|VWwBcmR9WpM)wuNW_zVFC3__!y=HInFTsvEmgLMBF?| zEUu8)7fIxjz?+Q!f>2YG!t9qtizfOnvuCm&-wi>xsmqG#c{#m+SUWgRueyj&?^?~? z$NsOBEg~n$mH{YG^aAh?PLp79I+7TGjLeQUmp4s)DEB0a6{F2)Z>R&ylC4=Sx8;u9 zY5ROh6lDT)A!Ib{HfXazY&MGK z4PHeBRc!_y0-2Ny;Dnr_*YZy+2yFpui@17R(3F{Gkc*sRhhh_y8AIIn8B?hA`8ffY zFPIhgOb^jPl1ytDCwY{Lk;Is3+hkF6TcDpSVzLI{woS7s**;-LGiiNj3&%bU!z(2f z(0@odQ+rq8@BcnrjU0C}W$np61&gAnH3na?Jf<0cNL^d3TEF9CHr-uT=!*-vWYT+V zy4mQ9g{j$uNLBx&m`jF2!P2$d;k*JI7T8E|=thVpshL1R9FPB>jF1n6h+s<@<;l_- zu#wJ><9=J?8?k9|V4!J6t1Wh6k>5`QqTz!=yPOCF=28{*WUmCG_|9P48)w2ed25k- z5^8K#jc!v+Fg6*ezw{(>OpX*?fIpa|U#U%q-02$_$ZCiJ-=sCE7a4Pjz zuzRE;aZ*R5f6Q0Kd>Y^N7Pe8DZggwNrih&zkXiulD48odDmXVJ+{ z-Yw$sqP(%q4X=z9wFyDff3+Ju3GFjt?LFO2add4NzG)Y{uRZ0!w1eLhBWfM_BK$r~ zogSEWL=a6=?VZ-)miBk;Ul%)SkUbgnnV8JTGkF|Z9NrET2I ztSgc%B#`m+(^zIRSUYQF;>vS}8X3;O>nxTUzPwy*h-I4wSJno7u3`UeTb1%JKnCjM zQn58siu*OV4LOWcqw}`QE=%>9&40B(21z~*7{~)n8A=Vi7(UxGA!0bjteg;J0e2k-h5;MFtrDrSy=4bgW9sa z*uEXmg~|OM9Zf+721f!Bo3%p3kt-wzbPJORzh!w%uJr8X$JH=O$JpG|XCEkvdrM{$ znUr?MhR+PFgsxIOiQb~-levgMh>7ye`MSQAnGs{)MGE7=g!~g%bTmA9aBw8h-6e(b z7tMuqxU48wpnpsPC+|d`aDo+wx-J)s#{`Cy(CWnG_G*hb+8zV_tms(Hh7!xyx1N8D z`bn?CF=`$yExmCQ$-2d?`n`?`#@(*|uG2YAVphe;2obANjoxhbDSf%$6KYib4ng%T zWi)G)S!bp3WBT@m-pP30+dX~!LVu_Iy}kWY+c$36L6PF=xfmhI3Ipx4^}4tjRgL`_ zaX9Lr+{3U35+4uZu$`|Kqa7{E<=N&u4I8@P*=_4skfJbKR5vc`|CG+LZyQcequX0ss}K?5C>C9d zwMp_w5L*?lH@q_?Hz#D5SyCZajkN(Jou6 zVq${fx>A<-*xia;j1?Kj!kU_aE-^A@P1ygL6l~tp}pbq^#Dl7lR_F$ntCt-&2 zK=Xdn$Po5=-jkZ5+#P9}c`HiwZqbiqyQ@)5X~e!|Q1%Bf4-3FpwNjij>+;otK~KmQY2fThm<{I&P%xee&65t;<*q zrSY@oOSIdI8;#uKi||zZqMcws$Rw6nX`l|L(cLIM|z9ThK9aNbd_MP#Jw2LigAidVRm z%yae83WM0MMO6_tHC&rBl@@!l7AfH@0Qd~DXFcepWIk3M;iHdcMfm8B`l6)~ZFa^($6TS_Za?Ls z&m^j%Kz8~JKBr?eg!sYXPnr5)gX?P;Q;qA-ke|q!k-M@}5XQtf# zx!U|;I|*;oyw7}n72p6h!nnyETZI>^58}X3q9kPmSA;xzqpP42s5%+uO9fDOblgFP zf?NONvGdtan9bcZKQJn z>T#IxWwm}4^Din2Ma;~zs_e(bmSnH2FYE<4h}yUC$|Zl~0k;ki`7${S;|$**$v`zV z3)=;G1~|4`d+)f#z9T9qYp#n_;UpO`Ny)j-XGo5|lW!=mi5dqHABK$rJIzstd9?*MP29b`h{I8_58Sudk6|D+_j`+Kef~fZS)B> z4Bx$d_xQ<^uMZN*b}}h>1<@`@MxX53{4HD7zVw8^7@F%g?3*s_{?h2VkOBKepzF%@ zDA82wNulqKwJ4IG>y*WDczEj*$0`~!1qWI5?e5GBl0+pzvEDMqP2%Li+%-guswyAe zYRhJ_+_8H~EW%NIX;#Rc7wj9uksgi)#KYUZS#$STU8$W6zs-{{O{bX;_E>*#*kc`q zrZK5(&*{0WGl@yFfji1;90!cB+u3d328yiF+cY-1JRjhzLHpK}=f%x(IhQ9@c?xm8-n@g0!CL3i+)WpvXOyN=V=V)*#@#25vv_bWGb@GCGh6OGv21oeA0=O7Eod|%-yk{!f(*ponJTNX_0KMuC30d_ zveJlPGdk@lQ1DI`DgUaIl@wweTBy?A#4v;fJ$*n^atLFc^vUWne?oOqEa0kxVa&c1 z8QB@dZcLO`GJ`c$jDYNSyNS=gHec2<{Mo}IJ$OyUzv-IqD|WacW+B1!lrtnx_@{4Z zsxpyJ?@yCw&7!2rLVNOz;w!{mBv*>P!=7^?9A(d09?ZfcCB?iV6#yRNXT+Y-`3q$H34)fKbJFNTd zX8upvvX2zhHlxb*HZJcl{Pfy!Nw2R5;m%um;af@;3^~N$^F>mL>F1KF22P{lu*lqA zTJ^FDGV*)QJ!!wVAx6M@Z)m~NaB^bNMqKtqk9>()i*kh7sUvH87tarM;vw(}B*wXx zEzC+!I99CCb4#BS0xfj%(sG@tIK>Fh>@zc$O%aX4PbCHb&Y|TyjJ#TpCmE?oG?L-w z9*Y0pNwCDC6c3_Ipli_)yp42k&q1iNf{M&@j@2@S6-uOlQ^%x3FW`#CQcjHYN8u3y zxtu+cQ{@K}9+`v7TajrxX?ZVNNPsbBVKLoZ>M7u(RjCvT*KHAQ>Mf|Yx<43=_FiGA zld@rK(0M%{x3S|@;YD+d(d?IJkw>y1qoq@gGY4^fi;cq#o{QZpsH_RvfxaS~)b(XL z?E0n738HOx8G(6TQk+$j21@jt)LG=9h)r0bYlCM-l1}BDFsw2bIMuBfqbMUBun#B& zXc@yaydk4zk}(lTCsLe^dFnB20ntNrzdtQJM=?3|=ETMllHF}bF+0oRz5D2SY{^4& z5NPTys6(ZNM1nzQ_ykI-Lpp<}w;hsYvsR|;q?@NmLnhvr3%MVtKK$6S=?`xK9C~b4 z-cdLf`O(-Z?P>`l=tW*hL_y3_&c=dc76GI|`OeV>(+iCqomWi(WiXu+p9A9w>S7!C z2+JLMf>Q17EQBiCv$btp)A!T+mRHi^|K;+kTGvazNr2RyRC(tZt!I#2 zPRBF8RGKX68)P)3u>a&5Dm;xD&&>7!#U7N@<$zSa>R}ioya-jh@ZlKa#Gn4`jZ*`= zAS2rfaU_S$iujC@;O4oQd63@tcR3r!T(yC9z>(TcfYv1_!M1m=UjGD<3^{Ek?@9Z$ z`1knl{th3^-rjv!epo)d)h8Y~zmrOb-obKu3-6J(_X>X)X0@Gw^K;pLlPy7Yx3Y7( z(FMHe!N|m!87=cD?Gw)r_D)xu@=^&r*oxsXv?lVyBux*4I$x;fDFoH4DrWqnv;(VW za!shUr0WeZ^&ksvz^qGmhc2j)9B~SAH+^dH=xT{w=t@2gqHVAQhk!P3#-?U@nJ!o$ znCc=((cmaCJ(EoJQWBlL!iwxcXmqcxLQ2v-H%E5@Biv>?R~>s$`up-0WG=y)v)&|(Se1+PL4tf zer{A}KiAlB`06?t8jUenwwFrdI2i$f?@=i|kwS7H$`t;H0>%Y&U$W?EPQ-%8>w}_>cb?!lOrT?Htrj8&`a^a-D4JPm{?u zGZNgDmvJ$~Ywbz(&&KRuUi!=ET;)~!WV>@DvFlaU_s32U89ehED8!jgI!6SL@>*R$ zlV8t4{5X@x)+`$4rCujHf${z}?%@C|AV#u{k+&0~k`dZyGaipQROJ`!*c<*AQ#%<$ zouznPnc1@(B7sMISH(d(W@_53%9=t(n1hZtx$BQbgM)!Ug+@qg63_K-nj`dBPc;<(3-C!t`dWO?-}T+~*IkMkeZ(oVi4ry*&rzm3XlJlPQPe*C!iU+*(|qKi4veve+G5FLMl=o&-s9W(1$2dLyoo(%LxBCml= zJy@mp+og)R2op&Br3P7q?|kmkZgim>>#Q%5)&x8GBGFcFjVF{D5WUD-i7kzfC+_%s zN08c*d1f+;VwC)vknB3`4W%-m&iuccMZQo9*C+oZIpeH@MEV&iEy<4`=fcb|v-0CK=he zfuwJ4AKfwJdL$bRlIvz$0k@g>T?WuuO5IA+wSY~aW!q7yL)Gy&>G&D)dKZhm62T6k zK>O@ORdz||vWti)zQrWE9fa0_F_p%#m}S|)!#%c$76PF88`RzgRQ3`4C*;JZ?mn{I z>b;B{ATd;?`uSEA^_!;fiwP~uRaji+bIlc&iB0`63Ap~KDIgP@wmW4>CspHkJM<}viK`1Tq4aILp1~=)jmIPm zvu`OnGYf$GEJlNlK;KoVV+&~XxucCdnr_2zw<3d`M)od|QHMIR!Jc`k!)9X+>rgA2 zZ|ywpx(c~NgfzEAjH$=?FgCf>0Yi%{u<|){-ACAc^X|*cn$GPYiFWMLyq1B`gdk2= z%9`z}5Q5d0(2EOZh&H)y@@ENT)uj-t!dB8(FUrIbr~Ypg2Axum0ktkl>dpK*E&^x- zAnqxo@wZA`c)X0Mw?uWhG)VPRxa6V(tU9fU8*8hQTKka;&6a+n&nsxS*po$UCoT5- zvhNgDjlPakwA;y)0fWQv)Oa>&5IcgFZv{n`{-A3@gMo~&MFyp}OE-n)cIicK$91E8 z{61bfD#ny*?%gNK_vijDD9+yAF{;+RYes4LqMoLd6j2bra8$E*t{z3XypKFAclOk% z{*_hRe#>47%!mw3D8-zOq)bYriVvg6^8&Xd|3FKzQ1H{@HfpBPEfRBCV`bNKADl0` zT}b3Y;6|_qr^Kf7feH)OIsw?_m%_GUuNu-qOcovO1(T_>wA`kE$5ejjU_a1s;sO^_ zHtyYIHAV`1)3&{5k~*C$H21pe1~XmOWY-rRy&rv8avRP2C#NwCp%Eh;a(q&Bq%fA^ zXl7#AUEw z?#~jGE;gB^>vy3j_z+VqhS;zq?82|Ms4kZGKL7ERJT%k55+KCO{Vp&$U+>QCddGD4 z9kjrvv1ti5VFVUaiz+d(ZnEZ?qsbCOqLvs~>^3uUh@nCw6zvjgaLTbEk29+EcB$^R z)_ez^ha6*a!wz@a{f3s27@wVwW5ga>vQ(jLL#Rr_`?(8;P;e3tRMUBvyU+=W{dJWQsg2_zd_>2NX>4T{0*##t`tY!@t>?4+q_JfG$yXV zk6@P^W>b#eTml#J7pgS&oV17Bxuo>h2gV z>ClcXIcmeST{j5DkQa3lW zeXmM(%wk=9!?zSy>kMP_2pbGX$V`J6bXo~xm)#OnvDZnnctNR9u1f-19|=NGQy4pF zYB58#9Pa=i?v&%7oUB(gK-^w;dRN4;m~X)}HDwEGFt(EU%J#j#)qR(CgsE7s0(XO$$pN@KUa>F4&X zS}StE1V$s%+O`qlItNFr&BcEdb2|u&bm-#7j|)UB97D%x>zz^&OAuSslv#3;d(SZ@ ztDi>XQcZ&Gf&JkTt-aMEL{+M}?qZOEOXfj55?5w#JT7o+_wOqEr_>XdCs}wZ-btzH zoCz$j%c(4G>Mz$L{rb@kJL|$k^}{c>s=w&j_%)H#A)w%P?^;ZEC&r#%x;#>I-ziof z&m5Wq&}`RNs4x9>2KHA{Sj-*;y{=JuqD>SF7Fo;$Qd?IqANw61Ud5&+Zjy^`Ovz({ z@4f%U{~m%j{uf?!;mF#NN$yj4iuo<+PNVT&*3AIe!d3~q4mNs8K80dD!DoNHMglCw zF*ePpn9YXchqo57k(S`5Q?L_@_%)Zh+c;I^*ozLoO4@Zg}?*b|#G}GWsA6{zNHRRB%od{~ODP B>b(E} literal 0 HcmV?d00001 diff --git a/src/lib/gates.mjs b/src/lib/gates.mjs new file mode 100644 index 0000000..e1da16f --- /dev/null +++ b/src/lib/gates.mjs @@ -0,0 +1,43 @@ +/** + * The readiness gates, as data. + * + * Priority, the rule each gate enforces and its title live here rather than inside the closure + * that runs them, because two readers need them and only one of them has a repository to scan. + * `devia check` attaches the behaviour; `devia context` asks "is this rule machine-enforced, and + * does it block?" so it can cite a compact reference instead of spending the rule's full text on + * the agent (`AGT-013`). + * + * One table, two readers: that is what stops the two answers from drifting apart. + */ +export const GATES = [ + { id: "MEM-PRESENT", priority: "P0", rule: "AGT-002", title: "Project memory exists" }, + { id: "MEM-FILLED", priority: "P1", rule: "MEM-009", title: "Overview is filled in" }, + { id: "MEM-DEBT-P0", priority: "P0", rule: "MEM-002", title: "No P0 debt recorded as unbuilt" }, + { id: "MEM-WAIVERS", priority: "P1", rule: "GOVERNANCE", title: "No expired waiver" }, + { id: "CI-PRESENT", priority: "P0", rule: "OPS-001", title: "CI runs on pull requests" }, + { id: "CI-GATES", priority: "P1", rule: "OPS-001", title: "CI runs tests and static checks" }, + { id: "SEC-ENV", priority: "P0", rule: "SEC-002", title: "No environment file committed" }, + { id: "SEC-SECRETS", priority: "P0", rule: "SEC-002", title: "No secret-shaped strings in the tree" }, + { id: "OPS-BYPASS", priority: "P0", rule: "OPS-003", title: "No check bypass wired into the repository" }, + { id: "TST-PRESENT", priority: "P0", rule: "TST-001", title: "Automated tests exist" }, + { id: "TST-SKIPPED", priority: "P1", rule: "TST-003", title: "No disabled tests" }, + { id: "TST-SCRIPT", priority: "P1", rule: "TST-001", title: "A test command exists" }, + { id: "OPS-LOCKFILE", priority: "P1", rule: "OPS-004", title: "Dependency lockfile committed" }, + { id: "DB-MIGRATIONS", priority: "P1", rule: "DB-001", title: "Schema changes are versioned migrations" }, + { id: "AGT-CONTRACT", priority: "P1", rule: "AGT-001", title: "Agent contract at the repository root" }, + { id: "CTX-BUDGET", priority: "P2", rule: "AGT-013", title: "Context target holds the mandatory set" }, + { id: "DOC-README", priority: "P2", rule: "—", title: "README present" }, + { id: "UI-A11Y-TOOLING", priority: "P2", rule: "A11Y-001", title: "Accessibility tooling available" }, + { id: "OBS-ERRORS", priority: "P2", rule: "OBS-002", title: "Errors reach something a human watches" }, +]; + +/** Rule id -> the gates that enforce it. Derived, so it cannot disagree with `GATES`. */ +export function gatesByRule() { + const map = new Map(); + for (const g of GATES) { + if (g.rule === "—") continue; + if (!map.has(g.rule)) map.set(g.rule, []); + map.get(g.rule).push(g); + } + return map; +} diff --git a/src/lib/sanitize.mjs b/src/lib/sanitize.mjs new file mode 100644 index 0000000..157e775 --- /dev/null +++ b/src/lib/sanitize.mjs @@ -0,0 +1,141 @@ +import os from "node:os"; +import path from "node:path"; + +/** + * What may leave the machine. + * + * A contribution to devia is built from inside a user's private repository. That repository + * stays the user's: everything entering a contribution payload passes through here first, and + * `residue()` re-reads the finished payload and refuses it when anything survived. + * + * Redaction is reported, never silent. A payload cleaned without saying so is a payload nobody + * can review, and review is the last gate before an upload (`PRIV-005`). + */ + +/** + * Secret shapes. This list is the single source of truth: `devia check` scans a repository with + * it and the sanitizer scans an outbound payload with it, so a pattern added for one is + * immediately true for the other. + * + * It is a coarse pattern match, not entropy analysis (`12_DEBT.md` D5). A clean result means + * these shapes were not found — never that nothing sensitive is present, which is why + * `contribute` also puts the whole payload in front of a human before anything is sent. + */ +export const SECRET_PATTERNS = [ + [/AKIA[0-9A-Z]{16}/, "AWS access key id"], + [/-----BEGIN (RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/, "private key"], + [/sk_live_[0-9a-zA-Z]{16,}/, "live secret key"], + [/gh[pousr]_[0-9A-Za-z]{30,}/, "GitHub token"], + [/xox[baprs]-[0-9A-Za-z-]{10,}/, "Slack token"], + [/AIza[0-9A-Za-z_-]{35}/, "Google API key"], + [/eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/, "JWT"], +]; + +const REDACTED = "[redacted]"; + +const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; + +/** + * `API_TOKEN = "..."` in the shapes a config file and a source file actually use. + * + * The sensitive word has to be a whole segment of the key, not a substring of it: matching "key" + * anywhere would redact `monkey: banana` and shred the text it was meant to protect. So the + * snake pattern requires an underscore boundary or the bare word, and the camel pattern requires + * a capital — which `monkey` does not have and `accessToken` does. + * + * The placeholder is excluded from the value. Without that, sanitizing an already-sanitized file + * matches its own output and reports a redaction that protected nothing, inflating the count + * every time the text passes through. + */ +const VALUE = `(?!\\[redacted\\]|"\\[redacted\\]"|'\\[redacted\\]')("[^"\\n]*"|'[^'\\n]*'|[^\\s#,;)]+)`; +const SEP = "(\\s*[=:]\\s*)"; + +const CREDENTIALS = [ + new RegExp( + `\\b((?:[A-Za-z][A-Za-z0-9]*_)*(?:token|secret|password|passwd|api_?key|credentials?))${SEP}${VALUE}`, + "gi" + ), + new RegExp( + `\\b([a-z][A-Za-z0-9]*(?:Token|Secret|Password|Passwd|ApiKey|Key|Credentials?))${SEP}${VALUE}`, + "g" + ), +]; + +/** Keep the quoting the file used, so a redacted fixture still parses as what it was. */ +function redactValue(_, key, sep, value) { + const quote = value[0] === '"' || value[0] === "'" ? value[0] : ""; + return `${key}${sep}${quote}${REDACTED}${quote}`; +} + +function escapeRe(s) { + return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** A path, matched with either separator: the same home directory is written both ways. */ +function pathRe(dir) { + if (!dir) return null; + const source = escapeRe(dir).replace(/\\\\|\//g, "[\\\\/]"); + return new RegExp(source, "gi"); +} + +function accountRe(home) { + const account = home ? path.basename(home) : ""; + // Two characters is a word, not an account name. Redacting it would shred the prose. + if (account.length < 3) return null; + return new RegExp(`\\b${escapeRe(account)}\\b`, "gi"); +} + +/** + * Redact a string. Returns the cleaned text and what was removed, counted by kind, so a report + * can say "3 absolute paths, 1 email address" instead of "sanitized: true". + */ +export function sanitize(text, { root = null } = {}) { + let out = String(text ?? ""); + const removed = new Map(); + const note = (kind, n) => { + if (n) removed.set(kind, (removed.get(kind) || 0) + n); + }; + + const replace = (re, kind, replacer) => { + if (!re) return; + const hits = out.match(re); + if (!hits) return; + note(kind, hits.length); + out = out.replace(re, replacer || REDACTED); + }; + + // Secrets first: a token inside a URL must not be reduced to a hostname and then shipped. + for (const [re, label] of SECRET_PATTERNS) { + replace(new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g"), label); + } + + for (const re of CREDENTIALS) replace(re, "credential assignment", redactValue); + + // The repository root becomes a stable placeholder rather than a path naming the project. + if (root) replace(pathRe(path.resolve(root)), "repository path", ""); + + const home = os.homedir(); + replace(EMAIL, "email address"); + replace(pathRe(home), "home directory"); + replace(accountRe(home), "account name"); + replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, "ip address"); + + return { text: out, removed: [...removed].map(([kind, count]) => ({ kind, count })) }; +} + +/** + * What survived. Run against a finished payload: a non-empty result blocks the upload rather + * than warning about it, because a warning on the last screen before a network call is a + * warning that gets accepted. + */ +export function residue(text) { + const s = String(text ?? ""); + const found = []; + for (const [re, label] of SECRET_PATTERNS) { + if (re.test(s)) found.push(label); + } + const home = pathRe(os.homedir()); + if (home && home.test(s)) found.push("home directory"); + if (new RegExp(EMAIL.source).test(s)) found.push("email address"); + return found; +} diff --git a/src/lib/tokens.mjs b/src/lib/tokens.mjs new file mode 100644 index 0000000..2ec1efb --- /dev/null +++ b/src/lib/tokens.mjs @@ -0,0 +1,75 @@ +/** + * Token estimation, without a tokenizer. + * + * `ARC-004` rules out shipping a BPE vocabulary, and a context budget still has to be counted + * against something. So this is an *estimate* — deterministic, stable across runs, and reported + * as an estimate everywhere it surfaces. It is never presented as an exact token count, because + * a number that looks exact and is not teaches people to trust it. + * + * The model: a word costs roughly one token per 4.5 characters, a punctuation run about one per + * two characters, and a line break one. Measured against the ratios published for English prose + * and for Markdown with tables and fenced code, it lands within roughly ±20%. That is accurate + * enough to decide what fits in a budget and to report a reduction, and not accurate enough to + * quote as a bill. + */ + +const WORD = /[A-Za-z0-9_'’-]+/y; +const PUNCT = /[^A-Za-z0-9_'’\-\s]+/y; + +/** Estimated tokens for a string. Deterministic: the same text always costs the same. */ +export function estimateTokens(text) { + const s = String(text ?? ""); + let i = 0; + let total = 0; + + while (i < s.length) { + const c = s[i]; + + if (c === "\n") { + total += 1; + i++; + continue; + } + if (c === " " || c === "\t" || c === "\r") { + i++; + continue; + } + + WORD.lastIndex = i; + const word = WORD.exec(s); + if (word) { + total += Math.max(1, Math.round(word[0].length / 4.5)); + i = WORD.lastIndex; + continue; + } + + PUNCT.lastIndex = i; + const punct = PUNCT.exec(s); + if (punct) { + total += Math.ceil(punct[0].length / 2); + i = PUNCT.lastIndex; + continue; + } + + // A character neither pattern claimed (an emoji, a CJK glyph): one token, never zero. + total += 1; + i++; + } + + return total; +} + +/** Estimated tokens for several strings at once. */ +export function estimateAll(parts) { + return parts.reduce((n, p) => n + estimateTokens(p), 0); +} + +/** + * `before -> after` as a percentage saved, rounded to one decimal. Returns 0 when there was + * nothing to save, rather than a division by zero dressed as an improvement. + */ +export function reduction(before, after) { + if (!before || before <= 0) return 0; + const pct = ((before - after) / before) * 100; + return Math.round(pct * 10) / 10; +} diff --git a/templates/agents/AGENTS.md b/templates/agents/AGENTS.md index 1140ed5..1fa0657 100644 --- a/templates/agents/AGENTS.md +++ b/templates/agents/AGENTS.md @@ -13,6 +13,12 @@ This repository uses **devia**: a standard plus a living project memory in `.dev If `.devia/` is missing, run `npm i -D @schneiderjoseph/devia && npx devia init`, then fill `00_OVERVIEW.md` before writing code. +Rather than reading the whole standard, ask for the part this task needs: + +```bash +npx devia context "" # --explain says why each item is there +``` + ## While working - Never invent endpoints, fields, config keys or business rules. Unknown means ask, or record it @@ -33,3 +39,7 @@ Update `.devia/` in the same change (see `.devia/impact-map.yaml`), then report Rules by ID: `npx devia rules --id SEC-001`, or by domain: `npx devia rules --domain database`. A pinned copy lives under `.devia/standard/` only if this project ran `devia sync`. + +Found a problem in devia itself rather than in this project? `npx devia contribute` prepares an +issue or a pull request from evidence, without exposing this repository, and sends nothing +without `--yes`. diff --git a/templates/agents/CLAUDE.md b/templates/agents/CLAUDE.md index 690f107..aff6e47 100644 --- a/templates/agents/CLAUDE.md +++ b/templates/agents/CLAUDE.md @@ -33,3 +33,13 @@ Report the checks that ran, the rule IDs involved, and what you did **not** veri Rules by ID: `npx devia rules --id SEC-001`, or by domain: `npx devia rules --domain database`. A pinned copy lives under `.devia/standard/` only if this project ran `devia sync`. + +For the rules that apply to the task in front of you, rather than all of them: + +```bash +npx devia context "" # --explain says why each item is there +``` + +Found a problem in devia itself rather than in this project? `npx devia contribute` prepares an +issue or a pull request from evidence, without exposing this repository, and sends nothing +without `--yes`. diff --git a/templates/agents/copilot-instructions.md b/templates/agents/copilot-instructions.md index 87926c1..8cc47a1 100644 --- a/templates/agents/copilot-instructions.md +++ b/templates/agents/copilot-instructions.md @@ -13,3 +13,4 @@ This repository runs on devia. The project memory is `.devia/`. complete states — including empty, loading and error. - Suggest the matching `.devia/` update alongside the code change. - Never suggest disabling a test, skipping a hook, or loosening a check to make CI pass. +- `npx devia context ""` returns the rules that apply to one task instead of all of them. diff --git a/templates/agents/cursor.mdc b/templates/agents/cursor.mdc index 6f08a25..8d68508 100644 --- a/templates/agents/cursor.mdc +++ b/templates/agents/cursor.mdc @@ -25,3 +25,7 @@ This repository runs on devia. `.devia/` is the project memory and it is not opt 8. Update `.devia/` in the same change, per `.devia/impact-map.yaml` (MEM-009). 9. Cite rule IDs, and always report what you did **not** verify (AGT-006, AGT-007). 10. Never claim done or production ready without naming the checks that ran (AGT-005). +11. For the rules that apply to the task rather than all of them, run + `npx devia context ""`; `--explain` says why each item was included (AGT-013). +12. A problem in devia itself goes through `npx devia contribute` — evidence and a reproduction + first, this repository never uploaded, nothing sent without `--yes` (AGT-012, PRIV-005). diff --git a/templates/agents/windsurfrules.md b/templates/agents/windsurfrules.md index 22ea914..7f0d52d 100644 --- a/templates/agents/windsurfrules.md +++ b/templates/agents/windsurfrules.md @@ -15,3 +15,6 @@ This repository runs on devia. `.devia/` is the project memory. 7. Never disable a test, bypass a hook, or weaken a rule to go green. 8. Update `.devia/` in the same change (`.devia/impact-map.yaml`). 9. Report the checks you ran, the rule IDs, and what you did not verify. +10. `npx devia context ""` gives the rules for one task instead of the whole standard. +11. A problem in devia itself goes through `npx devia contribute`: reproduce it first, and + nothing leaves this repository without `--yes`. diff --git a/templates/project/AGENTS.md b/templates/project/AGENTS.md index f67498b..96c48cf 100644 --- a/templates/project/AGENTS.md +++ b/templates/project/AGENTS.md @@ -29,8 +29,9 @@ Then work. Then update this memory in the same change. ## Checks ```bash -npx devia validate # memory integrity -npx devia check # readiness gates (P0 blocks) +npx devia context "" # the rules that apply here, not all of them +npx devia validate # memory integrity +npx devia check # readiness gates (P0 blocks) ``` TODO(devia): add this project's own commands — install, dev, test, lint, migrate. diff --git a/templates/project/README.md b/templates/project/README.md index 92f4471..fce931c 100644 --- a/templates/project/README.md +++ b/templates/project/README.md @@ -24,7 +24,7 @@ Created by `devia init` (devia {{DEVIA_VERSION}}, {{DATE}}). | 13 | [`13_RECIPES.md`](13_RECIPES.md) | How do I do this routine task here? | | 14 | [`14_INDEX.md`](14_INDEX.md) | Where do I find X? | -Machine files: [`devia.json`](devia.json) (profile, maturity, pinned version) and +Machine files: [`devia.json`](devia.json) (profile, maturity, pinned version, context budget) and [`impact-map.yaml`](impact-map.yaml) (change type → files to update). The standard itself is not copied in here. Read it with `npx devia rules --id SEC-001` or @@ -33,6 +33,34 @@ disk — an agent with no network, or an audit that must show the exact wording — `npx devia sync` pins a version-locked copy under `standard/`, and `14_INDEX.md` then points at it. +## Reading this is not reading all of it + +Everything above plus the rule registry is more than one task needs. Ask for the slice: + +```bash +npx devia context "add POST /api/orders" +npx devia context "fix the empty state" --files src/components/Orders.tsx --explain +``` + +It returns this project's own never/always lines, the blocking rules for the surfaces involved +and the impact-map duty first, then whatever else fits the target in `devia.json` +(`context.budget`). + +Three numbers are always reported, because they are three different things: + +```text +Target 1200 what you asked for +Mandatory floor 962 what the blocking items cost +Selected 1187 what you got +``` + +`context.mode` decides what happens when the floor is larger than the target. `advisory` (the +default) delivers the mandatory items whole and says `OVER TARGET`; `strict` never exceeds the +target and compresses them toward their identifiers instead, never dropping one. + +Keeping `10_NEVER_ALWAYS.md` pruned matters here: every line is admitted before the target is +consulted, so a line nobody has ever violated costs every task that runs after it. + ## The two registries | File | Means | Never | @@ -50,3 +78,13 @@ npx devia sync # pin the standard under standard/, or refresh a pinned co ``` `.devia/` is committed. It is part of the repository, not a local scratch pad. + +## If devia itself is what went wrong + +`npx devia contribute` turns a devia problem you hit here into an issue or a pull request. It +runs locally, builds a standalone reproduction rather than sending this repository, sanitizes +anything you explicitly include, prints every byte before sending, and sends nothing without +`--yes`. A candidate is only eligible once devia has reproduced the problem itself. + +Records live in `contributions/`; the generated `payload/` is disposable and gitignored. Turn +the feature off entirely with `"contribution": { "enabled": false }` in `devia.json`. diff --git a/tests/cli.test.mjs b/tests/cli.test.mjs index ff8e593..9cbe169 100644 --- a/tests/cli.test.mjs +++ b/tests/cli.test.mjs @@ -445,6 +445,38 @@ test("init does not pin the standard, and the memory still resolves without it", } }); +// The template tells adopters that `payload/` is disposable and gitignored. Saying so without +// writing it is the kind of false statement this repository treats as a defect, so `init` +// writes a nested ignore file inside the directory devia owns rather than editing the project's. +test("init writes the ignore file the memory's README promises", () => { + const dir = scratch(); + try { + devia(["init", "--root", dir, "--no-vendor"], dir); + const ignore = fs.readFileSync(path.join(dir, ".devia", ".gitignore"), "utf8"); + assert.match(ignore, /^reader\.html$/m); + assert.match(ignore, /^contributions\/\*\/payload\/$/m); + // The project's own .gitignore is never touched: devia owns .devia/, not the root file. + assert.ok(!fs.existsSync(path.join(dir, ".gitignore"))); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("init writes the context budget and leaves the contribution identity unset", () => { + const dir = scratch(); + try { + devia(["init", "--root", dir, "--no-vendor"], dir); + const config = JSON.parse(fs.readFileSync(path.join(dir, ".devia", "devia.json"), "utf8")); + assert.ok(config.context.budget > 0, "the target must be visible, not folklore"); + assert.equal(config.context.mode, "advisory", "the default mode never exceeds a promise"); + assert.equal(config.contribution.enabled, true); + // No identity means nothing can be published, whatever else is configured. + assert.equal(config.contribution.identity, undefined); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test("devia.json records the CLI version, not the standard version", () => { const dir = scratch(); try { diff --git a/tests/context.test.mjs b/tests/context.test.mjs new file mode 100644 index 0000000..36a03d5 --- /dev/null +++ b/tests/context.test.mjs @@ -0,0 +1,426 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { packageRoot } from "../src/lib/fs.mjs"; +import { + buildCorpus, + classify, + select, + render, + routeDomains, + terms, + budgetFor, + DEFAULT_BUDGET, +} from "../src/lib/context.mjs"; + +const bin = path.join(packageRoot, "bin", "devia.mjs"); +const root = packageRoot; +const deviaDir = path.join(root, ".devia"); + +const { FORCE_COLOR, ...cleanEnv } = process.env; + +function devia(args, cwd, { allowFailure = false } = {}) { + const options = { cwd, encoding: "utf8", env: { ...cleanEnv, NO_COLOR: "1" } }; + try { + return { code: 0, out: execFileSync(process.execPath, [bin, ...args], options) }; + } catch (e) { + if (!allowFailure) throw e; + return { code: e.status ?? 1, out: e.stdout || "" }; + } +} + +function run(task, { files = [], target = 4000, domains = [], mode = "advisory", changeTypes = [] } = {}) { + const corpus = buildCorpus({ root, deviaDir }); + const routed = classify(corpus, { task, files, domains, changeTypes }); + return { ...select(corpus, { target, mode }), routed, corpus }; +} + +const ids = (selection) => new Set(selection.included.map((i) => i.id)); + +test("task words route to the domains that own them", () => { + const d = routeDomains("add a migration for the orders table", []); + assert.ok(d.has("database")); + assert.ok(!d.has("accessibility"), "a schema change is not a screen"); +}); + +test("a changed path routes even when the words do not", () => { + const d = routeDomains("tidy this up", ["src/components/Button.tsx"]); + assert.ok(d.has("ui")); + assert.ok(d.has("accessibility")); +}); + +// Routing on the words a task happens to use is not routing on what the task is. A write +// endpoint is an authorization surface whether or not anyone typed "authorization". +test("an endpoint implies the authorization surface it is", () => { + const d = routeDomains("add POST /api/orders", ["src/api/orders.ts"]); + assert.ok(d.has("api")); + assert.ok(d.has("security"), "api must imply security"); + assert.ok( + [...d.get("security")].some((why) => /implied by api/.test(why)), + "and must say why" + ); +}); + +test("plural and singular meet", () => { + const t = terms("add the orders table"); + assert.ok(t.has("orders") && t.has("order")); +}); + +test("irrelevant domains are excluded, with a reason", () => { + const s = run("add a migration for the orders table", { files: ["migrations/1.sql"] }); + const excluded = s.excluded.find((i) => i.id === "A11Y-006"); + assert.ok(excluded, "a screen rule has no business in a schema change"); + assert.equal(excluded.reason, "not relevant"); + assert.match(excluded.why[0], /domain accessibility is not in scope/); +}); + +test("every selected item can say why it is there", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"] }); + for (const item of s.included) { + assert.ok(item.why.length > 0, `${item.id} was selected with no reason`); + } +}); + +// The whole safety constraint, stated as a test: a budget may cut, and it may never cut this. +test("a blocking rule survives a budget far too small to hold it", () => { + const tiny = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 1 }); + assert.equal(tiny.overrun, true, "an impossible budget must be reported as an overrun"); + for (const id of ["SEC-001", "SEC-003", "AGT-004", "MEM-009"]) { + assert.ok(ids(tiny).has(id), `${id} must survive a budget of 1`); + } + assert.ok(tiny.spent > 1, "the overrun is real, not a rounding of the report"); +}); + +test("the blocking set is identical at every budget", () => { + const task = "add POST /api/orders"; + const files = ["src/api/orders.ts"]; + const blocking = (budget) => + [...ids(run(task, { files, target: budget }))] + .filter((id) => /^[A-Z]+-\d+$/.test(id)) + .sort(); + const small = run(task, { files, target: 1 }).included.filter((i) => i.tier === "T0"); + const large = run(task, { files, target: 100000 }).included.filter((i) => i.tier === "T0"); + assert.deepEqual( + small.map((i) => i.id).sort(), + large.map((i) => i.id).sort(), + "the budget must not decide what blocks" + ); + assert.ok(blocking(100000).length > blocking(1).length, "a larger budget still adds more"); +}); + +test("a larger budget adds items and never removes one", () => { + const task = "build the empty state for the orders screen"; + const files = ["src/components/Orders.tsx"]; + const small = ids(run(task, { files, target: 1500 })); + const large = ids(run(task, { files, target: 8000 })); + for (const id of small) { + assert.ok(large.has(id), `${id} was in the small selection and vanished from the large one`); + } + assert.ok(large.size > small.size); +}); + +test("the project's own never/always lines are never budget-evicted", () => { + const tiny = run("anything at all", { target: 1 }); + const lines = [...ids(tiny)].filter((id) => id.startsWith("10_NEVER_ALWAYS.md#")); + assert.ok(lines.length > 5, "this repository has earned more than five of them"); + assert.deepEqual( + tiny.excluded.filter((i) => i.id.startsWith("10_NEVER_ALWAYS.md#")), + [], + "a line this project earned is not a candidate for eviction" + ); +}); + +// --- budget semantics ------------------------------------------------------------------------ +// +// Three numbers, kept apart. Reporting only the target and the selection made a stated design +// ("a mandatory item is never evicted") read as a broken promise. + +test("target, mandatory floor and selected are reported separately", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 200 }); + assert.equal(s.target, 200); + assert.ok(s.floor > s.target, "this corpus cannot hold its mandatory items in 200 tokens"); + assert.equal(s.spent, s.floor, "nothing optional fits once the floor is over the target"); + assert.equal(s.status, "over"); + assert.equal(s.compliant, false, "advisory does not pretend to comply"); +}); + +test("advisory keeps every mandatory item whole, whatever the target", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 1, mode: "advisory" }); + assert.equal(s.status, "over"); + assert.ok(s.included.every((i) => i.level === "full"), "advisory never degrades a form"); + assert.ok(ids(s).has("SEC-001")); +}); + +// The promise the word "strict" makes. It is the one the first version quietly broke. +test("strict never exceeds its target", () => { + for (const target of [300, 600, 1200, 2400, 6000]) { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target, mode: "strict" }); + assert.ok( + s.spent <= target, + `strict selected ${s.spent} against a target of ${target} (${s.status})` + ); + assert.equal(s.compliant, true); + } +}); + +test("strict compresses a mandatory item rather than dropping it", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 600, mode: "strict" }); + assert.equal(s.status, "degraded"); + // Still present, still named — the floor compresses, it does not disappear. + for (const id of ["SEC-001", "SEC-003", "AGT-004", "MEM-009"]) { + assert.ok(ids(s).has(id), `${id} must survive compression`); + } + assert.ok(s.degraded.length, "what was given up is reported"); + assert.ok(s.included.some((i) => i.level !== "full"), "something actually shrank"); +}); + +// Degrading everything and then spending the freed tokens on optional rules at full text is +// precisely backwards, and is what the first implementation did. +test("strict degrades only as far as it has to", () => { + const tight = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 600, mode: "strict" }); + const loose = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 2400, mode: "strict" }); + const shrunk = (s) => s.included.filter((i) => i.level !== "full").length; + assert.ok(shrunk(loose) < shrunk(tight), "a larger target must give more text back"); + + const ample = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 100000, mode: "strict" }); + assert.equal(ample.status, "within"); + assert.equal(shrunk(ample), 0, "an ample target degrades nothing at all"); +}); + +test("a target too small even for identifiers is impossible, not exceeded", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 20, mode: "strict" }); + assert.equal(s.status, "impossible"); + assert.ok(s.floor > 20, "and it says what the smallest possible floor costs"); +}); + +test("a compressed rule still tells the agent where to read it", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 400, mode: "strict" }); + const text = render(s, { task: "add POST /api/orders" }); + assert.match(text, /devia rules --id/, "a reference must say how to expand it"); + assert.match(text, /mandatory floor/, "the three numbers travel with the context"); +}); + +// --- impact-map routing (AGT-013, D12) --------------------------------------------------------- +// +// The impact map is the one routing table the project wrote, in the project's own vocabulary. + +test("a change type routes the domains of the memory files it declares", () => { + // Nothing in this sentence is a security keyword. The project's own impact map says a + // permission change updates 04_PERMISSIONS.md, and that file speaks for security. + const s = run("record a permission_change for the maintainer", { target: 100000 }); + const why = [...s.routed.domains.get("security") || []]; + assert.ok(why.some((w) => /permission_change updates 04_PERMISSIONS\.md/.test(w)), why.join("; ")); + assert.ok(ids(s).has("SEC-001")); +}); + +test("the memory files a change type names are promoted, and say why", () => { + const s = run("add a new endpoint", { target: 100000 }); + const section = s.included.find((i) => i.kind === "memory" && i.id.startsWith("02_SURFACES.md#")); + assert.ok(section, "the file the impact map names must be delivered"); + assert.ok(section.why.some((w) => /impact map says this change updates 02_SURFACES\.md/.test(w))); +}); + +test("--type declares the change explicitly", () => { + const s = run("tidy things up", { target: 100000, changeTypes: ["permission_change"] }); + assert.ok(s.routed.changeTypes.includes("permission_change")); + const impact = s.included.find((i) => i.kind === "impact" && i.id === "permission_change"); + assert.ok(impact.why.some((w) => /--type/.test(w))); +}); + +test("one shared word does not make a task a change type", () => { + // `state_machine_change` shares "state" with this task and nothing else. + const s = run("fix the empty state on the orders screen", { target: 100000 }); + assert.ok(!s.routed.changeTypes.includes("state_machine_change"), s.routed.changeTypes.join(", ")); +}); + +// AGT-013: what devia verifies itself is cited; what only a human can check is stated in full. +test("a rule a devia gate blocks on is cited, not recited", () => { + const s = run("initialise the memory", { target: 100000 }); + const compact = s.corpus.find((i) => i.id === "AGT-002"); + assert.equal(compact.compacted, true); + assert.match(compact.text, /devia check/); + assert.ok(compact.tokens < compact.rawTokens, "citing must actually cost less"); +}); + +test("a P0 rule whose only gate warns keeps its full text", () => { + // MEM-009 is P0; MEM-FILLED, the gate that touches it, is P1. Compacting it would swap a + // blocking obligation for a gate that lets the change through. + const s = run("change the architecture", { target: 100000 }); + const rule = s.corpus.find((i) => i.id === "MEM-009"); + assert.equal(rule.compacted, false); + assert.equal(rule.text, rule.full); +}); + +test("the impact map duty for the change type is included", () => { + const s = run("add a new endpoint to the CLI", { target: 100000 }); + const impact = s.included.find((i) => i.kind === "impact" && i.id === "new_endpoint"); + assert.ok(impact, "a task that is a known change type carries its MEM-009 duty"); + assert.match(impact.text, /02_SURFACES\.md/); + assert.equal(impact.tier, "T0"); +}); + +test("an unrelated change type is not included", () => { + const s = run("add a new endpoint to the CLI", { target: 100000 }); + assert.ok(!ids(s).has("design_token")); +}); + +test("the selection is smaller than the corpus it came from", () => { + const s = run("fix a typo in the readme", { target: DEFAULT_BUDGET }); + assert.ok(s.spent < s.raw / 4, `${s.spent} of ${s.raw} is not a reduction worth the name`); +}); + +test("the rendered context carries every selected item", () => { + const s = run("add POST /api/orders", { files: ["src/api/orders.ts"], target: 4000 }); + const text = render(s, { task: "add POST /api/orders" }); + for (const item of s.included.slice(0, 20)) { + assert.ok(text.includes(item.text.split("\n")[0]), `${item.id} is missing from the render`); + } +}); + +test("--domain forces a domain the words did not reach", () => { + const s = run("tidy this up", { target: 100000, domains: ["database"] }); + assert.ok(ids(s).has("DB-001")); + const item = s.included.find((i) => i.id === "DB-001"); + assert.ok(item.why.some((w) => /--domain/.test(w))); +}); + +test("the target and the mode come from devia.json, and degrade to the defaults", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "devia-budget-")); + const cfg = path.join(dir, "devia.json"); + const write = (context) => fs.writeFileSync(cfg, JSON.stringify({ context })); + try { + assert.deepEqual(budgetFor(dir, {}), { target: DEFAULT_BUDGET, mode: "advisory" }); + + write({ budget: 777, mode: "strict" }); + assert.deepEqual(budgetFor(dir, {}), { target: 777, mode: "strict" }); + assert.equal(budgetFor(dir, { budget: "350" }).target, 350, "a flag beats the config"); + assert.equal(budgetFor(dir, { mode: "advisory" }).mode, "advisory"); + assert.equal(budgetFor(dir, { strict: true }).mode, "strict"); + + // The first shipped spelling still works: an adopter who wrote it keeps working untouched. + write({ maxTokens: 999 }); + assert.equal(budgetFor(dir, {}).target, 999, "maxTokens is still honoured"); + + write({ budget: "nonsense", mode: "whatever" }); + assert.deepEqual( + budgetFor(dir, {}), + { target: DEFAULT_BUDGET, mode: "advisory" }, + "a bad value degrades, never throws" + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --- through the binary ------------------------------------------------------------------- + +test("devia context reports its accounting in json", () => { + const report = JSON.parse(devia(["context", "add POST /api/orders", "--json", "--root", root], root).out); + assert.equal(report.ok, true); + assert.ok(report.context.raw_tokens > report.context.selected_tokens); + assert.ok(report.context.reduction_pct > 50); + assert.ok(report.domains.some((d) => d.domain === "security")); + assert.ok(report.included.every((i) => i.why.length > 0)); +}); + +test("devia context --full is the baseline the reduction is measured against", () => { + const full = JSON.parse(devia(["context", "--full", "--json", "--root", root], root).out); + const optimised = JSON.parse(devia(["context", "add POST /api/orders", "--json", "--root", root], root).out); + assert.equal(full.tokens, optimised.context.raw_tokens, "both must report the same corpus"); + assert.ok(full.text.length > 10000); +}); + +test("devia context refuses a repository with no memory", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "devia-nomem-")); + try { + const res = devia(["context", "anything", "--root", dir], dir, { allowFailure: true }); + assert.equal(res.code, 1); + assert.match(res.out, /no \.devia/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("devia context --explain names what it withheld and why", () => { + const out = devia(["context", "fix a typo", "--budget", "700", "--explain", "--root", root], root).out; + assert.match(out, /Included/); + assert.match(out, /Not relevant/); + assert.match(out, /Target|Mandatory floor/); +}); + +// G11: a target nobody revisits quietly becomes a permanent overrun, so `check` reports it. +test("devia check reports whether the target can hold the mandatory set", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "devia-ctxgate-")); + try { + fs.writeFileSync(path.join(dir, "package.json"), JSON.stringify({ name: "app" })); + devia(["init", "--root", dir, "--no-vendor"], dir); + const cfg = path.join(dir, ".devia", "devia.json"); + const of = () => { + const out = devia(["check", "--root", dir, "--json"], dir, { allowFailure: true }).out; + return JSON.parse(out).results.find((r) => r.id === "CTX-BUDGET"); + }; + + const ample = JSON.parse(fs.readFileSync(cfg, "utf8")); + ample.context = { budget: 100000 }; + fs.writeFileSync(cfg, JSON.stringify(ample, null, 2)); + assert.equal(of().kind, "PASS"); + assert.match(of().detail, /baseline floor \d+ of 100000/); + + // A target no task could ever fit is a finding, not a silent overrun. + const tiny = { ...ample, context: { budget: 50 } }; + fs.writeFileSync(cfg, JSON.stringify(tiny, null, 2)); + const warned = of(); + assert.equal(warned.kind, "WARN"); + assert.match(warned.detail, /every task starts at \d+ tokens, above the 50 target/); + + // Strict mode answers the same question differently: it compresses rather than overruns. + const strict = { ...ample, context: { budget: 600, mode: "strict" } }; + fs.writeFileSync(cfg, JSON.stringify(strict, null, 2)); + assert.equal(of().kind, "PASS"); + assert.match(of().detail, /strict: compresses to fit 600/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the benchmark passes: the saving never costs a blocking rule", () => { + const out = execFileSync( + process.execPath, + [path.join(packageRoot, "scripts", "benchmark-context.mjs"), "--json"], + { cwd: root, encoding: "utf8", env: { ...cleanEnv, NO_COLOR: "1" } } + ); + const report = JSON.parse(out); + assert.equal(report.ok, true); + for (const row of report.rows) { + assert.deepEqual(row.missed, [], `${row.scenario} @ ${row.target} lost a blocking rule`); + assert.deepEqual(row.misrouted, [], `${row.scenario} @ ${row.target} misrouted a rule`); + // A generous target against a small corpus has little to cut, and pretending otherwise + // would be a claim about the tool rather than a measurement. The saving is only asserted + // where the target actually binds. + assert.ok(row.reduction_pct >= 0, "a selection is never larger than the corpus it came from"); + if (row.target < row.raw_tokens / 2) { + assert.ok( + row.reduction_pct > 50, + `${row.scenario} @ ${row.target} (${row.mode}) cut only ${row.reduction_pct}%` + ); + } + if (row.mode === "strict") { + assert.ok( + row.selected_tokens <= row.target, + `strict must never exceed: ${row.selected_tokens} > ${row.target}` + ); + } + // The only reason a run may exceed its target is the mandatory floor. An over-target run + // that also carries optional context would mean the target stopped meaning anything. + if (row.selected_tokens > row.target) { + assert.equal( + row.selected_tokens, + row.mandatory_floor, + `${row.scenario} @ ${row.target} went over target carrying optional items` + ); + } + } +}); diff --git a/tests/contribute.test.mjs b/tests/contribute.test.mjs new file mode 100644 index 0000000..64e8dc4 --- /dev/null +++ b/tests/contribute.test.mjs @@ -0,0 +1,780 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { packageRoot } from "../src/lib/fs.mjs"; +import { + holds, + verdict, + stateOf, + claimHash, + eligibility, + missing, + pluck, + identityCheck, + fixtureHash, + sourceHash, + evidenceChain, + reproPath, + upstream, + REPRO_CAPS, +} from "../src/lib/contribution.mjs"; + +const bin = path.join(packageRoot, "bin", "devia.mjs"); +const { FORCE_COLOR, ...cleanEnv } = process.env; + +function devia(args, cwd, { allowFailure = true, env = {} } = {}) { + const options = { cwd, encoding: "utf8", env: { ...cleanEnv, NO_COLOR: "1", ...env } }; + try { + return { code: 0, out: execFileSync(process.execPath, [bin, ...args], options), err: "" }; + } catch (e) { + if (!allowFailure) throw e; + return { code: e.status ?? 1, out: e.stdout || "", err: e.stderr || "" }; + } +} + +/** A repository with a memory, a secret and a private-looking name. */ +function scratch() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "devia-contrib-")); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ name: "acme-internal-billing", version: "1.0.0" }, null, 2) + ); + fs.mkdirSync(path.join(dir, "src"), { recursive: true }); + fs.writeFileSync( + path.join(dir, "src", "config.js"), + `export const awsKey = "AKIA${"IOSFODNN7EXAMPLE"}";\n` + + `export const owner = "alice@acme-internal.example";\n` + + `export const API_TOKEN = "live-token-value";\n` + ); + fs.writeFileSync(path.join(dir, ".env"), "SECRET_TOKEN=hunter2\n"); + devia(["init", "--root", dir, "--no-vendor"], dir, { allowFailure: false }); + return dir; +} + +/** + * The assertion is on the `blocking` array, not on the gate id. + * + * A gate id appears in `--json` whether the gate passed or failed, so `matches: "SEC-SECRETS"` + * would hold on a clean repository too and confirm itself. The blocking list carries only the + * P0 gates that actually failed, so it distinguishes the two behaviours — which is the whole + * job of the pair. + */ +const FAILED = '"blocking": \\[[^\\]]*"SEC-SECRETS"'; + +const OBSERVE = [ + "contribute", "new", + "--type", "false_positive", + "--gate", "SEC-SECRETS", + "--rule", "SEC-002", + "--summary", "SEC-SECRETS reports a fixture file as a live credential", + "--expected", "a fixture is not reported as a secret", + "--actual", "the gate fails and names the fixture", + "--argv", "check --json", + "--actual-matches", FAILED, + "--expect-absent", FAILED, +]; + +const record = (dir, id = "C1") => + JSON.parse(fs.readFileSync(path.join(dir, ".devia", "contributions", id, "record.json"), "utf8")); + +// --- the evidence model ----------------------------------------------------------------------- + +test("a profile that states nothing never holds", () => { + assert.equal(holds({}, { code: 0, out: "" }), false); + assert.equal(holds(null, { code: 0, out: "" }), false); +}); + +test("every stated field must hold, and an unstated one is not an opinion", () => { + const run = { code: 1, out: "SEC-SECRETS failed" }; + assert.equal(holds({ exit: 1 }, run), true); + assert.equal(holds({ exit: 1, matches: "SEC-SECRETS" }, run), true); + assert.equal(holds({ exit: 1, matches: "TST-PRESENT" }, run), false); + assert.equal(holds({ exit: 0, matches: "SEC-SECRETS" }, run), false); + assert.equal(holds({ absent: "TST-PRESENT" }, run), true); +}); + +test("a numeric claim reads a dotted path out of the json body", () => { + const run = { code: 0, out: JSON.stringify({ context: { selected_tokens: 3900 } }) }; + assert.equal(holds({ metric: "context.selected_tokens", op: ">", value: 1000 }, run), true); + assert.equal(holds({ metric: "context.selected_tokens", op: "<=", value: 1000 }, run), false); + assert.equal(holds({ metric: "context.nope", op: ">", value: 1 }, run), false); + assert.equal(pluck({ a: { b: 2 } }, "a.b"), 2); + assert.equal(pluck({ a: null }, "a.b.c"), undefined); +}); + +test("a claim that is not json cannot confirm a numeric assertion", () => { + const run = { code: 0, out: "not json at all" }; + assert.equal(holds({ metric: "context.selected_tokens", op: ">", value: 1 }, run), false); +}); + +test("a pattern that does not compile matches nothing instead of throwing", () => { + assert.equal(holds({ matches: "([unclosed" }, { code: 0, out: "anything" }), false); +}); + +test("neither profile matching is reported, never rounded to one of them", () => { + const rec = { + observation: { argv: ["check"], actual: { exit: 1 }, expected: { exit: 0 } }, + }; + assert.equal(verdict(rec, { code: 1, out: "" }).outcome, "reproduced"); + // "expected", not "fixed": one run cannot tell a fix from a fixture that never failed. + assert.equal(verdict(rec, { code: 0, out: "" }).outcome, "expected"); + assert.equal(verdict(rec, { code: 2, out: "" }).outcome, "inconclusive"); +}); + +test("two profiles that describe the same behaviour are inconclusive, not a reproduction", () => { + const rec = { + observation: { argv: ["check"], actual: { matches: "x" }, expected: { matches: "x" } }, + }; + assert.equal(verdict(rec, { code: 0, out: "x" }).outcome, "inconclusive"); +}); + +// --- the loop, through the binary ------------------------------------------------------------- + +test("a speculative improvement is not eligible, and says so", () => { + const dir = scratch(); + try { + const res = devia( + ["contribute", "new", "--root", dir, "--type", "feature", + "--summary", "devia could support monorepo profiles", + "--expected", "a profile per workspace", "--actual", "one profile per repository"], + dir + ); + assert.match(res.out, /incomplete|recorded/); + // No observation, so it cannot reach `reproduced`, so it cannot be submitted. + const submit = devia(["contribute", "submit", "C1", "--root", dir], dir); + assert.equal(submit.code, 1); + assert.match(submit.out, /missing observation\.argv|not reproduced/); + assert.match(submit.out, /Not eligible/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a manual proposal is allowed, and is an issue rather than a pull request", () => { + const dir = scratch(); + try { + devia( + ["contribute", "new", "--root", dir, "--manual", "--type", "feature", + "--summary", "a profile per workspace would help monorepos", + "--expected", "one profile per workspace", "--actual", "one profile per repository"], + dir + ); + const show = JSON.parse(devia(["contribute", "show", "C1", "--root", dir, "--json"], dir).out); + assert.equal(show.record.source, "manual"); + assert.equal(show.eligibility.route, "issue"); + assert.ok(show.eligibility.notes.some((n) => /never to a pull request/.test(n))); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a candidate is reproduced because devia reproduced it, not because it says so", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + assert.equal(stateOf(record(dir)), "observed"); + + // Verification is refused until there is somewhere to run it. + const early = devia(["contribute", "verify", "C1", "--root", dir], dir); + assert.equal(early.code, 1); + assert.match(early.out, /no reproduction/); + + devia(["contribute", "repro", "C1", "--root", dir], dir); + // The scaffold alone contains no secret, so the expected behaviour is what happens. That is + // not a fix — devia has never seen this fixture do anything else — so it is rejected. + const clean = devia(["contribute", "verify", "C1", "--root", dir], dir); + assert.equal(clean.code, 1); + assert.match(clean.out, /rejected/); + assert.match(clean.out, /Make the\s+fixture fail first/); + assert.equal(stateOf(record(dir)), "rejected"); + + // Put the reported behaviour into the fixture, and it reproduces. + fs.writeFileSync( + path.join(dir, ".devia", "contributions", "C1", "repro", "fixture.js"), + `const example = "AKIA${"IOSFODNN7EXAMPLE"}";\n` + ); + const found = devia(["contribute", "verify", "C1", "--root", dir], dir); + assert.equal(found.code, 0); + assert.match(found.out, /reproduced/); + assert.equal(stateOf(record(dir)), "reproduced"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** + * A devia checkout that differs from the installed one. + * + * `fixed` requires the two runs to disagree about the devia source, so proving it needs a real + * second devia — not a fixture edited until it passes. This copies what `sourceHash` hashes plus + * what the process needs to start, and patches one file so the behaviour genuinely changes. + */ +function deviaCopy(patch) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "devia-fork-")); + const copy = (rel) => { + const from = path.join(packageRoot, rel); + if (!fs.existsSync(from)) return; + fs.cpSync(from, path.join(dir, rel), { recursive: true }); + }; + for (const rel of ["bin", "src", "package.json", "VERSION"]) copy(rel); + const target = path.join(dir, patch.file); + fs.writeFileSync(target, patch.edit(fs.readFileSync(target, "utf8"))); + return dir; +} + +// `fixed` is two observations that have to be the same experiment: devia saw the problem, then +// devia saw it gone, with the same fixture and a different devia. +test("fixed requires the same fixture and a genuinely different devia", () => { + const dir = scratch(); + let fork = null; + try { + readyCandidate(dir); + assert.equal(stateOf(record(dir)), "reproduced"); + + // The "fix": a devia whose secret scanner no longer carries the pattern the fixture trips. + fork = deviaCopy({ + file: path.join("src", "lib", "sanitize.mjs"), + edit: (src) => src.replace('[/AKIA[0-9A-Z]{16}/, "AWS access key id"],', ""), + }); + assert.notEqual(sourceHash(fork), sourceHash(packageRoot), "the fork must really differ"); + + const after = devia(["contribute", "verify", "C1", "--root", dir, "--devia", fork], dir); + assert.equal(after.code, 0); + assert.equal(stateOf(record(dir)), "fixed"); + + const chain = evidenceChain(record(dir)); + assert.deepEqual(chain.breaks, [], "same fixture, different devia: nothing to object to"); + assert.equal(chain.steps.length, 2); + assert.equal(chain.steps[0].fixture, chain.steps[1].fixture, "the experiment did not change"); + assert.notEqual(chain.steps[0].devia, chain.steps[1].devia, "devia did"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + if (fork) fs.rmSync(fork, { recursive: true, force: true }); + } +}); + +// --- the evidence chain (D10) ----------------------------------------------------------------- +// +// "It used to fail and now it passes" only means something if both runs used the same fixture +// and a different devia. Without those two digests, editing the fixture reads exactly like +// fixing the tool. + +test("each run is bound to the fixture and the devia source that produced it", () => { + const dir = scratch(); + try { + readyCandidate(dir); + const v = record(dir).verification; + assert.match(v.fixture, /^[0-9a-f]{16}$/, "the fixture is hashed"); + assert.match(v.devia, /^[0-9a-f]{16}$/, "so is the devia that ran it"); + assert.equal(record(dir).reproduced.fixture, v.fixture); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a hash changes when the thing it hashes changes, and not otherwise", () => { + const dir = scratch(); + try { + readyCandidate(dir); + const repro = path.join(dir, ".devia", "contributions", "C1", "repro"); + const before = fixtureHash(repro); + assert.equal(fixtureHash(repro), before, "the same tree always hashes the same"); + + fs.writeFileSync(path.join(repro, "extra.js"), "// one more file\n"); + assert.notEqual(fixtureHash(repro), before, "an added file must change it"); + + assert.equal(sourceHash(packageRoot), sourceHash(packageRoot)); + assert.equal(fixtureHash(path.join(dir, "nowhere")), null); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// The whole point. Editing the fixture until it passes is not a fix, and the record says so +// instead of recording a fix that never happened. +test("a fixture edited between the two runs is not a fix", () => { + const dir = scratch(); + const fixture = path.join(dir, ".devia", "contributions", "C1", "repro", "fixture.js"); + try { + readyCandidate(dir); + assert.equal(stateOf(record(dir)), "reproduced"); + + // The behaviour changes, but because the fixture changed — not because devia did. + fs.writeFileSync(fixture, "const example = 1;\n"); + const after = devia(["contribute", "verify", "C1", "--root", dir], dir); + + assert.equal(stateOf(record(dir)), "reproduced", "it must not be promoted to fixed"); + assert.match(after.out, /evidence chain/); + assert.match(after.out, /the reproduction changed between the two runs/); + + const chain = evidenceChain(record(dir)); + assert.ok(chain.breaks.length, "the break is stated, not inferred by the reader"); + + // And it cannot be submitted as a pull request on that basis. + const submit = devia(["contribute", "submit", "C1", "--root", dir], dir); + assert.match(submit.out, /route\s+issue/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the same fixture and the same devia twice is not a fix either", () => { + const chain = (over) => { + const base = { + id: "C1", + type: "bug", + source: "real_usage", + observation: { argv: ["check"], expected: { exit: 0 }, actual: { exit: 1 } }, + }; + const claim = claimHash(base); + return evidenceChain({ + ...base, + reproduced: { claim, fixture: "aaaa", devia: "1111", ran: "2026-09-12" }, + verification: { claim, outcome: "expected", ran: "2026-09-13", ...over }, + }); + }; + + assert.ok( + chain({ fixture: "aaaa", devia: "1111" }).breaks.some((b) => /same devia source/.test(b)), + "nothing in devia changed between the two runs" + ); + assert.ok( + chain({ fixture: "bbbb", devia: "2222" }).breaks.some((b) => /reproduction changed/.test(b)), + "the experiment was not the same experiment" + ); + // The one shape that is a fix: same fixture, different devia. + assert.deepEqual(chain({ fixture: "aaaa", devia: "2222" }).breaks, []); +}); + +test("a record written before hashing existed is re-verified, not trusted", () => { + const base = { + id: "C1", + type: "bug", + source: "real_usage", + observation: { argv: ["check"], expected: { exit: 0 }, actual: { exit: 1 } }, + problem: { summary: "s", expected: "e", actual: "a" }, + devia: { version: "0.7.0" }, + }; + const claim = claimHash(base); + const legacy = { + ...base, + reproduced: { claim, ran: "2026-09-12" }, + verification: { claim, outcome: "expected", ran: "2026-09-13" }, + }; + assert.equal(stateOf(legacy), "reproduced", "an unbound pair is not promoted to fixed"); + assert.ok(evidenceChain(legacy).breaks.some((b) => /predates fixture hashing/.test(b))); +}); + +test("the published report carries the chain a maintainer can check", () => { + const dir = scratch(); + try { + readyCandidate(dir); + const show = JSON.parse(devia(["contribute", "show", "C1", "--root", dir, "--json"], dir).out); + assert.match(show.body, /## Evidence chain/); + assert.match(show.body, /\| reproduced \|/); + assert.match(show.body, /Fixture \| devia source/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// A verdict belongs to the claim it was made about. Editing the claim afterwards and keeping the +// verdict is exactly how an unreproduced problem would reach a maintainer. +test("editing the claim after verification drops the state back to observed", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir], dir); + fs.writeFileSync( + path.join(dir, ".devia", "contributions", "C1", "repro", "fixture.js"), + `const example = "AKIA${"IOSFODNN7EXAMPLE"}";\n` + ); + devia(["contribute", "verify", "C1", "--root", dir], dir); + assert.equal(stateOf(record(dir)), "reproduced"); + + const file = path.join(dir, ".devia", "contributions", "C1", "record.json"); + const edited = JSON.parse(fs.readFileSync(file, "utf8")); + edited.observation.actual.matches = "SOMETHING-ELSE"; + fs.writeFileSync(file, JSON.stringify(edited, null, 2)); + + assert.equal(stateOf(edited), "observed", "a stale verdict must not survive its claim"); + assert.notEqual(claimHash(edited), edited.verification.claim); + const submit = devia(["contribute", "submit", "C1", "--root", dir], dir); + assert.equal(submit.code, 1); + assert.match(submit.out, /not reproduced/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --- the privacy boundary --------------------------------------------------------------------- + +test("an environment file is refused outright, not sanitized", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + const res = devia(["contribute", "repro", "C1", "--root", dir, "--include", ".env"], dir); + assert.equal(res.code, 1); + assert.match(res.out, /an environment file is never copied/); + assert.ok( + !fs.existsSync(path.join(dir, ".devia", "contributions", "C1", "repro", ".env")), + "nothing from an environment file may reach the fixture" + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("an included file is sanitized on the way in, and the redactions are recorded", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir, "--include", "src/config.js"], dir); + + const copied = fs.readFileSync( + path.join(dir, ".devia", "contributions", "C1", "repro", "config.js"), + "utf8" + ); + assert.ok(!copied.includes("AKIA"), "a key must not reach the fixture"); + assert.ok(!copied.includes("acme-internal.example"), "an address must not reach the fixture"); + assert.ok(!copied.includes("live-token-value"), "a token must not reach the fixture"); + + const kinds = record(dir).reproduction.included[0].redactions.map((r) => r.kind); + assert.ok(kinds.includes("AWS access key id")); + assert.ok(kinds.includes("email address")); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("nothing sensitive survives anywhere in a prepared payload", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir, "--include", "src/config.js"], dir); + devia(["contribute", "submit", "C1", "--root", dir], dir); + + const payload = path.join(dir, ".devia", "contributions", "C1", "payload"); + const seen = []; + const walk = (d) => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) walk(p); + else seen.push(fs.readFileSync(p, "utf8")); + } + }; + walk(payload); + const all = seen.join("\n"); + for (const forbidden of [ + "AKIA", + "acme-internal.example", + "live-token-value", + "hunter2", + path.basename(dir), + os.homedir(), + ]) { + assert.ok(!all.includes(forbidden), `"${forbidden}" leaked into the payload`); + } + assert.ok(all.includes("MANIFEST") || fs.existsSync(path.join(payload, "MANIFEST.md"))); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the manifest names every byte that would be sent", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir], dir); + const show = JSON.parse(devia(["contribute", "show", "C1", "--root", dir, "--json"], dir).out); + assert.ok(show.payload.length >= 2); + for (const f of show.payload) { + assert.ok(f.bytes > 0 && f.sha256.length === 16, `${f.name} is not accounted for`); + } + assert.deepEqual(show.residue, [], "a prepared payload must be clean"); + assert.match(show.body, /What is not in this report/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the report states what was actually redacted, not that something was", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir, "--include", "src/config.js"], dir); + const show = JSON.parse(devia(["contribute", "show", "C1", "--root", dir, "--json"], dir).out); + assert.match(show.body, /- sanitized: yes — .*AWS access key id/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --- the remote boundary ---------------------------------------------------------------------- + +test("nothing is sent without --yes, and the command is shown instead", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir], dir); + fs.writeFileSync( + path.join(dir, ".devia", "contributions", "C1", "repro", "fixture.js"), + `const example = "AKIA${"IOSFODNN7EXAMPLE"}";\n` + ); + devia(["contribute", "verify", "C1", "--root", dir], dir); + + const res = devia(["contribute", "submit", "C1", "--root", dir], dir); + assert.equal(res.code, 0); + assert.match(res.out, /gh issue create --repo/); + assert.match(res.out, /--yes authorises the remote operation/); + assert.match(res.out, /devia holds no GitHub token/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function readyCandidate(dir) { + devia([...OBSERVE, "--root", dir], dir); + devia(["contribute", "repro", "C1", "--root", dir], dir); + fs.writeFileSync( + path.join(dir, ".devia", "contributions", "C1", "repro", "fixture.js"), + `const example = "AKIA${"IOSFODNN7EXAMPLE"}";\n` + ); + devia(["contribute", "verify", "C1", "--root", dir], dir); +} + +function configure(dir, contribution) { + const file = path.join(dir, ".devia", "devia.json"); + const config = JSON.parse(fs.readFileSync(file, "utf8")); + config.contribution = contribution; + fs.writeFileSync(file, JSON.stringify(config, null, 2)); +} + +test("--yes without a declared identity sends nothing", () => { + const dir = scratch(); + try { + readyCandidate(dir); + const res = devia(["contribute", "submit", "C1", "--root", dir, "--yes"], dir); + assert.equal(res.code, 1); + assert.match(res.out, /no contribution identity/); + assert.match(res.out, /Nothing was sent/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// The maintainer account is the one identity a contribution may never wear. +test("the maintainer account is refused as a contribution identity", () => { + const dir = scratch(); + try { + readyCandidate(dir); + configure(dir, { enabled: true, identity: upstream().owner }); + const res = devia(["contribute", "submit", "C1", "--root", dir, "--yes"], dir); + assert.equal(res.code, 1); + assert.match(res.out, /maintainer account/); + assert.match(res.out, /Nothing was sent/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("gh being absent or authenticated as someone else stops the upload", () => { + const dir = scratch(); + try { + readyCandidate(dir); + configure(dir, { enabled: true, identity: "some-contributor-account" }); + // PATH is emptied so `gh` cannot resolve: devia must report that, never work around it. + const res = devia(["contribute", "submit", "C1", "--root", dir, "--yes"], dir, { + env: { PATH: path.join(dir, "nothing-here"), Path: path.join(dir, "nothing-here") }, + }); + assert.equal(res.code, 1); + assert.match(res.out, /gh is not installed|gh is not authenticated|authenticated as/); + assert.match(res.out, /Nothing was sent/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// A record is a file a person can edit. "../../.." would turn a reproduction into a directory +// read somewhere else entirely, so the name is validated rather than trusted. +test("a hand-edited reproduction path cannot escape the candidate's directory", () => { + const deviaDir = path.join(packageRoot, ".devia"); + const base = { id: "C1", reproduction: { path: "repro" } }; + assert.ok(reproPath(deviaDir, base).endsWith(path.join("C1", "repro"))); + + for (const bad of ["../../..", "../../../etc", "a/b", ".", "", null, 42]) { + assert.equal( + reproPath(deviaDir, { id: "C1", reproduction: { path: bad } }), + null, + `${JSON.stringify(bad)} must not resolve to a directory` + ); + } +}); + +test("a file too large for the whole fixture is refused, not truncated", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + fs.writeFileSync(path.join(dir, "huge.txt"), "x".repeat(REPRO_CAPS.bytes + 1)); + const res = devia(["contribute", "repro", "C1", "--root", dir, "--include", "huge.txt"], dir); + assert.equal(res.code, 1); + assert.match(res.out, /too large/); + assert.ok(!fs.existsSync(path.join(dir, ".devia", "contributions", "C1", "repro", "huge.txt"))); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// The identity decision is a pure function so every branch is reachable without a GitHub CLI +// standing in front of it — including the one that matters most, an account devia could not read. +test("publishing is refused by default and allowed only on an exact match", () => { + const owner = "maintainer-account"; + const as = (account) => () => ({ ok: true, account }); + + const cases = [ + [{ identity: null, lookup: as("anyone") }, /no contribution identity/], + [{ identity: owner, lookup: as(owner) }, /maintainer account/], + [{ identity: "me", lookup: () => ({ ok: false, missing: true }) }, /not installed/], + [{ identity: "me", lookup: () => ({ ok: false }) }, /not authenticated/], + // gh answered, but devia could not tell who it answered as. Proceeding here would publish + // under whoever gh happens to be. + [{ identity: "me", lookup: as(null) }, /could not read which account/], + [{ identity: "me", lookup: as("someone-else") }, /authenticated as someone-else/], + ]; + for (const [input, expected] of cases) { + const res = identityCheck({ owner, ...input }); + assert.equal(res.ok, false, `${JSON.stringify(input.identity)} must be refused`); + assert.match(res.why, expected); + } + + const allowed = identityCheck({ identity: "Me", owner, lookup: as("me") }); + assert.equal(allowed.ok, true, "a case-insensitive exact match is the only way through"); +}); + +test("a repository with no declared identity never asks GitHub who is logged in", () => { + let asked = false; + identityCheck({ + identity: null, + owner: "maintainer-account", + lookup: () => { + asked = true; + return { ok: true, account: "someone" }; + }, + }); + assert.equal(asked, false, "the local half is settled before anything runs `gh`"); +}); + +test("a security defect is routed to the private path, never to an issue", () => { + const dir = scratch(); + try { + devia( + ["contribute", "new", "--root", dir, "--type", "security", + "--summary", "the scanner can be made to skip a file", + "--expected", "every file is scanned", "--actual", "a crafted path is skipped", + "--argv", "check --json", "--actual-exit", "0", "--expect-exit", "1"], + dir + ); + const res = devia(["contribute", "submit", "C1", "--root", dir, "--yes"], dir); + assert.equal(res.code, 1); + assert.match(res.out, /reported privately/); + assert.match(res.out, /SECURITY\.md/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the feature can be turned off entirely", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + configure(dir, { enabled: false }); + const res = devia(["contribute", "repro", "C1", "--root", dir], dir); + assert.equal(res.code, 0); + assert.match(res.out, /disabled for this repository/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --- routing -------------------------------------------------------------------------------- + +test("a fix without a regression test is an issue, and with one is a pull request", () => { + const deviaDir = path.join(packageRoot, ".devia"); + const base = { + id: "C1", + type: "bug", + source: "real_usage", + devia: { version: "0.6.0" }, + problem: { summary: "s", expected: "e", actual: "a" }, + observation: { argv: ["check"], expected: { exit: 0 }, actual: { exit: 1 } }, + reproduction: { path: "repro" }, + }; + // The pair that makes a record `fixed`: the same fixture, a different devia, and a behaviour + // that changed between them. + base.reproduced = { claim: claimHash(base), fixture: "aaaa", devia: "1111" }; + base.verification = { outcome: "expected", claim: claimHash(base), fixture: "aaaa", devia: "2222" }; + + const noTests = eligibility({ ...base, fix: { repo: packageRoot, tests: [] } }, { deviaDir }); + assert.equal(noTests.route, "issue"); + assert.ok(noTests.notes.some((n) => /regression test/.test(n))); + + const withTests = eligibility( + { ...base, fix: { repo: packageRoot, tests: ["tests/cli.test.mjs"], files: 2, lines: 30 } }, + { deviaDir } + ); + assert.equal(withTests.route, "pull_request"); + + const tooBig = eligibility( + { ...base, fix: { repo: packageRoot, tests: ["tests/cli.test.mjs"], files: 40, lines: 4000 } }, + { deviaDir } + ); + assert.equal(tooBig.route, "issue", "a large change is agreed before it is patched"); + + const named = eligibility( + { ...base, fix: { repo: packageRoot, tests: ["tests/does-not-exist.test.mjs"] } }, + { deviaDir } + ); + assert.ok(named.blockers.some((b) => /not found/.test(b)), "a named test must exist"); +}); + +test("an incomplete record names every field it is missing", () => { + const gaps = missing({ type: "bug", source: "real_usage", devia: {}, problem: {} }); + for (const field of ["devia.version", "problem.summary", "observation.argv"]) { + assert.ok(gaps.includes(field), `${field} should be reported as missing`); + } +}); + +test("ids are monotone and are not reissued after a removal", () => { + const dir = scratch(); + try { + devia([...OBSERVE, "--root", dir], dir); + devia([...OBSERVE, "--root", dir], dir); + assert.ok(fs.existsSync(path.join(dir, ".devia", "contributions", "C2"))); + devia(["contribute", "rm", "C2", "--root", dir], dir); + devia([...OBSERVE, "--root", dir], dir); + assert.ok( + fs.existsSync(path.join(dir, ".devia", "contributions", "C2")), + "C2 is the next free number once C2 is gone — the registry is per repository" + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("--argv may not smuggle a --root past the fixture", () => { + const dir = scratch(); + try { + const res = devia( + ["contribute", "new", "--root", dir, "--type", "bug", "--summary", "s", + "--expected", "e", "--actual", "a", "--argv", "check --root /somewhere/else"], + dir + ); + assert.equal(res.code, 2); + assert.match(res.out, /may not carry --root/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/sanitize.test.mjs b/tests/sanitize.test.mjs new file mode 100644 index 0000000..30df4ea --- /dev/null +++ b/tests/sanitize.test.mjs @@ -0,0 +1,84 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import os from "node:os"; +import path from "node:path"; +import { sanitize, residue, SECRET_PATTERNS } from "../src/lib/sanitize.mjs"; + +// Built from parts so this repository does not itself carry a secret-shaped string: what the +// sanitizer has to remove is what the test constructs, not what the file contains. +const AWS = "AKIA" + "IOSFODNN7EXAMPLE"; + +test("secret shapes are removed and reported by kind", () => { + const { text, removed } = sanitize(`const key = "${AWS}";`); + assert.ok(!text.includes(AWS)); + assert.deepEqual(removed, [{ kind: "AWS access key id", count: 1 }]); +}); + +test("the repository path becomes a placeholder, not a project name", () => { + const root = path.join(os.tmpdir(), "acme-internal-billing"); + const { text } = sanitize(`failed at ${path.join(root, "src", "a.js")}`, { root }); + assert.ok(!text.includes("acme-internal-billing")); + assert.match(text, //); +}); + +test("the home directory and the account name in it are both removed", () => { + const home = os.homedir(); + const account = path.basename(home); + const { text } = sanitize(`wrote ${path.join(home, "notes.txt")} for ${account}`); + assert.ok(!text.includes(home), "the home directory must not survive"); + if (account.length >= 3) { + assert.ok(!new RegExp(`\\b${account}\\b`, "i").test(text), "the account name must not survive"); + } +}); + +test("an email address never reaches a payload", () => { + const { text, removed } = sanitize("reported by alice.smith@acme-internal.example"); + assert.ok(!text.includes("acme-internal.example")); + assert.ok(removed.some((r) => r.kind === "email address")); +}); + +// A fixture has to keep working after it is cleaned: a redaction that breaks the file's syntax +// destroys the reproduction it was protecting. +test("a redacted assignment keeps the quoting the file used", () => { + assert.match(sanitize('const STRIPE_SECRET = "sk_test_abc";').text, /= "\[redacted\]";/); + assert.match(sanitize("API_TOKEN=abc123").text, /^API_TOKEN=\[redacted\]$/); + assert.match(sanitize("password: 'hunter2'").text, /'\[redacted\]'/); + assert.match(sanitize("const accessToken = 'abc'").text, /'\[redacted\]'/); +}); + +// The sensitive word must be a whole segment of the key. Matching "key" anywhere redacts +// `monkey: banana`, which destroys the text the sanitizer exists to preserve. +test("a word that merely contains a sensitive word is left alone", () => { + for (const line of ["monkey: banana", "donkey = 3", "keyboard: qwerty", "tokenizer = split"]) { + assert.equal(sanitize(line).text, line, `${line} must survive untouched`); + } +}); + +// Redaction is reported, and the report is what a human reviews. Re-matching the placeholder +// would inflate that count every time the text passed through, describing protection that did +// not happen. +test("sanitizing twice changes nothing and reports nothing the second time", () => { + const once = sanitize(`API_TOKEN = "abc123"\nkey = "${AWS}"\nmail a@b.example`); + const twice = sanitize(once.text); + assert.equal(twice.text, once.text); + assert.deepEqual(twice.removed, []); +}); + +test("residue finds what a payload must never carry, and is empty once cleaned", () => { + assert.deepEqual(residue(`key=${AWS}`), ["AWS access key id"]); + assert.deepEqual(residue(sanitize(`key=${AWS}`).text), []); + assert.ok(residue(`at ${os.homedir()}`).includes("home directory")); +}); + +test("the secret pattern list is shared, not copied", async () => { + const check = await import("../src/commands/check.mjs"); + assert.ok(SECRET_PATTERNS.length >= 7); + // check.mjs imports the list rather than keeping its own: one addition, two readers. + assert.ok(check.default, "check must still load with the shared list"); +}); + +test("a short account name is not scrubbed out of ordinary prose", () => { + // Two letters is a word, not an identity: redacting it would shred the text it protects. + const { text } = sanitize("go to the db and read the id"); + assert.equal(text, "go to the db and read the id"); +}); diff --git a/tests/tokens.test.mjs b/tests/tokens.test.mjs new file mode 100644 index 0000000..1a0dbef --- /dev/null +++ b/tests/tokens.test.mjs @@ -0,0 +1,53 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { estimateTokens, estimateAll, reduction } from "../src/lib/tokens.mjs"; + +test("an estimate is deterministic", () => { + const text = "SEC-001 · MUST · P0 — Server-side authorization on every sensitive operation"; + assert.equal(estimateTokens(text), estimateTokens(text)); +}); + +test("cost grows with the text and is never zero for non-empty input", () => { + assert.equal(estimateTokens(""), 0); + assert.ok(estimateTokens("a") >= 1); + assert.ok(estimateTokens("one two three") > estimateTokens("one two")); +}); + +// The estimate exists to decide what fits in a budget, so it has to stay in the same +// neighbourhood as the ratios published for English. Claiming precision it does not have is the +// failure mode here, not being a few percent off. +test("prose lands near the accepted characters-per-token ratio", () => { + const prose = + "An agent that starts cold re-derives the project on every task: it invents APIs, " + + "re-litigates decisions that were already made, and reports done because the unit tests " + + "passed. Documentation does not fix that."; + const naive = prose.length / 4; + const estimate = estimateTokens(prose); + const drift = Math.abs(estimate - naive) / naive; + assert.ok(drift < 0.25, `estimate ${estimate} drifts ${Math.round(drift * 100)}% from ${naive}`); +}); + +test("markdown structure costs more per character than prose", () => { + const row = "| SEC-001 | Server-side authorization | MUST | P0 | security | yes | active |"; + const prose = "Server side authorization applies to every sensitive operation here now"; + assert.ok( + estimateTokens(row) / row.length > estimateTokens(prose) / prose.length, + "a table row is denser in punctuation and must not be costed as prose" + ); +}); + +test("a newline costs a token, so structure is not free", () => { + assert.equal(estimateTokens("a\nb") - estimateTokens("a b"), 1); +}); + +test("estimateAll sums the parts", () => { + const parts = ["alpha beta", "gamma", "| delta |"]; + assert.equal(estimateAll(parts), parts.reduce((n, p) => n + estimateTokens(p), 0)); +}); + +test("reduction never invents an improvement out of nothing", () => { + assert.equal(reduction(0, 0), 0); + assert.equal(reduction(-5, 1), 0); + assert.equal(reduction(1000, 250), 75); + assert.equal(reduction(1000, 1000), 0); +}); From 07bbf5ae6299ba89809f6d46449347078cd035ca Mon Sep 17 00:00:00 2001 From: Schneider <224583183+schneiderjoseph@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:47:17 -0400 Subject: [PATCH 2/3] The benchmark's own JSON was truncated on POSIX CI caught it on ubuntu node 22 and nowhere else: `not ok 58 - the benchmark passes: the saving never costs a blocking rule`, failing on `Unterminated string in JSON at position 146032`. Not a selection defect -- the report is 182 kB and the test was reading the first 146 kB of it. `console.log` then `process.exit()` loses whatever is still buffered. On Windows stdout to a pipe is synchronous, so the write always completed; on POSIX it is asynchronous and the exit cut it off once the report outgrew the pipe buffer. ubuntu node 20 won the race, node 22 did not. A test that passes on three of four runners because of timing is the worst kind of green. `fs.writeSync(1, ...)` blocks until the bytes are gone, so the exit on the next line cannot truncate it. The failure path drops to `process.exitCode`, which is what bin/devia.mjs and src/lib/ui.mjs already use -- the CLI never calls process.exit() and was never exposed to this. Scope: scripts/benchmark-context.mjs only. Nothing in bin/ or src/ changes, so the published 0.8.0 CLI is unaffected and the version is not bumped. The 0.8.0 tarball on npm carries the unfixed script; no adopter runs it. Verified: `node scripts/benchmark-context.mjs --json` emits 182585 bytes that parse whole -- ok true, 352 rows, 0 failures. 135 tests pass. Benchmark unchanged at 352 runs, recall 100%, routing 100%, strict 176/176. --- scripts/benchmark-context.mjs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark-context.mjs b/scripts/benchmark-context.mjs index 7d4106e..7c742fd 100644 --- a/scripts/benchmark-context.mjs +++ b/scripts/benchmark-context.mjs @@ -343,7 +343,10 @@ for (const shape of corpora) { cleanup(); if (json) { - console.log(JSON.stringify({ ok: failures.length === 0, failures, rows }, null, 2)); + // console.log then exit truncates a pipe on POSIX, where stdout is async and this report + // runs well past a pipe buffer: the reader parses half a document. writeSync(1) blocks + // until it is out, so exiting on the next line cannot cut it short. + fs.writeSync(1, JSON.stringify({ ok: failures.length === 0, failures, rows }, null, 2) + "\n"); process.exit(failures.length ? 1 : 0); } @@ -396,6 +399,6 @@ if (failures.length) { } if (failures.length > 25) console.error(` …and ${failures.length - 25} more`); console.error("\n A saving that drops a rule the task needed is not a saving."); - process.exit(1); + process.exitCode = 1; } console.log(""); From af1e029ef3b38fe844155b611417607f4d5af8df Mon Sep 17 00:00:00 2001 From: Schneider <224583183+schneiderjoseph@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:52:34 -0400 Subject: [PATCH 3/3] One writeSync is not enough for 180 kB down a pipe The previous commit was the right diagnosis and the wrong remedy. CI failed again on ubuntu node 22, at byte 146032 of 182402 -- the same byte as before. An identical cut point is not a race; a single fs.writeSync on a non-blocking pipe writes what currently fits and returns short, and the return value was being discarded. The remaining 36 kB were never written. Loop until every byte is gone, treating EAGAIN as "the reader has not drained yet" rather than an error. Windows never showed this because stdout there is synchronous and the first call always completed; a redirect to a file did not show it either, because a file fd is not non-blocking. Only a pipe -- which is exactly what execFileSync gives the test -- could. Still scripts/benchmark-context.mjs only. bin/ and src/ are untouched and the published 0.8.0 CLI remains unaffected. Verified: the report through a pipe is 182402 bytes and parses whole -- ok true, 352 rows, 0 failures. 135 tests pass locally, but the defect only reproduces on POSIX, so ubuntu is the run that decides this. --- scripts/benchmark-context.mjs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/benchmark-context.mjs b/scripts/benchmark-context.mjs index 7c742fd..55f0b2e 100644 --- a/scripts/benchmark-context.mjs +++ b/scripts/benchmark-context.mjs @@ -343,10 +343,18 @@ for (const shape of corpora) { cleanup(); if (json) { - // console.log then exit truncates a pipe on POSIX, where stdout is async and this report - // runs well past a pipe buffer: the reader parses half a document. writeSync(1) blocks - // until it is out, so exiting on the next line cannot cut it short. - fs.writeSync(1, JSON.stringify({ ok: failures.length === 0, failures, rows }, null, 2) + "\n"); + // This report is ~180 kB and stdout may be a pipe. console.log then exit loses whatever is + // still buffered, and a single writeSync is not enough either: on a non-blocking pipe it + // writes what currently fits and returns short, or raises EAGAIN. Either way the reader + // parses half a document. Loop until every byte is gone, then exit. + const out = Buffer.from(JSON.stringify({ ok: failures.length === 0, failures, rows }, null, 2) + "\n"); + for (let off = 0; off < out.length; ) { + try { + off += fs.writeSync(1, out, off, out.length - off); + } catch (err) { + if (err.code !== "EAGAIN") throw err; + } + } process.exit(failures.length ? 1 : 0); }