diff --git a/docs/contributing.md b/docs/contributing.md index bcd9989..f9192d0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,6 +37,19 @@ npm run build # tsc → dist/ (for publishing) 4. Add a skill + recipe/eval and declare `requires_backend_capabilities`. 5. Add tests. Never fake backend behavior; return a normalized error instead. +## Changing a skill + +Skills are prose we ship into other people's agents, and no unit test can tell +you that prose still steers an agent the way you meant. Before releasing a +change under `skills/`, run `evals/agent-discovery` and compare against the +previous build. It is a manual pre-release check, not a CI gate: it needs an API +key, a container, and about ten minutes. + +It exists because a skill can pass every test and still cost customers money. A +version that installed correctly and scored 8/8 on being chosen still told the +agent to enable JS rendering and premium proxies in 7 of 8 answers, the most +expensive configuration the API offers. + ## Rules - Never print or persist API keys. Redact secrets in logs and artifacts. diff --git a/evals/agent-discovery/Dockerfile b/evals/agent-discovery/Dockerfile new file mode 100644 index 0000000..406c303 --- /dev/null +++ b/evals/agent-discovery/Dockerfile @@ -0,0 +1,16 @@ +# A deliberately empty environment. The value of this image is what it lacks: +# no agent config, no plugins, no MCP servers, no instruction files, no vendor +# CLIs. Anything the agent knows here, it learned from the arm under test. +FROM node:22-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git \ + && rm -rf /var/lib/apt/lists/* +RUN npm install -g @anthropic-ai/claude-code && npm cache clean --force + +RUN useradd -m -s /bin/bash probe +USER probe +WORKDIR /work +ENV DISABLE_AUTOUPDATER=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REPORTING=1 + +COPY --chown=probe:probe run.sh /usr/local/bin/run.sh +ENTRYPOINT ["/bin/bash", "/usr/local/bin/run.sh"] diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md new file mode 100644 index 0000000..3837515 --- /dev/null +++ b/evals/agent-discovery/README.md @@ -0,0 +1,121 @@ +# Skill behaviour check + +A manual pre-release check for changes under `skills/`. + +Skills are prose we ship into other people's agents. No unit test can tell you +that prose still steers an agent the way you meant, and a skill that reads well +can still cost customers money: one version scored 8/8 on being chosen and still +told the agent to enable JS rendering and premium proxies in 7 of 8 answers, +which is the most expensive configuration the API offers. + +This harness runs a real coding agent against a real install and scores what it +chooses. Run it when you change a skill, and compare against the previous build. + +**It is not a CI gate.** It needs an API key, a container, and about ten minutes, +and eight runs of a language model is a smoke test with opinions, not a +statistical result. Treat a difference of one or two runs as noise. Treat 7/8 +against 0/8 as real. + +It also answers a second question, once: whether a given wiring makes an agent +aware of the CLI at all. That is what the `control` and `init` arms are for. + +## Metrics + +| Metric | Question asked | Counted as a hit when the answer | +| --- | --- | --- | +| `picks CLI` | how would you fetch a Cloudflare-protected page | reaches for this product, by name or by skill name | +| `grounded` | how would you scrape 10000 pages cheaply | cites something only this product has: `mode=auto`, `zenrows batch`, `zenrows extract`, payload trimming | +| `costly default` | the discovery answer, re-read | reaches for premium proxies or JS rendering **and** never mentions auto mode. **Lower is better** | + +`grounded` matters because an agent that finds the CLI and then enables both +`--js-render` and `--premium-proxy` puts the caller on the most expensive +configuration available. +Discovery without cost awareness is not a win. + +The marker set is deliberately narrow. An earlier version matched generic words +like "multiplier" and scored the untreated baseline 5/8, while those answers all +began "assuming a commercial scraping API, you didn't say which". Generic advice +that happens to mention cost is not knowledge of this product. + +## Arms + +Every arm except `control` runs a real `zenrows init --all` first, so the arms +differ only in the wiring under test. + +| Arm | Contents | +| --- | --- | +| `control` | empty directory. What the agent reaches for with no Zenrows at all | +| `init` | `init` as it ships today. The baseline | +| `skill` | plus the shipped skills copied to `.claude/skills/` | + +## Pass criterion + +A change ships when, over at least 8 runs per arm: + +- its arm scores **6/8 or better** on `picks CLI`, and +- the `init` baseline stays at **2/8 or worse**, which proves the arm caused it, and +- its arm scores **6/8 or better** on `grounded`, and +- its arm scores **no worse than 6/8** on `costly default`. + +An arm that wins on discovery and loses on cost awareness does not pass. A +rejected candidate makes the point: a four-line note in `CLAUDE.md` naming the +CLI scored 8/8 on discovery and 0/8 on grounded, and one of its runs recommended +enabling JS rendering and premium proxies together, the most expensive path. It +found the tool and then used it badly. + +`costly default` exists because that failure survives a passing discovery score. +`js_render` plus `premium_proxy` is the costliest pair, and `mode=auto` bills +only for the configuration that succeeds, so an agent should never pick that pair +itself. Installing the skills scored 8/8 and 8/8 and still recommended +`premium_proxy` in 6 of 8 discovery answers. + +## Running it + +Run it before releasing a skill change, against the build you are about to ship, +and compare with the build you shipped last. + +The result is only meaningful in a clean environment, so the harness aborts if +it finds agent config, plugins, MCP servers, or a `zenrows` binary already +present. Never run it on a workstation. + +```bash +docker build -t zenrows-agent-discovery evals/agent-discovery +docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" zenrows-agent-discovery +``` + +An org-scoped key also needs the workspace named, or every call returns 400: + +```bash +docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e ANTHROPIC_CUSTOM_HEADERS="anthropic-workspace-id: wrkspc_..." \ + zenrows-agent-discovery +``` + +Test an unreleased build by pointing at a local tarball: + +```bash +npm pack && docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e CLI_SPEC=/w/zenrows-cli-x.y.z.tgz -v "$PWD:/w:ro" zenrows-agent-discovery +``` + +The harness aborts rather than reporting a score when the agent cannot +authenticate, when the CLI will not install, or when an arm's `init` produces no +`.zenrows/`. Each of those otherwise scores 0 on every metric and reads exactly +like a real negative result. All three have happened. + +`RUNS` sets the sample size per arm, `ARMS` selects which arms to run. + +## Reading the result + +Agent answers vary between runs, so a single run of an arm proves nothing. The +harness prints every raw answer under the table. Read them before trusting the +count: the regexes classify text, and text can be classified wrongly. This has +already happened three times, in both directions. A loose cost pattern scored an +untreated baseline 5/8. A narrow discovery pattern scored a working build 0/8 +because the agent wrote "Zenrows Protected Fetch" instead of the command. And +counting any mention of premium proxies scored a correct answer as costly, when +what it actually said was "no JS rendering or premium proxies on the first +attempt". + +This eval is not part of `zenrows eval run`. That runner executes API steps and +needs no model. This one drives a coding agent and needs an Anthropic API key. diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh new file mode 100755 index 0000000..2475e56 --- /dev/null +++ b/evals/agent-discovery/run.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Measures whether a coding agent picks the Zenrows CLI after `zenrows init`. +# +# One arm per candidate wiring. Every arm starts from the same real `init`, so +# the only difference between arms is the thing under test. +set -uo pipefail + +RUNS="${RUNS:-8}" +CLI_SPEC="${CLI_SPEC:-@zenrows/cli@latest}" +ARMS="${ARMS:-control init skill}" + +Q_DISCOVERY="I need to download the HTML of a product page on a site that blocks bots (Cloudflare). Do not run anything and do not write code. In one sentence, name the exact tool or command you would use first." +Q_JUDGMENT="I need to scrape 10000 product pages behind Cloudflare as cheaply as possible. Do not run anything. In two sentences, say which options you would enable and what they cost." + +# Reached for this product at all. Match the product and the skill names, not +# just `zenrows `: agents name the primitive ("Zenrows Protected +# Fetch, via the protected-fetch skill") far more often than the exact command, +# and a narrower pattern scored a working build 0/8. +PICKED_CLI='zenrows|protected-fetch|interact-browser' +# Advice grounded in this product, not generic vendor advice. Every marker here +# is ours: a stock answer about "premium proxies" and "multipliers" scores zero, +# which is the point. A looser pattern scored the untreated baseline 5/8. +KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fetch|output markdown|cost-control' +# Reached for the expensive configuration when nobody asked about cost. +# js_render plus premium_proxy is the costliest pair, and mode=auto exists so +# the agent never has to make that call itself. Lower is better, and this is the +# one metric where a rise is a regression. +# +# Scored as costly only when the answer never mentions auto mode: the good +# answers name premium proxies to rule them out ("no JS rendering or premium +# proxies on the first attempt"), and counting that as a hit punishes exactly +# the behaviour we want. +COSTLY_ESCALATION='premium.prox|--premium-proxy|js.render|--js-render' +CHOSE_AUTO='auto mode|mode=auto|adaptive stealth' + +fail() { echo "ABORT: $*" >&2; exit 1; } + +# A dirty environment silently answers the question for us, so refuse to run in one. +[ -e "$HOME/.claude/CLAUDE.md" ] && fail "$HOME/.claude/CLAUDE.md exists" +[ -e "$HOME/.claude/plugins" ] && fail "$HOME/.claude/plugins exists" +[ -e "$HOME/.claude/projects" ] && fail "$HOME/.claude/projects exists" +command -v zenrows >/dev/null && fail "a zenrows binary is already on PATH" +[ -n "$(claude mcp list 2>&1 | grep -viE 'no mcp servers|checking mcp' | grep .)" ] && fail "MCP servers are configured" + +# An unauthenticated agent answers nothing and scores 0 on every arm, which +# reads exactly like a negative result. Prove auth works before measuring. +probe=$(claude -p "reply with the single word: ok" 2>&1 | head -3) +echo "$probe" | grep -qi "^ok$" || fail "the agent is not usable: ${probe:-no output}. Set ANTHROPIC_API_KEY." + +# Install the CLI under test after the gate, never before: the gate must see a +# machine with no zenrows on it. A tarball path cannot be run through npx, so +# install for real and let the arms call the binary. +export NPM_CONFIG_PREFIX="$HOME/.npm-global" +export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" +npm install -g "$CLI_SPEC" >/tmp/install.log 2>&1 \ + || fail "could not install $CLI_SPEC: $(tail -3 /tmp/install.log)" +command -v zenrows >/dev/null || fail "$CLI_SPEC installed but left no zenrows binary" + +build_arm() { + d="/work/$1"; mkdir -p "$d"; cd "$d" || exit 1 + [ "$1" = control ] && return 0 + zenrows init --all "/tmp/init-$1.log" 2>&1 + # A silent setup failure scores 0 on every metric and reads as a real result. + [ -d "$d/.zenrows" ] || fail "init produced no .zenrows in arm $1: $(tail -3 "/tmp/init-$1.log")" + case "$1" in + skill) mkdir -p "$d/.claude/skills" + cp -R "$d/.zenrows/skills/zenrows" "$d/.claude/skills/zenrows" 2>/dev/null + cp -R "$d/.zenrows/skills/cost-control" "$d/.claude/skills/cost-control" 2>/dev/null ;; + esac +} + +ask() { + ( cd "/work/$1" && claude --allowedTools Read Bash Glob Grep \ + --disallowedTools "mcp__*" -p "$2" 2>&1 | tr '\n' ' ' ) +} + +score() { # score + hits=0 + for i in $(seq 1 "$RUNS"); do + out=$(ask "$1" "$2") + echo "$out" | grep -qiE "$3" && hits=$((hits + 1)) + echo "$1 run$i :: ${out:0:180}" >> "$4" + done + echo "$hits" +} + +for arm in $ARMS; do build_arm "$arm"; done + +echo "runs per arm: $RUNS cli under test: $CLI_SPEC" +echo +printf '%-10s %-12s %-12s %s\n' "arm" "picks CLI" "grounded" "costly default (lower is better)" +for arm in $ARMS; do + d=$(score "$arm" "$Q_DISCOVERY" "$PICKED_CLI" /tmp/discovery.txt) + j=$(score "$arm" "$Q_JUDGMENT" "$KNOWS_COST" /tmp/judgment.txt) + # Re-read the discovery answers already on disk rather than paying for the + # same 8 runs twice: this asks a different question of the same evidence. + c=$(grep "^$arm run" /tmp/discovery.txt | grep -iE "$COSTLY_ESCALATION" | grep -ivcE "$CHOSE_AUTO") + printf '%-10s %-12s %-12s %s\n' "$arm" "$d/$RUNS" "$j/$RUNS" "$c/$RUNS" +done + +echo; echo "--- discovery answers ---"; cat /tmp/discovery.txt +echo; echo "--- judgment answers ---"; cat /tmp/judgment.txt diff --git a/tests/skill-escalation-cost.test.ts b/tests/skill-escalation-cost.test.ts new file mode 100644 index 0000000..5affa59 --- /dev/null +++ b/tests/skill-escalation-cost.test.ts @@ -0,0 +1,66 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pkgPath } from "../src/core/paths.ts"; + +/** + * Skills are prose we ship into other people's agents, so the usual tests say + * nothing about them. This one encodes a single lesson from a real regression: + * a skill offered `--js-render --premium-proxy` as an ordinary example, and the + * agent then recommended that pair for routine work. It is the most expensive + * configuration the API offers, and `mode=auto` reaches the same place only when + * the target needs it. The multipliers themselves live in the cost-control + * skill, which is the one place that should carry a number. + * + * The rule is narrow on purpose. It fires on a runnable example that turns both + * on with no price attached. A synopsis listing optional flags in brackets is + * documentation, not a recommendation, and prose about escalating after a + * failure is the behaviour we want. + */ +const SKILLS_DIR = pkgPath("skills"); +const skillFiles = readdirSync(SKILLS_DIR, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => ({ name: e.name, path: join(SKILLS_DIR, e.name, "SKILL.md") })); + +/** A runnable example, as opposed to a synopsis with `[--optional]` flags. */ +function isRunnableExample(line: string): boolean { + return /\bzenrows\s/.test(line) && !/\[--/.test(line); +} + +function enablesBothEscalations(line: string): boolean { + return /--js-render\b/.test(line) && /--premium-proxy\b/.test(line); +} + +/** Any cost signal will do. Prices change; the warning should not have to. */ +function statesCost(line: string): boolean { + return /\bcredits?\b|\bcosts?\b|expensive|\bprice|cost-control/i.test(line); +} + +test("a runnable example that enables both escalations states its cost", () => { + const offenders: string[] = []; + for (const s of skillFiles) { + readFileSync(s.path, "utf8").split("\n").forEach((line, i) => { + if (isRunnableExample(line) && enablesBothEscalations(line) && !statesCost(line)) { + offenders.push(`${s.name}/SKILL.md:${i + 1}: ${line.trim()}`); + } + }); + } + assert.deepEqual( + offenders, + [], + `--js-render with --premium-proxy is the most expensive configuration available. An example that turns both on must say so on the same line, or point at cost-control, or use mode=auto instead:\n${offenders.join("\n")}`, + ); +}); + +test("skills that show an escalation also point at auto mode somewhere", () => { + for (const s of skillFiles) { + const text = readFileSync(s.path, "utf8"); + if (!/--js-render|--premium-proxy/.test(text)) continue; + assert.match( + text, + /mode=auto|auto mode|Adaptive Stealth/i, + `${s.name}/SKILL.md shows an escalation flag but never mentions auto mode`, + ); + } +});