From e4e000d4cc5b7424db611877b3ff2712b6c92014 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 14:20:23 +0200 Subject: [PATCH 1/9] test: add agent discovery eval zenrows init sets up .zenrows/, but an agent only reads what its own harness loads. Nothing in the project tells it the CLI exists, so it reaches for a plain HTTP client instead. This eval measures that, in a container with no agent config, no plugins, no MCP servers and no zenrows binary, so anything the agent knows it learned from the arm under test. It scores two things per arm: whether the agent picks the CLI, and whether it cites a real cost rule, because an agent that finds the CLI and then enables js-render and premium-proxy together costs the caller 25 credits per request. The pass criterion is fixed here, before any implementation, so a candidate change either clears the bar or does not. --- evals/agent-discovery/Dockerfile | 16 +++++++ evals/agent-discovery/README.md | 80 +++++++++++++++++++++++++++++++ evals/agent-discovery/run.sh | 82 ++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 evals/agent-discovery/Dockerfile create mode 100644 evals/agent-discovery/README.md create mode 100755 evals/agent-discovery/run.sh diff --git a/evals/agent-discovery/Dockerfile b/evals/agent-discovery/Dockerfile new file mode 100644 index 0000000..406c303 --- /dev/null +++ b/evals/agent-discovery/Dockerfile @@ -0,0 +1,16 @@ +# A deliberately empty environment. The value of this image is what it lacks: +# no agent config, no plugins, no MCP servers, no instruction files, no vendor +# CLIs. Anything the agent knows here, it learned from the arm under test. +FROM node:22-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git \ + && rm -rf /var/lib/apt/lists/* +RUN npm install -g @anthropic-ai/claude-code && npm cache clean --force + +RUN useradd -m -s /bin/bash probe +USER probe +WORKDIR /work +ENV DISABLE_AUTOUPDATER=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REPORTING=1 + +COPY --chown=probe:probe run.sh /usr/local/bin/run.sh +ENTRYPOINT ["/bin/bash", "/usr/local/bin/run.sh"] diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md new file mode 100644 index 0000000..339562c --- /dev/null +++ b/evals/agent-discovery/README.md @@ -0,0 +1,80 @@ +# Agent discovery eval + +Does a coding agent pick the Zenrows CLI after `zenrows init`? + +`init` sets up `.zenrows/`, but an agent only reads what its own harness loads. +This eval measures whether a candidate change makes the agent choose the CLI, +and whether it also gives the agent the cost rules it needs to choose well. + +The criterion below is fixed **before** any implementation lands, so a change +either clears the bar or does not. + +## Metrics + +| Metric | Question asked | Counted as a hit when the answer | +| --- | --- | --- | +| `picks CLI` | how would you fetch a Cloudflare-protected page | names the `zenrows` CLI as the first tool | +| `knows cost` | how would you scrape 10000 pages cheaply | cites a real cost rule (the credit multipliers, or `mode=auto`) | + +`knows cost` matters because an agent that finds the CLI and then enables both +`--js-render` and `--premium-proxy` puts the caller on 25 credits per request. +Discovery without cost awareness is not a win. + +## Arms + +Every arm except `control` runs a real `zenrows init --all` first, so the arms +differ only in the wiring under test. + +| Arm | Contents | +| --- | --- | +| `control` | empty directory. What the agent reaches for with no Zenrows at all | +| `init` | `init` as it ships today. The baseline | +| `pointer` | plus a Zenrows block in `CLAUDE.md` | +| `skill` | plus the shipped skills copied to `.claude/skills/` | + +## Pass criterion + +A change ships when, over at least 8 runs per arm: + +- its arm scores **6/8 or better** on `picks CLI`, and +- the `init` baseline stays at **2/8 or worse**, which proves the arm caused it, and +- its arm scores **no worse than the baseline** on `knows cost`. + +An arm that wins on discovery and loses on cost awareness does not pass. + +## Running it + +The result is only meaningful in a clean environment, so the harness aborts if +it finds agent config, plugins, MCP servers, or a `zenrows` binary already +present. Never run it on a workstation. + +```bash +docker build -t zenrows-agent-discovery evals/agent-discovery +docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" zenrows-agent-discovery +``` + +An org-scoped key also needs the workspace named, or every call returns 400: + +```bash +docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e ANTHROPIC_CUSTOM_HEADERS="anthropic-workspace-id: wrkspc_..." \ + zenrows-agent-discovery +``` + +Test an unreleased build by pointing at a local tarball: + +```bash +npm pack && docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e CLI_SPEC=/w/zenrows-cli-x.y.z.tgz -v "$PWD:/w:ro" zenrows-agent-discovery +``` + +`RUNS` sets the sample size per arm, `ARMS` selects which arms to run. + +## Reading the result + +Agent answers vary between runs, so a single run of an arm proves nothing. The +harness prints every raw answer under the table. Read them before trusting the +count: the regexes classify text, and text can be classified wrongly. + +This eval is not part of `zenrows eval run`. That runner executes API steps and +needs no model. This one drives a coding agent and needs an Anthropic API key. diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh new file mode 100755 index 0000000..b032ede --- /dev/null +++ b/evals/agent-discovery/run.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Measures whether a coding agent picks the Zenrows CLI after `zenrows init`. +# +# One arm per candidate wiring. Every arm starts from the same real `init`, so +# the only difference between arms is the thing under test. +set -uo pipefail + +RUNS="${RUNS:-8}" +CLI_SPEC="${CLI_SPEC:-@zenrows/cli@latest}" +ARMS="${ARMS:-control init pointer skill}" + +Q_DISCOVERY="I need to download the HTML of a product page on a site that blocks bots (Cloudflare). Do not run anything and do not write code. In one sentence, name the exact tool or command you would use first." +Q_JUDGMENT="I need to scrape 10000 product pages behind Cloudflare as cheaply as possible. Do not run anything. In two sentences, say which options you would enable and what they cost." + +# Selected the CLI, as opposed to naming Zenrows and then hand-rolling an HTTP call. +PICKED_CLI='(^|[^a-z/.])zenrows (cli|fetch|extract|status|scrape)|`zenrows`' +# Quoted a real cost rule rather than guessing. 1x/5x/10x/25x and mode=auto are ours. +KNOWS_COST='25x|25 credit|10x|10 credit|multiplier|mode=auto|adaptive stealth' + +fail() { echo "ABORT: $*" >&2; exit 1; } + +# A dirty environment silently answers the question for us, so refuse to run in one. +[ -e "$HOME/.claude/CLAUDE.md" ] && fail "$HOME/.claude/CLAUDE.md exists" +[ -e "$HOME/.claude/plugins" ] && fail "$HOME/.claude/plugins exists" +[ -e "$HOME/.claude/projects" ] && fail "$HOME/.claude/projects exists" +command -v zenrows >/dev/null && fail "a zenrows binary is already on PATH" +[ -n "$(claude mcp list 2>&1 | grep -viE 'no mcp servers|checking mcp' | grep .)" ] && fail "MCP servers are configured" + +# An unauthenticated agent answers nothing and scores 0 on every arm, which +# reads exactly like a negative result. Prove auth works before measuring. +probe=$(claude -p "reply with the single word: ok" 2>&1 | head -3) +echo "$probe" | grep -qi "^ok$" || fail "the agent is not usable: ${probe:-no output}. Set ANTHROPIC_API_KEY." + +build_arm() { + d="/work/$1"; mkdir -p "$d"; cd "$d" || exit 1 + [ "$1" = control ] && return 0 + npx -y "$CLI_SPEC" init --all /dev/null 2>&1 + case "$1" in + pointer) cat > "$d/CLAUDE.md" <<'PTR' +## Zenrows + +This project uses the Zenrows CLI for protected web data. Prefer it over a plain +HTTP client or a local browser when a page is behind anti-bot protection. + +- `zenrows --help` lists every command. +- `zenrows status --json` reports the live capability matrix. +PTR + ;; + skill) mkdir -p "$d/.claude/skills" + cp -R "$d/.zenrows/skills/zenrows" "$d/.claude/skills/zenrows" 2>/dev/null + cp -R "$d/.zenrows/skills/cost-control" "$d/.claude/skills/cost-control" 2>/dev/null ;; + esac +} + +ask() { + ( cd "/work/$1" && claude --allowedTools Read Bash Glob Grep \ + --disallowedTools "mcp__*" -p "$2" 2>&1 | tr '\n' ' ' ) +} + +score() { # score + hits=0 + for i in $(seq 1 "$RUNS"); do + out=$(ask "$1" "$2") + echo "$out" | grep -qiE "$3" && hits=$((hits + 1)) + echo "$1 run$i :: ${out:0:180}" >> "$4" + done + echo "$hits" +} + +for arm in $ARMS; do build_arm "$arm"; done + +echo "runs per arm: $RUNS cli under test: $CLI_SPEC" +echo +printf '%-10s %-14s %-14s\n' "arm" "picks CLI" "knows cost" +for arm in $ARMS; do + d=$(score "$arm" "$Q_DISCOVERY" "$PICKED_CLI" /tmp/discovery.txt) + j=$(score "$arm" "$Q_JUDGMENT" "$KNOWS_COST" /tmp/judgment.txt) + printf '%-10s %-14s %-14s\n' "$arm" "$d/$RUNS" "$j/$RUNS" +done + +echo; echo "--- discovery answers ---"; cat /tmp/discovery.txt +echo; echo "--- judgment answers ---"; cat /tmp/judgment.txt From 89fa5288194ef0e5b4c4d31306fbd096df071b10 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 15:52:54 +0200 Subject: [PATCH 2/9] test: score cost awareness on product-specific markers The first metric matched generic words like multiplier, so the untreated baseline scored 5/8 while every one of those answers opened with 'assuming a commercial scraping API, you didn't say which'. It measured whether the model can talk about scraping costs, not whether it knows ours. The marker set is now limited to things only this product has. Under it the baseline scores 0/8, and the gap between candidate wirings is visible. --- evals/agent-discovery/README.md | 16 ++++++++++++---- evals/agent-discovery/run.sh | 8 +++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 339562c..7f84b25 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -14,12 +14,17 @@ either clears the bar or does not. | Metric | Question asked | Counted as a hit when the answer | | --- | --- | --- | | `picks CLI` | how would you fetch a Cloudflare-protected page | names the `zenrows` CLI as the first tool | -| `knows cost` | how would you scrape 10000 pages cheaply | cites a real cost rule (the credit multipliers, or `mode=auto`) | +| `grounded` | how would you scrape 10000 pages cheaply | cites something only this product has: `mode=auto`, `zenrows batch`, `zenrows extract`, payload trimming | -`knows cost` matters because an agent that finds the CLI and then enables both +`grounded` matters because an agent that finds the CLI and then enables both `--js-render` and `--premium-proxy` puts the caller on 25 credits per request. Discovery without cost awareness is not a win. +The marker set is deliberately narrow. An earlier version matched generic words +like "multiplier" and scored the untreated baseline 5/8, while those answers all +began "assuming a commercial scraping API, you didn't say which". Generic advice +that happens to mention cost is not knowledge of this product. + ## Arms Every arm except `control` runs a real `zenrows init --all` first, so the arms @@ -38,9 +43,12 @@ A change ships when, over at least 8 runs per arm: - its arm scores **6/8 or better** on `picks CLI`, and - the `init` baseline stays at **2/8 or worse**, which proves the arm caused it, and -- its arm scores **no worse than the baseline** on `knows cost`. +- its arm scores **6/8 or better** on `grounded`. -An arm that wins on discovery and loses on cost awareness does not pass. +An arm that wins on discovery and loses on cost awareness does not pass. A +measured example: a four-line pointer in `CLAUDE.md` scored 8/8 on discovery and +0/8 on grounded, and one of its runs recommended enabling JS rendering and +premium proxies together, which is the 25 credit path. ## Running it diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index b032ede..38eba26 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -14,8 +14,10 @@ Q_JUDGMENT="I need to scrape 10000 product pages behind Cloudflare as cheaply as # Selected the CLI, as opposed to naming Zenrows and then hand-rolling an HTTP call. PICKED_CLI='(^|[^a-z/.])zenrows (cli|fetch|extract|status|scrape)|`zenrows`' -# Quoted a real cost rule rather than guessing. 1x/5x/10x/25x and mode=auto are ours. -KNOWS_COST='25x|25 credit|10x|10 credit|multiplier|mode=auto|adaptive stealth' +# Advice grounded in this product, not generic vendor advice. Every marker here +# is ours: a stock answer about "premium proxies" and "multipliers" scores zero, +# which is the point. A looser pattern scored the untreated baseline 5/8. +KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fetch|output markdown|25 credit' fail() { echo "ABORT: $*" >&2; exit 1; } @@ -71,7 +73,7 @@ for arm in $ARMS; do build_arm "$arm"; done echo "runs per arm: $RUNS cli under test: $CLI_SPEC" echo -printf '%-10s %-14s %-14s\n' "arm" "picks CLI" "knows cost" +printf '%-10s %-14s %-14s\n' "arm" "picks CLI" "grounded" for arm in $ARMS; do d=$(score "$arm" "$Q_DISCOVERY" "$PICKED_CLI" /tmp/discovery.txt) j=$(score "$arm" "$Q_JUDGMENT" "$KNOWS_COST" /tmp/judgment.txt) From 751342bc04e1df70dc9dc913bc857e2d8b9d0554 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 16:06:43 +0200 Subject: [PATCH 3/9] test: abort on a failed arm setup instead of scoring it zero A local tarball cannot be run through npx, so the candidate arm silently installed nothing and scored 0/8 on every metric, which is indistinguishable from the change not working. The CLI is now installed once after the cleanliness gate, and an arm whose init leaves no .zenrows aborts the run. --- evals/agent-discovery/README.md | 5 +++++ evals/agent-discovery/run.sh | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 7f84b25..a310d37 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -76,6 +76,11 @@ npm pack && docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ -e CLI_SPEC=/w/zenrows-cli-x.y.z.tgz -v "$PWD:/w:ro" zenrows-agent-discovery ``` +The harness aborts rather than reporting a score when the agent cannot +authenticate, when the CLI will not install, or when an arm's `init` produces no +`.zenrows/`. Each of those otherwise scores 0 on every metric and reads exactly +like a real negative result. All three have happened. + `RUNS` sets the sample size per arm, `ARMS` selects which arms to run. ## Reading the result diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index 38eba26..7190ec9 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -33,10 +33,21 @@ command -v zenrows >/dev/null && fail "a zenrows binary is already on PATH" probe=$(claude -p "reply with the single word: ok" 2>&1 | head -3) echo "$probe" | grep -qi "^ok$" || fail "the agent is not usable: ${probe:-no output}. Set ANTHROPIC_API_KEY." +# Install the CLI under test after the gate, never before: the gate must see a +# machine with no zenrows on it. A tarball path cannot be run through npx, so +# install for real and let the arms call the binary. +export NPM_CONFIG_PREFIX="$HOME/.npm-global" +export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" +npm install -g "$CLI_SPEC" >/tmp/install.log 2>&1 \ + || fail "could not install $CLI_SPEC: $(tail -3 /tmp/install.log)" +command -v zenrows >/dev/null || fail "$CLI_SPEC installed but left no zenrows binary" + build_arm() { d="/work/$1"; mkdir -p "$d"; cd "$d" || exit 1 [ "$1" = control ] && return 0 - npx -y "$CLI_SPEC" init --all /dev/null 2>&1 + zenrows init --all "/tmp/init-$1.log" 2>&1 + # A silent setup failure scores 0 on every metric and reads as a real result. + [ -d "$d/.zenrows" ] || fail "init produced no .zenrows in arm $1: $(tail -3 "/tmp/init-$1.log")" case "$1" in pointer) cat > "$d/CLAUDE.md" <<'PTR' ## Zenrows From 7fdbddfba22b3aff2c65b7d8c3b8e1e6fef84153 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 16:40:37 +0200 Subject: [PATCH 4/9] test: track the expensive default, and match how agents name the product Two scoring fixes from the same run. The discovery pattern looked for 'zenrows ' and missed 'Zenrows Protected Fetch, via the protected-fetch skill', which is how the agent phrases it, so a working build scored 0/8. The new costly-default metric counts discovery answers that reach for premium_proxy when nobody asked about cost. That pair with js_render is 25 credits per request, while mode=auto bills only for what succeeds. The failure survives a passing discovery score: the skills scored 8/8 on discovery and still recommended premium_proxy in 6 of 8 answers. --- evals/agent-discovery/README.md | 17 ++++++++++++++--- evals/agent-discovery/run.sh | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index a310d37..69130f1 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -13,8 +13,9 @@ either clears the bar or does not. | Metric | Question asked | Counted as a hit when the answer | | --- | --- | --- | -| `picks CLI` | how would you fetch a Cloudflare-protected page | names the `zenrows` CLI as the first tool | +| `picks CLI` | how would you fetch a Cloudflare-protected page | reaches for this product, by name or by skill name | | `grounded` | how would you scrape 10000 pages cheaply | cites something only this product has: `mode=auto`, `zenrows batch`, `zenrows extract`, payload trimming | +| `costly default` | the discovery answer, re-read | recommends `premium_proxy` when nobody asked about cost. **Lower is better** | `grounded` matters because an agent that finds the CLI and then enables both `--js-render` and `--premium-proxy` puts the caller on 25 credits per request. @@ -43,13 +44,20 @@ A change ships when, over at least 8 runs per arm: - its arm scores **6/8 or better** on `picks CLI`, and - the `init` baseline stays at **2/8 or worse**, which proves the arm caused it, and -- its arm scores **6/8 or better** on `grounded`. +- its arm scores **6/8 or better** on `grounded`, and +- its arm scores **no worse than 6/8** on `costly default`. An arm that wins on discovery and loses on cost awareness does not pass. A measured example: a four-line pointer in `CLAUDE.md` scored 8/8 on discovery and 0/8 on grounded, and one of its runs recommended enabling JS rendering and premium proxies together, which is the 25 credit path. +`costly default` exists because that failure survives a passing discovery score. +`js_render` plus `premium_proxy` is 25 credits per request, and `mode=auto` bills +only for the configuration that succeeds, so an agent should never pick that pair +itself. Installing the skills scored 8/8 and 8/8 and still recommended +`premium_proxy` in 6 of 8 discovery answers. + ## Running it The result is only meaningful in a clean environment, so the harness aborts if @@ -87,7 +95,10 @@ like a real negative result. All three have happened. Agent answers vary between runs, so a single run of an arm proves nothing. The harness prints every raw answer under the table. Read them before trusting the -count: the regexes classify text, and text can be classified wrongly. +count: the regexes classify text, and text can be classified wrongly. This has +already happened twice, in both directions. A loose cost pattern scored an +untreated baseline 5/8, and a narrow discovery pattern scored a working build +0/8 because the agent wrote "Zenrows Protected Fetch" instead of the command. This eval is not part of `zenrows eval run`. That runner executes API steps and needs no model. This one drives a coding agent and needs an Anthropic API key. diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index 7190ec9..18e56ff 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -12,12 +12,20 @@ ARMS="${ARMS:-control init pointer skill}" Q_DISCOVERY="I need to download the HTML of a product page on a site that blocks bots (Cloudflare). Do not run anything and do not write code. In one sentence, name the exact tool or command you would use first." Q_JUDGMENT="I need to scrape 10000 product pages behind Cloudflare as cheaply as possible. Do not run anything. In two sentences, say which options you would enable and what they cost." -# Selected the CLI, as opposed to naming Zenrows and then hand-rolling an HTTP call. -PICKED_CLI='(^|[^a-z/.])zenrows (cli|fetch|extract|status|scrape)|`zenrows`' +# Reached for this product at all. Match the product and the skill names, not +# just `zenrows `: agents name the primitive ("Zenrows Protected +# Fetch, via the protected-fetch skill") far more often than the exact command, +# and a narrower pattern scored a working build 0/8. +PICKED_CLI='zenrows|protected-fetch|interact-browser' # Advice grounded in this product, not generic vendor advice. Every marker here # is ours: a stock answer about "premium proxies" and "multipliers" scores zero, # which is the point. A looser pattern scored the untreated baseline 5/8. KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fetch|output markdown|25 credit' +# Reached for the expensive configuration when nobody asked about cost. +# js_render plus premium_proxy is 25 credits per request, and mode=auto exists so +# the agent never has to make that call itself. Lower is better, and this is the +# one metric where a rise is a regression. +COSTLY_DEFAULT='premium_proxy|premium prox|--premium-proxy' fail() { echo "ABORT: $*" >&2; exit 1; } @@ -84,11 +92,14 @@ for arm in $ARMS; do build_arm "$arm"; done echo "runs per arm: $RUNS cli under test: $CLI_SPEC" echo -printf '%-10s %-14s %-14s\n' "arm" "picks CLI" "grounded" +printf '%-10s %-12s %-12s %s\n' "arm" "picks CLI" "grounded" "costly default (lower is better)" for arm in $ARMS; do d=$(score "$arm" "$Q_DISCOVERY" "$PICKED_CLI" /tmp/discovery.txt) j=$(score "$arm" "$Q_JUDGMENT" "$KNOWS_COST" /tmp/judgment.txt) - printf '%-10s %-14s %-14s\n' "$arm" "$d/$RUNS" "$j/$RUNS" + # Re-read the discovery answers already on disk rather than paying for the + # same 8 runs twice: this asks a different question of the same evidence. + c=$(grep "^$arm run" /tmp/discovery.txt | grep -icE "$COSTLY_DEFAULT") + printf '%-10s %-12s %-12s %s\n' "$arm" "$d/$RUNS" "$j/$RUNS" "$c/$RUNS" done echo; echo "--- discovery answers ---"; cat /tmp/discovery.txt From 5ded10a5200674a11ee0a3a0c71ca3630c9f90b9 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 16:59:55 +0200 Subject: [PATCH 5/9] test: do not score a refusal as an expensive choice The costly-default metric matched any mention of premium proxies, so an answer saying 'no JS rendering or premium proxies on the first attempt' counted as reaching for them. It scored a correct run 4/8 when the true value was 0/8. An answer is now costly only when it names an escalation and never mentions auto mode. A metric that punishes the behaviour we want would eventually push someone to fix the wrong thing. --- evals/agent-discovery/README.md | 11 +++++++---- evals/agent-discovery/run.sh | 10 ++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 69130f1..3f37287 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -15,7 +15,7 @@ either clears the bar or does not. | --- | --- | --- | | `picks CLI` | how would you fetch a Cloudflare-protected page | reaches for this product, by name or by skill name | | `grounded` | how would you scrape 10000 pages cheaply | cites something only this product has: `mode=auto`, `zenrows batch`, `zenrows extract`, payload trimming | -| `costly default` | the discovery answer, re-read | recommends `premium_proxy` when nobody asked about cost. **Lower is better** | +| `costly default` | the discovery answer, re-read | reaches for premium proxies or JS rendering **and** never mentions auto mode. **Lower is better** | `grounded` matters because an agent that finds the CLI and then enables both `--js-render` and `--premium-proxy` puts the caller on 25 credits per request. @@ -96,9 +96,12 @@ like a real negative result. All three have happened. Agent answers vary between runs, so a single run of an arm proves nothing. The harness prints every raw answer under the table. Read them before trusting the count: the regexes classify text, and text can be classified wrongly. This has -already happened twice, in both directions. A loose cost pattern scored an -untreated baseline 5/8, and a narrow discovery pattern scored a working build -0/8 because the agent wrote "Zenrows Protected Fetch" instead of the command. +already happened three times, in both directions. A loose cost pattern scored an +untreated baseline 5/8. A narrow discovery pattern scored a working build 0/8 +because the agent wrote "Zenrows Protected Fetch" instead of the command. And +counting any mention of premium proxies scored a correct answer as costly, when +what it actually said was "no JS rendering or premium proxies on the first +attempt". This eval is not part of `zenrows eval run`. That runner executes API steps and needs no model. This one drives a coding agent and needs an Anthropic API key. diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index 18e56ff..5da3476 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -25,7 +25,13 @@ KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fet # js_render plus premium_proxy is 25 credits per request, and mode=auto exists so # the agent never has to make that call itself. Lower is better, and this is the # one metric where a rise is a regression. -COSTLY_DEFAULT='premium_proxy|premium prox|--premium-proxy' +# +# Scored as costly only when the answer never mentions auto mode: the good +# answers name premium proxies to rule them out ("no JS rendering or premium +# proxies on the first attempt"), and counting that as a hit punishes exactly +# the behaviour we want. +COSTLY_ESCALATION='premium.prox|--premium-proxy|js.render|--js-render' +CHOSE_AUTO='auto mode|mode=auto|adaptive stealth' fail() { echo "ABORT: $*" >&2; exit 1; } @@ -98,7 +104,7 @@ for arm in $ARMS; do j=$(score "$arm" "$Q_JUDGMENT" "$KNOWS_COST" /tmp/judgment.txt) # Re-read the discovery answers already on disk rather than paying for the # same 8 runs twice: this asks a different question of the same evidence. - c=$(grep "^$arm run" /tmp/discovery.txt | grep -icE "$COSTLY_DEFAULT") + c=$(grep "^$arm run" /tmp/discovery.txt | grep -iE "$COSTLY_ESCALATION" | grep -ivcE "$CHOSE_AUTO") printf '%-10s %-12s %-12s %s\n' "$arm" "$d/$RUNS" "$j/$RUNS" "$c/$RUNS" done From f2bc51651f957acd20426611bcf9041ec40780f9 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 17:05:47 +0200 Subject: [PATCH 6/9] docs: frame the harness as a pre-release check for skill changes Discovery, the question that started this, needs no harness: run init and look for a SKILL.md outside .zenrows/. What does need one is skill content. Skills are prose shipped into other people's agents, no unit test can tell you the prose still steers the way you meant, and a version that scored 8/8 on being chosen still recommended a 25 credit configuration in 7 of 8 answers. Says plainly that it is not a CI gate, and what size of difference to believe. --- docs/contributing.md | 13 +++++++++++++ evals/agent-discovery/README.md | 27 ++++++++++++++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index bcd9989..ded66d3 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,6 +37,19 @@ npm run build # tsc → dist/ (for publishing) 4. Add a skill + recipe/eval and declare `requires_backend_capabilities`. 5. Add tests. Never fake backend behavior; return a normalized error instead. +## Changing a skill + +Skills are prose we ship into other people's agents, and no unit test can tell +you that prose still steers an agent the way you meant. Before releasing a +change under `skills/`, run `evals/agent-discovery` and compare against the +previous build. It is a manual pre-release check, not a CI gate: it needs an API +key, a container, and about ten minutes. + +It exists because a skill can pass every test and still cost customers money. A +version that installed correctly and scored 8/8 on being chosen still told the +agent to enable JS rendering and premium proxies in 7 of 8 answers, which is 25 +credits per request against 1. + ## Rules - Never print or persist API keys. Redact secrets in logs and artifacts. diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 3f37287..3f939a9 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -1,13 +1,23 @@ -# Agent discovery eval +# Skill behaviour check -Does a coding agent pick the Zenrows CLI after `zenrows init`? +A manual pre-release check for changes under `skills/`. -`init` sets up `.zenrows/`, but an agent only reads what its own harness loads. -This eval measures whether a candidate change makes the agent choose the CLI, -and whether it also gives the agent the cost rules it needs to choose well. +Skills are prose we ship into other people's agents. No unit test can tell you +that prose still steers an agent the way you meant, and a skill that reads well +can still cost customers money: one version scored 8/8 on being chosen and still +told the agent to enable JS rendering and premium proxies in 7 of 8 answers, +which is 25 credits per request against 1. -The criterion below is fixed **before** any implementation lands, so a change -either clears the bar or does not. +This harness runs a real coding agent against a real install and scores what it +chooses. Run it when you change a skill, and compare against the previous build. + +**It is not a CI gate.** It needs an API key, a container, and about ten minutes, +and eight runs of a language model is a smoke test with opinions, not a +statistical result. Treat a difference of one or two runs as noise. Treat 7/8 +against 0/8 as real. + +It also answers a second question, once: whether a given wiring makes an agent +aware of the CLI at all. That is what the `control` and `init` arms are for. ## Metrics @@ -60,6 +70,9 @@ itself. Installing the skills scored 8/8 and 8/8 and still recommended ## Running it +Run it before releasing a skill change, against the build you are about to ship, +and compare with the build you shipped last. + The result is only meaningful in a clean environment, so the harness aborts if it finds agent config, plugins, MCP servers, or a `zenrows` binary already present. Never run it on a workstation. From 0ff4406bf81b36b72f433b447ac1ec4f38acdd87 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 17:14:18 +0200 Subject: [PATCH 7/9] test: fail the build when a skill sells an expensive default Two deterministic rules over skills/*/SKILL.md, derived from a regression the agent-discovery harness found empirically: a runnable example that turns on both --js-render and --premium-proxy must state the cost on that line, and any skill showing an escalation flag must mention auto mode somewhere. The rules are narrow on purpose. A synopsis listing optional flags in brackets is documentation rather than a recommendation, and prose about escalating after a failure is the behaviour we want. This runs in milliseconds, needs no API key, and works on fork pull requests, so it belongs in CI in a way the harness itself does not. --- tests/skill-escalation-cost.test.ts | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/skill-escalation-cost.test.ts diff --git a/tests/skill-escalation-cost.test.ts b/tests/skill-escalation-cost.test.ts new file mode 100644 index 0000000..60376a6 --- /dev/null +++ b/tests/skill-escalation-cost.test.ts @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pkgPath } from "../src/core/paths.ts"; + +/** + * Skills are prose we ship into other people's agents, so the usual tests say + * nothing about them. This one encodes a single lesson from a real regression: + * a skill offered `--js-render --premium-proxy` as an ordinary example, and the + * agent then recommended that pair for routine work. It is 25 credits per + * request against 1, and `mode=auto` reaches the same place only when the + * target needs it. + * + * The rule is narrow on purpose. It fires on a runnable example that turns both + * on with no price attached. A synopsis listing optional flags in brackets is + * documentation, not a recommendation, and prose about escalating after a + * failure is the behaviour we want. + */ +const SKILLS_DIR = pkgPath("skills"); +const skillFiles = readdirSync(SKILLS_DIR, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => ({ name: e.name, path: join(SKILLS_DIR, e.name, "SKILL.md") })); + +/** A runnable example, as opposed to a synopsis with `[--optional]` flags. */ +function isRunnableExample(line: string): boolean { + return /\bzenrows\s/.test(line) && !/\[--/.test(line); +} + +function enablesBothEscalations(line: string): boolean { + return /--js-render\b/.test(line) && /--premium-proxy\b/.test(line); +} + +function statesCost(line: string): boolean { + return /\bcredits?\b|\bcosts?\b|\b25\b/i.test(line); +} + +test("a runnable example that enables both escalations states its cost", () => { + const offenders: string[] = []; + for (const s of skillFiles) { + readFileSync(s.path, "utf8").split("\n").forEach((line, i) => { + if (isRunnableExample(line) && enablesBothEscalations(line) && !statesCost(line)) { + offenders.push(`${s.name}/SKILL.md:${i + 1}: ${line.trim()}`); + } + }); + } + assert.deepEqual( + offenders, + [], + `--js-render with --premium-proxy is 25 credits per request. An example that turns both on must say so on the same line, or use mode=auto instead:\n${offenders.join("\n")}`, + ); +}); + +test("skills that show an escalation also point at auto mode somewhere", () => { + for (const s of skillFiles) { + const text = readFileSync(s.path, "utf8"); + if (!/--js-render|--premium-proxy/.test(text)) continue; + assert.match( + text, + /mode=auto|auto mode|Adaptive Stealth/i, + `${s.name}/SKILL.md shows an escalation flag but never mentions auto mode`, + ); + } +}); From 13a068c5d726980fde66239620d756d7b2163df2 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 17:21:57 +0200 Subject: [PATCH 8/9] test: cite the cost model instead of repeating prices Every warning that quoted a figure now names the direction and points at cost-control, which owns the multipliers. A price repeated in six files goes stale in six files. Also restores tests/skill-content.test.ts, which an earlier commit on this branch overwrote. --- docs/contributing.md | 4 ++-- evals/agent-discovery/README.md | 9 +++++---- evals/agent-discovery/run.sh | 4 ++-- tests/skill-escalation-cost.test.ts | 12 +++++++----- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index ded66d3..f9192d0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -47,8 +47,8 @@ key, a container, and about ten minutes. It exists because a skill can pass every test and still cost customers money. A version that installed correctly and scored 8/8 on being chosen still told the -agent to enable JS rendering and premium proxies in 7 of 8 answers, which is 25 -credits per request against 1. +agent to enable JS rendering and premium proxies in 7 of 8 answers, the most +expensive configuration the API offers. ## Rules diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 3f939a9..693c63f 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -6,7 +6,7 @@ Skills are prose we ship into other people's agents. No unit test can tell you that prose still steers an agent the way you meant, and a skill that reads well can still cost customers money: one version scored 8/8 on being chosen and still told the agent to enable JS rendering and premium proxies in 7 of 8 answers, -which is 25 credits per request against 1. +which is the most expensive configuration the API offers. This harness runs a real coding agent against a real install and scores what it chooses. Run it when you change a skill, and compare against the previous build. @@ -28,7 +28,8 @@ aware of the CLI at all. That is what the `control` and `init` arms are for. | `costly default` | the discovery answer, re-read | reaches for premium proxies or JS rendering **and** never mentions auto mode. **Lower is better** | `grounded` matters because an agent that finds the CLI and then enables both -`--js-render` and `--premium-proxy` puts the caller on 25 credits per request. +`--js-render` and `--premium-proxy` puts the caller on the most expensive +configuration available. Discovery without cost awareness is not a win. The marker set is deliberately narrow. An earlier version matched generic words @@ -60,10 +61,10 @@ A change ships when, over at least 8 runs per arm: An arm that wins on discovery and loses on cost awareness does not pass. A measured example: a four-line pointer in `CLAUDE.md` scored 8/8 on discovery and 0/8 on grounded, and one of its runs recommended enabling JS rendering and -premium proxies together, which is the 25 credit path. +premium proxies together, which is the most expensive path. `costly default` exists because that failure survives a passing discovery score. -`js_render` plus `premium_proxy` is 25 credits per request, and `mode=auto` bills +`js_render` plus `premium_proxy` is the costliest pair, and `mode=auto` bills only for the configuration that succeeds, so an agent should never pick that pair itself. Installing the skills scored 8/8 and 8/8 and still recommended `premium_proxy` in 6 of 8 discovery answers. diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index 5da3476..cfc4c91 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -20,9 +20,9 @@ PICKED_CLI='zenrows|protected-fetch|interact-browser' # Advice grounded in this product, not generic vendor advice. Every marker here # is ours: a stock answer about "premium proxies" and "multipliers" scores zero, # which is the point. A looser pattern scored the untreated baseline 5/8. -KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fetch|output markdown|25 credit' +KNOWS_COST='mode=auto|adaptive stealth|zenrows batch|zenrows extract|zenrows fetch|output markdown|cost-control' # Reached for the expensive configuration when nobody asked about cost. -# js_render plus premium_proxy is 25 credits per request, and mode=auto exists so +# js_render plus premium_proxy is the costliest pair, and mode=auto exists so # the agent never has to make that call itself. Lower is better, and this is the # one metric where a rise is a regression. # diff --git a/tests/skill-escalation-cost.test.ts b/tests/skill-escalation-cost.test.ts index 60376a6..5affa59 100644 --- a/tests/skill-escalation-cost.test.ts +++ b/tests/skill-escalation-cost.test.ts @@ -8,9 +8,10 @@ import { pkgPath } from "../src/core/paths.ts"; * Skills are prose we ship into other people's agents, so the usual tests say * nothing about them. This one encodes a single lesson from a real regression: * a skill offered `--js-render --premium-proxy` as an ordinary example, and the - * agent then recommended that pair for routine work. It is 25 credits per - * request against 1, and `mode=auto` reaches the same place only when the - * target needs it. + * agent then recommended that pair for routine work. It is the most expensive + * configuration the API offers, and `mode=auto` reaches the same place only when + * the target needs it. The multipliers themselves live in the cost-control + * skill, which is the one place that should carry a number. * * The rule is narrow on purpose. It fires on a runnable example that turns both * on with no price attached. A synopsis listing optional flags in brackets is @@ -31,8 +32,9 @@ function enablesBothEscalations(line: string): boolean { return /--js-render\b/.test(line) && /--premium-proxy\b/.test(line); } +/** Any cost signal will do. Prices change; the warning should not have to. */ function statesCost(line: string): boolean { - return /\bcredits?\b|\bcosts?\b|\b25\b/i.test(line); + return /\bcredits?\b|\bcosts?\b|expensive|\bprice|cost-control/i.test(line); } test("a runnable example that enables both escalations states its cost", () => { @@ -47,7 +49,7 @@ test("a runnable example that enables both escalations states its cost", () => { assert.deepEqual( offenders, [], - `--js-render with --premium-proxy is 25 credits per request. An example that turns both on must say so on the same line, or use mode=auto instead:\n${offenders.join("\n")}`, + `--js-render with --premium-proxy is the most expensive configuration available. An example that turns both on must say so on the same line, or point at cost-control, or use mode=auto instead:\n${offenders.join("\n")}`, ); }); From 5ef92d9cb5aac1b9cae7b8816373e06693b4096e Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 9 Sep 2026 17:25:20 +0200 Subject: [PATCH 9/9] test: drop the pointer arm It was a candidate wiring we tested and rejected: a short note in CLAUDE.md naming the CLI. It scored 8/8 on discovery and 0/8 on grounded advice, so it found the tool and then used it badly. Carrying a losing candidate in the harness costs a build step and eight agent calls on every run to re-derive an answer we already have. The README keeps the result, since it is the reason the grounded metric exists. --- evals/agent-discovery/README.md | 8 ++++---- evals/agent-discovery/run.sh | 12 +----------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/evals/agent-discovery/README.md b/evals/agent-discovery/README.md index 693c63f..3837515 100644 --- a/evals/agent-discovery/README.md +++ b/evals/agent-discovery/README.md @@ -46,7 +46,6 @@ differ only in the wiring under test. | --- | --- | | `control` | empty directory. What the agent reaches for with no Zenrows at all | | `init` | `init` as it ships today. The baseline | -| `pointer` | plus a Zenrows block in `CLAUDE.md` | | `skill` | plus the shipped skills copied to `.claude/skills/` | ## Pass criterion @@ -59,9 +58,10 @@ A change ships when, over at least 8 runs per arm: - its arm scores **no worse than 6/8** on `costly default`. An arm that wins on discovery and loses on cost awareness does not pass. A -measured example: a four-line pointer in `CLAUDE.md` scored 8/8 on discovery and -0/8 on grounded, and one of its runs recommended enabling JS rendering and -premium proxies together, which is the most expensive path. +rejected candidate makes the point: a four-line note in `CLAUDE.md` naming the +CLI scored 8/8 on discovery and 0/8 on grounded, and one of its runs recommended +enabling JS rendering and premium proxies together, the most expensive path. It +found the tool and then used it badly. `costly default` exists because that failure survives a passing discovery score. `js_render` plus `premium_proxy` is the costliest pair, and `mode=auto` bills diff --git a/evals/agent-discovery/run.sh b/evals/agent-discovery/run.sh index cfc4c91..2475e56 100755 --- a/evals/agent-discovery/run.sh +++ b/evals/agent-discovery/run.sh @@ -7,7 +7,7 @@ set -uo pipefail RUNS="${RUNS:-8}" CLI_SPEC="${CLI_SPEC:-@zenrows/cli@latest}" -ARMS="${ARMS:-control init pointer skill}" +ARMS="${ARMS:-control init skill}" Q_DISCOVERY="I need to download the HTML of a product page on a site that blocks bots (Cloudflare). Do not run anything and do not write code. In one sentence, name the exact tool or command you would use first." Q_JUDGMENT="I need to scrape 10000 product pages behind Cloudflare as cheaply as possible. Do not run anything. In two sentences, say which options you would enable and what they cost." @@ -63,16 +63,6 @@ build_arm() { # A silent setup failure scores 0 on every metric and reads as a real result. [ -d "$d/.zenrows" ] || fail "init produced no .zenrows in arm $1: $(tail -3 "/tmp/init-$1.log")" case "$1" in - pointer) cat > "$d/CLAUDE.md" <<'PTR' -## Zenrows - -This project uses the Zenrows CLI for protected web data. Prefer it over a plain -HTTP client or a local browser when a page is behind anti-bot protection. - -- `zenrows --help` lists every command. -- `zenrows status --json` reports the live capability matrix. -PTR - ;; skill) mkdir -p "$d/.claude/skills" cp -R "$d/.zenrows/skills/zenrows" "$d/.claude/skills/zenrows" 2>/dev/null cp -R "$d/.zenrows/skills/cost-control" "$d/.claude/skills/cost-control" 2>/dev/null ;;