From b894b98b9c388f4b620ee7ff24bb831dfd5d3645 Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 25 Aug 2026 22:57:41 -0400 Subject: [PATCH 1/3] test(pstack): fail when babysit drops bugbot triage --- tests/skill-collision-repro.sh | 121 +++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/tests/skill-collision-repro.sh b/tests/skill-collision-repro.sh index 41f9372..744ce75 100755 --- a/tests/skill-collision-repro.sh +++ b/tests/skill-collision-repro.sh @@ -104,6 +104,127 @@ else note "ok: default model quad identical across provider dispatch + 4 panel skills + setup-pstack ($canon_quad)" fi +# Dual Babysit entry points share one bugbot-triage.md. A second copy or a +# dropped relative path lets standalone /babysit apply a weaker policy. +bugbot_skill_rel="../poteto-mode/references/bugbot-triage.md" +bugbot_playbook_rel="../references/bugbot-triage.md" + +check_bugbot_triage_binding() { + local plugin="$1" + local bad="" + local canon="$plugin/skills/poteto-mode/references/bugbot-triage.md" + local skill="$plugin/skills/babysit/SKILL.md" + local playbook="$plugin/skills/poteto-mode/playbooks/babysit.md" + local copies n canon_abs resolved_dir resolved_abs + + if [ ! -f "$canon" ]; then + bad="${bad}canonical rubric missing: $canon"$'\n' + fi + if [ ! -f "$skill" ]; then + bad="${bad}standalone babysit skill missing: $skill"$'\n' + elif ! grep -Fq "$bugbot_skill_rel" "$skill"; then + bad="${bad}standalone babysit skill lost bugbot-triage binding ($bugbot_skill_rel)"$'\n' + fi + if [ ! -f "$playbook" ]; then + bad="${bad}poteto-mode babysit playbook missing: $playbook"$'\n' + elif ! grep -Fq "$bugbot_playbook_rel" "$playbook"; then + bad="${bad}poteto-mode babysit playbook lost bugbot-triage binding ($bugbot_playbook_rel)"$'\n' + fi + if [ ! -f "$plugin/skills/babysit/$bugbot_skill_rel" ]; then + bad="${bad}standalone relative link does not resolve: $bugbot_skill_rel"$'\n' + elif [ -f "$canon" ]; then + canon_abs="$(cd "$(dirname "$canon")" && pwd)/$(basename "$canon")" + resolved_dir="$(cd "$plugin/skills/babysit/$(dirname "$bugbot_skill_rel")" && pwd)" + resolved_abs="$resolved_dir/$(basename "$bugbot_skill_rel")" + if [ "$canon_abs" != "$resolved_abs" ]; then + bad="${bad}standalone relative link resolves to $resolved_abs, not $canon_abs"$'\n' + fi + fi + copies="$(find "$plugin" -name 'bugbot-triage.md' -print 2>/dev/null || true)" + n="$(printf '%s\n' "$copies" | awk 'NF { c++ } END { print c+0 }')" + if [ "$n" != "1" ]; then + bad="${bad}expected exactly 1 bugbot-triage.md under plugin, found $n"$'\n' + if [ -n "$copies" ]; then + bad="${bad}$copies"$'\n' + fi + fi + if [ -n "$bad" ]; then + printf '%s' "$bad" + return 1 + fi + return 0 +} + +write_valid_bugbot_plugin() { + local plugin="$1" + mkdir -p "$plugin/skills/babysit" \ + "$plugin/skills/poteto-mode/playbooks" \ + "$plugin/skills/poteto-mode/references" + printf '%s\n' "per [$bugbot_skill_rel]($bugbot_skill_rel)" > "$plugin/skills/babysit/SKILL.md" + printf '%s\n' "per \`$bugbot_playbook_rel\`" > "$plugin/skills/poteto-mode/playbooks/babysit.md" + printf '%s\n' "# rubric" > "$plugin/skills/poteto-mode/references/bugbot-triage.md" +} + +expect_bugbot_ok() { + local label="$1" + local plugin="$2" + local bad="" + local rc=0 + bad="$(check_bugbot_triage_binding "$plugin")" || rc=$? + if [ "$rc" -eq 0 ]; then + note "ok: $label" + else + note "FAIL: $label" + note "$bad" + fail=1 + fi +} + +expect_bugbot_fail() { + local label="$1" + local plugin="$2" + local rc=0 + check_bugbot_triage_binding "$plugin" >/dev/null || rc=$? + if [ "$rc" -eq 0 ]; then + note "FAIL: $label: expected a binding error, check passed" + fail=1 + else + note "ok: $label" + fi +} + +expect_bugbot_ok "babysit Bugbot binding on the packaged plugin" "$repo/plugins/pstack" + +bugbot_fx="$(mktemp -d)" +write_valid_bugbot_plugin "$bugbot_fx/ok" +expect_bugbot_ok "valid babysit Bugbot fixture" "$bugbot_fx/ok" + +write_valid_bugbot_plugin "$bugbot_fx/missing-canon" +rm -f "$bugbot_fx/missing-canon/skills/poteto-mode/references/bugbot-triage.md" +expect_bugbot_fail "canonical rubric absent" "$bugbot_fx/missing-canon" + +write_valid_bugbot_plugin "$bugbot_fx/skill-unbound" +printf '%s\n' "no rubric link" > "$bugbot_fx/skill-unbound/skills/babysit/SKILL.md" +expect_bugbot_fail "standalone babysit lost the Bugbot binding" "$bugbot_fx/skill-unbound" + +write_valid_bugbot_plugin "$bugbot_fx/playbook-unbound" +printf '%s\n' "no rubric link" > "$bugbot_fx/playbook-unbound/skills/poteto-mode/playbooks/babysit.md" +expect_bugbot_fail "poteto-mode babysit playbook lost the Bugbot binding" "$bugbot_fx/playbook-unbound" + +write_valid_bugbot_plugin "$bugbot_fx/broken-link" +rm -f "$bugbot_fx/broken-link/skills/poteto-mode/references/bugbot-triage.md" +mkdir -p "$bugbot_fx/broken-link/skills/poteto-mode/other" +printf '%s\n' "# rubric" > "$bugbot_fx/broken-link/skills/poteto-mode/other/bugbot-triage.md" +expect_bugbot_fail "standalone relative Bugbot link does not resolve" "$bugbot_fx/broken-link" + +write_valid_bugbot_plugin "$bugbot_fx/extra-copy" +mkdir -p "$bugbot_fx/extra-copy/skills/babysit/references" +cp "$bugbot_fx/extra-copy/skills/poteto-mode/references/bugbot-triage.md" \ + "$bugbot_fx/extra-copy/skills/babysit/references/bugbot-triage.md" +expect_bugbot_fail "extra bugbot-triage.md copy under the plugin" "$bugbot_fx/extra-copy" + +rm -rf "$bugbot_fx" + if [ "${PSTACK_STATIC_ONLY:-0}" = "1" ]; then exit "$fail" fi From db4723755b97a72540f7d34f369e0a0b54ead72f Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 25 Aug 2026 22:57:41 -0400 Subject: [PATCH 2/3] fix(pstack): bind standalone babysit to bugbot triage --- plugins/pstack/skills/babysit/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/pstack/skills/babysit/SKILL.md b/plugins/pstack/skills/babysit/SKILL.md index df1bc0f..39bdafe 100644 --- a/plugins/pstack/skills/babysit/SKILL.md +++ b/plugins/pstack/skills/babysit/SKILL.md @@ -29,6 +29,7 @@ Inside poteto-mode, the **Babysit** playbook ([`../poteto-mode/playbooks/babysit - Merge conflicts (`mergeStateStatus == DIRTY`): rebase or merge `main`; resolve; force-push only if the branch is yours and not shared. - Failing checks (`statusCheckRollup` entries with `conclusion: FAILURE`): pull logs with `gh run view --log-failed`. Root-cause the failure; fix the underlying code or test; commit; push. - Review comments (`gh pr view --json comments,reviews`): act only on feedback you actually agree with. When a comment has a single mechanical answer — a rename, a guard clause, a formatting nit — make the edit and quote the comment in the commit message. When it hinges on a judgement call, or you can't tell what's being asked, don't guess: leave it and reply with what you would have done. + - Review-bot comments (Bugbot and similar automation): classify as fix, dismiss, or ask before acting, per [`../poteto-mode/references/bugbot-triage.md`](../poteto-mode/references/bugbot-triage.md). Ask by default on security, data, and high-severity findings. 3. **Loop.** Use the Claude Code `loop` skill to pace re-checks. Pick the interval from what you're watching: - Active CI run: poll `gh pr checks --watch` (it blocks until checks finish, so no separate loop interval needed). From 4bf252ca7011d09db519a42524ce3e0a1f17fec7 Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 25 Aug 2026 23:18:53 -0400 Subject: [PATCH 3/3] fix(pstack): harden shared bugbot triage binding --- plugins/pstack/skills/babysit/SKILL.md | 4 +- tests/skill-collision-repro.sh | 155 +++++++------------------ 2 files changed, 42 insertions(+), 117 deletions(-) diff --git a/plugins/pstack/skills/babysit/SKILL.md b/plugins/pstack/skills/babysit/SKILL.md index 39bdafe..b816077 100644 --- a/plugins/pstack/skills/babysit/SKILL.md +++ b/plugins/pstack/skills/babysit/SKILL.md @@ -28,8 +28,8 @@ Inside poteto-mode, the **Babysit** playbook ([`../poteto-mode/playbooks/babysit 2. **Triage in priority order.** - Merge conflicts (`mergeStateStatus == DIRTY`): rebase or merge `main`; resolve; force-push only if the branch is yours and not shared. - Failing checks (`statusCheckRollup` entries with `conclusion: FAILURE`): pull logs with `gh run view --log-failed`. Root-cause the failure; fix the underlying code or test; commit; push. - - Review comments (`gh pr view --json comments,reviews`): act only on feedback you actually agree with. When a comment has a single mechanical answer — a rename, a guard clause, a formatting nit — make the edit and quote the comment in the commit message. When it hinges on a judgement call, or you can't tell what's being asked, don't guess: leave it and reply with what you would have done. - - Review-bot comments (Bugbot and similar automation): classify as fix, dismiss, or ask before acting, per [`../poteto-mode/references/bugbot-triage.md`](../poteto-mode/references/bugbot-triage.md). Ask by default on security, data, and high-severity findings. + - Review comments from human reviewers (`gh pr view --json comments,reviews`): act only on feedback you actually agree with. When a comment has a single mechanical answer — a rename, a guard clause, a formatting nit — make the edit and quote the comment in the commit message. When it hinges on a judgement call, or you can't tell what's being asked, don't guess: leave it and reply with what you would have done. + - Review-bot comments (Bugbot and similar automation): classify as fix, dismiss, or ask before acting, per [`../poteto-mode/references/bugbot-triage.md`](../poteto-mode/references/bugbot-triage.md). Follow the rubric's Ask by default categories, including security, data, and high-severity findings. 3. **Loop.** Use the Claude Code `loop` skill to pace re-checks. Pick the interval from what you're watching: - Active CI run: poll `gh pr checks --watch` (it blocks until checks finish, so no separate loop interval needed). diff --git a/tests/skill-collision-repro.sh b/tests/skill-collision-repro.sh index 744ce75..1b8a7be 100755 --- a/tests/skill-collision-repro.sh +++ b/tests/skill-collision-repro.sh @@ -104,126 +104,51 @@ else note "ok: default model quad identical across provider dispatch + 4 panel skills + setup-pstack ($canon_quad)" fi -# Dual Babysit entry points share one bugbot-triage.md. A second copy or a -# dropped relative path lets standalone /babysit apply a weaker policy. +plugin="$repo/plugins/pstack" +canon="$plugin/skills/poteto-mode/references/bugbot-triage.md" +skill="$plugin/skills/babysit/SKILL.md" +playbook="$plugin/skills/poteto-mode/playbooks/babysit.md" bugbot_skill_rel="../poteto-mode/references/bugbot-triage.md" bugbot_playbook_rel="../references/bugbot-triage.md" - -check_bugbot_triage_binding() { - local plugin="$1" - local bad="" - local canon="$plugin/skills/poteto-mode/references/bugbot-triage.md" - local skill="$plugin/skills/babysit/SKILL.md" - local playbook="$plugin/skills/poteto-mode/playbooks/babysit.md" - local copies n canon_abs resolved_dir resolved_abs - - if [ ! -f "$canon" ]; then - bad="${bad}canonical rubric missing: $canon"$'\n' - fi - if [ ! -f "$skill" ]; then - bad="${bad}standalone babysit skill missing: $skill"$'\n' - elif ! grep -Fq "$bugbot_skill_rel" "$skill"; then - bad="${bad}standalone babysit skill lost bugbot-triage binding ($bugbot_skill_rel)"$'\n' - fi - if [ ! -f "$playbook" ]; then - bad="${bad}poteto-mode babysit playbook missing: $playbook"$'\n' - elif ! grep -Fq "$bugbot_playbook_rel" "$playbook"; then - bad="${bad}poteto-mode babysit playbook lost bugbot-triage binding ($bugbot_playbook_rel)"$'\n' - fi - if [ ! -f "$plugin/skills/babysit/$bugbot_skill_rel" ]; then - bad="${bad}standalone relative link does not resolve: $bugbot_skill_rel"$'\n' - elif [ -f "$canon" ]; then - canon_abs="$(cd "$(dirname "$canon")" && pwd)/$(basename "$canon")" - resolved_dir="$(cd "$plugin/skills/babysit/$(dirname "$bugbot_skill_rel")" && pwd)" - resolved_abs="$resolved_dir/$(basename "$bugbot_skill_rel")" - if [ "$canon_abs" != "$resolved_abs" ]; then - bad="${bad}standalone relative link resolves to $resolved_abs, not $canon_abs"$'\n' - fi - fi - copies="$(find "$plugin" -name 'bugbot-triage.md' -print 2>/dev/null || true)" - n="$(printf '%s\n' "$copies" | awk 'NF { c++ } END { print c+0 }')" - if [ "$n" != "1" ]; then - bad="${bad}expected exactly 1 bugbot-triage.md under plugin, found $n"$'\n' - if [ -n "$copies" ]; then - bad="${bad}$copies"$'\n' - fi - fi - if [ -n "$bad" ]; then - printf '%s' "$bad" - return 1 +bugbot_bad="" +if [ ! -f "$canon" ]; then + bugbot_bad="${bugbot_bad}canonical rubric missing: $canon"$'\n' +fi +skill_op="$(grep -F 'Review-bot comments (Bugbot and similar automation):' "$skill" || true)" +skill_n="$(printf '%s\n' "$skill_op" | awk 'NF { c++ } END { print c+0 }')" +if [ "$skill_n" != "1" ]; then + bugbot_bad="${bugbot_bad}standalone babysit skill lost bugbot-triage operational line"$'\n' +else + skill_dest="$(printf '%s\n' "$skill_op" | sed -n 's/.*](\([^)]*\)).*/\1/p')" + if [ "$skill_dest" != "$bugbot_skill_rel" ]; then + bugbot_bad="${bugbot_bad}standalone babysit Markdown destination is [$skill_dest], not [$bugbot_skill_rel]"$'\n' fi - return 0 -} - -write_valid_bugbot_plugin() { - local plugin="$1" - mkdir -p "$plugin/skills/babysit" \ - "$plugin/skills/poteto-mode/playbooks" \ - "$plugin/skills/poteto-mode/references" - printf '%s\n' "per [$bugbot_skill_rel]($bugbot_skill_rel)" > "$plugin/skills/babysit/SKILL.md" - printf '%s\n' "per \`$bugbot_playbook_rel\`" > "$plugin/skills/poteto-mode/playbooks/babysit.md" - printf '%s\n' "# rubric" > "$plugin/skills/poteto-mode/references/bugbot-triage.md" -} - -expect_bugbot_ok() { - local label="$1" - local plugin="$2" - local bad="" - local rc=0 - bad="$(check_bugbot_triage_binding "$plugin")" || rc=$? - if [ "$rc" -eq 0 ]; then - note "ok: $label" - else - note "FAIL: $label" - note "$bad" - fail=1 + if ! printf '%s\n' "$skill_op" | grep -Fq 'classify as fix, dismiss, or ask'; then + bugbot_bad="${bugbot_bad}standalone babysit lost fix/dismiss/ask classification"$'\n' fi -} - -expect_bugbot_fail() { - local label="$1" - local plugin="$2" - local rc=0 - check_bugbot_triage_binding "$plugin" >/dev/null || rc=$? - if [ "$rc" -eq 0 ]; then - note "FAIL: $label: expected a binding error, check passed" - fail=1 - else - note "ok: $label" + if ! printf '%s\n' "$skill_op" | grep -Fq "Follow the rubric's Ask by default categories, including security, data, and high-severity findings."; then + bugbot_bad="${bugbot_bad}standalone babysit lost ask-by-default escalation"$'\n' fi -} - -expect_bugbot_ok "babysit Bugbot binding on the packaged plugin" "$repo/plugins/pstack" - -bugbot_fx="$(mktemp -d)" -write_valid_bugbot_plugin "$bugbot_fx/ok" -expect_bugbot_ok "valid babysit Bugbot fixture" "$bugbot_fx/ok" - -write_valid_bugbot_plugin "$bugbot_fx/missing-canon" -rm -f "$bugbot_fx/missing-canon/skills/poteto-mode/references/bugbot-triage.md" -expect_bugbot_fail "canonical rubric absent" "$bugbot_fx/missing-canon" - -write_valid_bugbot_plugin "$bugbot_fx/skill-unbound" -printf '%s\n' "no rubric link" > "$bugbot_fx/skill-unbound/skills/babysit/SKILL.md" -expect_bugbot_fail "standalone babysit lost the Bugbot binding" "$bugbot_fx/skill-unbound" - -write_valid_bugbot_plugin "$bugbot_fx/playbook-unbound" -printf '%s\n' "no rubric link" > "$bugbot_fx/playbook-unbound/skills/poteto-mode/playbooks/babysit.md" -expect_bugbot_fail "poteto-mode babysit playbook lost the Bugbot binding" "$bugbot_fx/playbook-unbound" - -write_valid_bugbot_plugin "$bugbot_fx/broken-link" -rm -f "$bugbot_fx/broken-link/skills/poteto-mode/references/bugbot-triage.md" -mkdir -p "$bugbot_fx/broken-link/skills/poteto-mode/other" -printf '%s\n' "# rubric" > "$bugbot_fx/broken-link/skills/poteto-mode/other/bugbot-triage.md" -expect_bugbot_fail "standalone relative Bugbot link does not resolve" "$bugbot_fx/broken-link" - -write_valid_bugbot_plugin "$bugbot_fx/extra-copy" -mkdir -p "$bugbot_fx/extra-copy/skills/babysit/references" -cp "$bugbot_fx/extra-copy/skills/poteto-mode/references/bugbot-triage.md" \ - "$bugbot_fx/extra-copy/skills/babysit/references/bugbot-triage.md" -expect_bugbot_fail "extra bugbot-triage.md copy under the plugin" "$bugbot_fx/extra-copy" - -rm -rf "$bugbot_fx" +fi +playbook_op="$(grep -E '^8\. \*\*Bugbot is triaged skeptically, always\.\*\*' "$playbook" || true)" +playbook_n="$(printf '%s\n' "$playbook_op" | awk 'NF { c++ } END { print c+0 }')" +if [ "$playbook_n" != "1" ]; then + bugbot_bad="${bugbot_bad}poteto-mode babysit playbook lost step-8 Bugbot operational line"$'\n' +elif ! printf '%s\n' "$playbook_op" | grep -Fq "$bugbot_playbook_rel"; then + bugbot_bad="${bugbot_bad}poteto-mode babysit playbook step 8 lost bugbot-triage binding ($bugbot_playbook_rel)"$'\n' +fi +copies="$(find "$plugin" -name 'bugbot-triage.md' ! -path '*/node_modules/*' -print 2>/dev/null || true)" +n="$(printf '%s\n' "$copies" | awk 'NF { c++ } END { print c+0 }')" +if [ "$n" != "1" ]; then + bugbot_bad="${bugbot_bad}expected exactly 1 bugbot-triage.md under plugin, found $n"$'\n' +fi +if [ -n "$bugbot_bad" ]; then + note "FAIL: babysit Bugbot binding on the packaged plugin" + note "$bugbot_bad" + fail=1 +else + note "ok: babysit Bugbot binding on the packaged plugin" +fi if [ "${PSTACK_STATIC_ONLY:-0}" = "1" ]; then exit "$fail"