fix(test-writing): degrade malformed defense stances and add deterministic evidence gates

plugin-tests/test-writing/verify_finding_evidence.bats

@@ -0,0 +1,192 @@
+#!/usr/bin/env bats
+# bats file_tags=test-writing,team-review,verify-finding-evidence
+# Tests for verify-finding-evidence.sh — the merge/adversarial-stage gate that
+# checks every kept finding's `current` block against the reviewed file's real
+# content (whitespace-normalized substring containment) and demotes a finding
+# that quotes code the file does not contain from its kept bucket into
+# `contested`, rather than letting a fabricated quote reach the report.
+bats_require_minimum_version 1.11.0
+
+load 'test_helper/common_setup'
+
+setup() {
+    WORKFLOW_DIR="${PLUGIN_DIR}/skills/phpunit-test-team-reviewing/workflow"
+    SCRIPT="${WORKFLOW_DIR}/verify-finding-evidence.sh"
+    REPO_DIR="${BATS_TEST_TMPDIR}/repo"
+    mkdir -p "${REPO_DIR}/tests/unit"
+    # shellcheck source=/dev/null  # SCRIPT is derived from PLUGIN_DIR at runtime
+    source "${SCRIPT}"
+
+    _write_bar_test_file
+}
+
+# The fixture file every test's finding is checked against: a real test
+# method containing one real assertion.
+_write_bar_test_file() {
+    {
+        echo "<?php"
+        echo "class BarTest extends TestCase {"
+        echo "    public function testReal(): void {"
+        echo "        \$x = 1;"
+        echo "        static::assertSame(1, \$x);"
+        echo "    }"
+        echo "}"
+    } > "${REPO_DIR}/tests/unit/BarTest.php"
+}
+
+# Write a one-file result JSON with a single finding of the given `current`
+# in the given kept bucket (errors|warnings|informational).
+_write_result() {
+    local path="$1" bucket="$2" current="$3"
+    jq -n --arg bucket "${bucket}" --arg current "${current}" \
+        '{files: [{path: "tests/unit/BarTest.php",
+                   errors: [], warnings: [], informational: [], contested: []}
+                  | .[$bucket] = [{finding_id: "CONV-001|testReal", rule_id: "CONV-001", current: $current}]]}' \
+        > "${path}"
+}
+
+# ============================================================================
+# Happy path — a finding whose `current` is a real substring of the file is
+# left in place; a finding with empty `current` is exempt from the check.
+# ============================================================================
+
+@test "keeps a finding whose current matches the file exactly" {
+    # shellcheck disable=SC2016  # the literal $x is PHP code under test, not a shell expansion
+    _write_result "${BATS_TEST_TMPDIR}/result.json" errors 'static::assertSame(1, $x);'
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" ""
+
+    run jq -c '.files[0] | {errors: (.errors | length), contested: (.contested | length)}' <<< "${output}"
+    assert_output '{"errors":1,"contested":0}'
+}
+
+@test "exempts a finding with empty current from the check" {
+    jq -n '{files: [{path: "tests/unit/BarTest.php",
+                      errors: [{finding_id: "TEAM-SPLIT|class-level", rule_id: "TEAM-SPLIT", current: ""}],
+                      warnings: [], informational: [], contested: []}]}' > "${BATS_TEST_TMPDIR}/result.json"
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" ""
+
+    run jq -c '.files[0] | {errors: (.errors | length), contested: (.contested | length)}' <<< "${output}"
+    assert_output '{"errors":1,"contested":0}'
+}
+
+# ============================================================================
+# Whitespace normalization — a quote reformatted with different indentation
+# or line breaks still passes, since both sides collapse whitespace runs to
+# a single space before the containment check.
+# ============================================================================
+
+@test "matches under whitespace normalization despite different indentation" {
+    # shellcheck disable=SC2016  # the literal $x is PHP code under test, not a shell expansion
+    _write_result "${BATS_TEST_TMPDIR}/result.json" errors "$(printf 'static::assertSame(1,\n      $x);')"
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" ""
+
+    run jq -c '.files[0] | {errors: (.errors | length), contested: (.contested | length)}' <<< "${output}"
+    assert_output '{"errors":1,"contested":0}'
+}
+
+# ============================================================================
+# Demotion — a fabricated quote (code the file does not contain) is moved
+# from its kept bucket into `contested`, tagged with an `outcome` reason, and
+# logged to stderr.
+# ============================================================================
+
+@test "demotes a fabricated quote into contested with an outcome and a log line" {
+    # shellcheck disable=SC2016  # the literal $doesNotExist is PHP code under test, not a shell expansion
+    _write_result "${BATS_TEST_TMPDIR}/result.json" errors 'static::assertSame(999, $doesNotExist);'
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" "verify-finding-evidence: demoted CONV-001|testReal in tests/unit/BarTest.php: current block not found under whitespace normalization"
+
+    run jq -c '.files[0] | {errors: (.errors | length), contested}' <<< "${output}"
+    assert_output --partial '"errors":0'
+    assert_output --partial '"finding_id":"CONV-001|testReal"'
+    assert_output --partial '"outcome":"evidence check: current block not found in tests/unit/BarTest.php under whitespace normalization"'
+}
+
+@test "syncs a demotion into adversarial_input, moving it from kept to contested there too" {
+    jq -n '{files: [{path: "tests/unit/BarTest.php",
+                      errors: [{finding_id: "CONV-001|testReal", rule_id: "CONV-001", current: "static::assertSame(999, $doesNotExist);"}],
+                      warnings: [], informational: [], contested: [],
+                      adversarial_input: {
+                        kept: [{finding_id: "CONV-001|testReal", rule_id: "CONV-001", current: "static::assertSame(999, $doesNotExist);"}],
+                        contested: []
+                      }}]}' > "${BATS_TEST_TMPDIR}/result.json"
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" "verify-finding-evidence: demoted CONV-001|testReal in tests/unit/BarTest.php: current block not found under whitespace normalization"
+
+    run jq -c '.files[0].adversarial_input | {kept: (.kept | length), contested}' <<< "${output}"
+    assert_output --partial '"kept":0'
+    assert_output --partial '"finding_id":"CONV-001|testReal"'
+    assert_output --partial '"outcome":"evidence check: current block not found in tests/unit/BarTest.php under whitespace normalization"'
+}
+
+# ============================================================================
+# Per-entry shape validation — a bucket that is not an array (e.g. `errors: {}`)
+# is a corrupted result, not zero candidates to skip past: it must fail hard
+# and name the entry and the offending field, never silently pass through as
+# a clean review.
+# ============================================================================
+
+@test "fails hard when errors is an object instead of an array, rather than silently passing through" {
+    jq -n '{files: [{path: "tests/unit/BarTest.php", errors: {}, warnings: [], informational: [], contested: []}]}' \
+        > "${BATS_TEST_TMPDIR}/result.json"
+
+    run verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "entry 0"
+    assert_output --partial "tests/unit/BarTest.php"
+    assert_output --partial "errors is not an array"
+}
+
+@test "demotes from the warnings bucket the same way as errors" {
+    _write_result "${BATS_TEST_TMPDIR}/result.json" warnings 'this code does not exist anywhere'
+
+    run --separate-stderr verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_success
+
+    run jq -c '.files[0] | {warnings: (.warnings | length), contested: (.contested | length)}' <<< "${output}"
+    assert_output '{"warnings":0,"contested":1}'
+}
+
+# ============================================================================
+# Hard failures — a target file that does not exist, or invalid result JSON,
+# both abort with a non-zero exit and a clear message; never a silent skip.
+# ============================================================================
+
+@test "fails hard when the referenced file does not exist on disk" {
+    jq -n '{files: [{path: "tests/unit/NoSuch.php",
+                      errors: [{finding_id: "CONV-001|testX", rule_id: "CONV-001", current: "something"}],
+                      warnings: [], informational: [], contested: []}]}' > "${BATS_TEST_TMPDIR}/result.json"
+
+    run verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "target file not found"
+}
+
+@test "fails hard on invalid result JSON" {
+    printf '%s\n' '{bad json' > "${BATS_TEST_TMPDIR}/result.json"
+
+    run verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "not valid JSON"
+}
+
+@test "fails hard when the result has no top-level files array" {
+    printf '%s\n' '{}' > "${BATS_TEST_TMPDIR}/result.json"
+
+    run verify_finding_evidence "${BATS_TEST_TMPDIR}/result.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "no top-level"
+}

plugin-tests/test-writing/verify_method_counts.bats

@@ -0,0 +1,154 @@
+#!/usr/bin/env bats
+# bats file_tags=test-writing,team-review,verify-method-counts
+# Tests for verify-method-counts.sh — the Phase-1 manifest gate that
+# deterministically re-counts each entry's test methods from disk before the
+# manifest freezes, replacing a subagent-reported mismatch with the extracted
+# truth rather than merely warning about it.
+bats_require_minimum_version 1.11.0
+
+load 'test_helper/common_setup'
+
+setup() {
+    WORKFLOW_DIR="${PLUGIN_DIR}/skills/phpunit-test-team-reviewing/workflow"
+    SCRIPT="${WORKFLOW_DIR}/verify-method-counts.sh"
+    REPO_DIR="${BATS_TEST_TMPDIR}/repo"
+    mkdir -p "${REPO_DIR}/tests/unit"
+    # shellcheck source=/dev/null  # SCRIPT is derived from PLUGIN_DIR at runtime
+    source "${SCRIPT}"
+}
+
+# Write a minimal test-class fixture with the given (bare) test method names.
+_write_test_class() {
+    local path="$1"
+    shift
+    {
+        echo "<?php"
+        echo "class Fixture extends TestCase {"
+        local name
+        for name in "$@"; do
+            echo "    public function ${name}(): void {}"
+        done
+        echo "}"
+    } > "${path}"
+}
+
+_write_manifest() {
+    local path="$1" test_file="$2" method_count="$3" methods_json="$4"
+    jq -n --arg path "${test_file}" --argjson count "${method_count}" --argjson methods "${methods_json}" \
+        '[{path: $path, method_count: $count, test_methods: $methods}]' > "${path}"
+}
+
+# ============================================================================
+# Happy path — the manifest already matches the extracted truth: no change,
+# no stderr log line.
+# ============================================================================
+
+@test "leaves a matching entry unchanged and logs nothing" {
+    _write_test_class "${REPO_DIR}/tests/unit/FooTest.php" testAlpha testBeta testGamma
+    _write_manifest "${BATS_TEST_TMPDIR}/manifest.json" "tests/unit/FooTest.php" 3 '["testAlpha","testBeta","testGamma"]'
+
+    run --separate-stderr verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" ""
+
+    run jq -c '.[0] | {method_count, test_methods}' <<< "${output}"
+    assert_output '{"method_count":3,"test_methods":["testAlpha","testBeta","testGamma"]}'
+}
+
+# ============================================================================
+# Mismatch — a subagent under- or over-counted; the script replaces both
+# fields with the extracted truth and logs exactly one line naming the file,
+# the old count, and the new count.
+# ============================================================================
+
+@test "corrects a mismatched entry, logs one line, and writes the extracted truth" {
+    _write_test_class "${REPO_DIR}/tests/unit/FooTest.php" testAlpha testBeta testGamma
+    _write_manifest "${BATS_TEST_TMPDIR}/manifest.json" "tests/unit/FooTest.php" 2 '["testAlpha","testBeta"]'
+
+    run --separate-stderr verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" "verify-method-counts: tests/unit/FooTest.php: method_count 2 -> 3"
+
+    run jq -c '.[0] | {method_count, test_methods}' <<< "${output}"
+    assert_output '{"method_count":3,"test_methods":["testAlpha","testBeta","testGamma"]}'
+}
+
+# ============================================================================
+# Hard failures — a missing entry file, or invalid manifest JSON, both abort
+# with a non-zero exit and a clear message; never a silent skip.
+# ============================================================================
+
+@test "fails hard when an entry's file cannot be read, rather than reporting a false zero count" {
+    _write_test_class "${REPO_DIR}/tests/unit/FooTest.php" testAlpha testBeta
+    chmod 000 "${REPO_DIR}/tests/unit/FooTest.php"
+    if [[ -r "${REPO_DIR}/tests/unit/FooTest.php" ]]; then
+        skip "running as a user that bypasses file permissions (e.g. root) — chmod 000 did not deny read"
+    fi
+    _write_manifest "${BATS_TEST_TMPDIR}/manifest.json" "tests/unit/FooTest.php" 2 '["testAlpha","testBeta"]'
+
+    run verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "grep failed reading"
+
+    chmod 644 "${REPO_DIR}/tests/unit/FooTest.php"
+}
+
+@test "fails hard when an entry's file does not exist on disk" {
+    _write_manifest "${BATS_TEST_TMPDIR}/manifest.json" "tests/unit/NoSuchTest.php" 1 '["testX"]'
+
+    run verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "entry file not found"
+}
+
+@test "fails hard on invalid manifest JSON" {
+    printf '%s\n' '{bad json' > "${BATS_TEST_TMPDIR}/manifest.json"
+
+    run verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "not valid JSON"
+}
+
+@test "fails hard when the manifest top level is not an array" {
+    printf '%s\n' '{}' > "${BATS_TEST_TMPDIR}/manifest.json"
+
+    run verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "not an array"
+}
+
+# ============================================================================
+# Order-sensitivity — a reordered-but-identical-set test_methods list is still
+# a mismatch: the extracted file-order list is the truth, not merely the set.
+# ============================================================================
+
+# ============================================================================
+# Per-entry shape validation — a wrongly-typed field (method_count as a
+# string, test_methods as an object) is a corrupted manifest, not a mismatch
+# to "correct": it must fail hard and name the entry and the offending field,
+# never silently pass through with method_count 0.
+# ============================================================================
+
+@test "fails hard when method_count is a string and test_methods is an object, rather than silently correcting" {
+    _write_test_class "${REPO_DIR}/tests/unit/FooTest.php" testAlpha testBeta testGamma
+    jq -n '[{path: "tests/unit/FooTest.php", method_count: "three", test_methods: {}}]' > "${BATS_TEST_TMPDIR}/manifest.json"
+
+    run verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_failure
+    assert_output --partial "entry 0"
+    assert_output --partial "tests/unit/FooTest.php"
+    assert_output --partial "method_count is not a number"
+    assert_output --partial "test_methods is not an array"
+}
+
+@test "corrects a reordered test_methods list to file order even though the set matches" {
+    _write_test_class "${REPO_DIR}/tests/unit/FooTest.php" testAlpha testBeta
+    _write_manifest "${BATS_TEST_TMPDIR}/manifest.json" "tests/unit/FooTest.php" 2 '["testBeta","testAlpha"]'
+
+    run --separate-stderr verify_method_counts "${BATS_TEST_TMPDIR}/manifest.json" "${REPO_DIR}"
+    assert_success
+    assert_equal "${stderr}" "verify-method-counts: tests/unit/FooTest.php: method_count 2 -> 2"
+
+    run jq -c '.[0].test_methods' <<< "${output}"
+    assert_output '["testAlpha","testBeta"]'
+}

plugins/test-writing/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "test-writing",
-  "version": "5.1.0",
+  "version": "5.1.1",
   "description": "Generate and validate PHPUnit tests for Shopware 6. Unit tests: analyzes source classes to detect category (DTO, Service, Flow/Event, DAL, Exception) and applies category-specific templates. Migration tests: analyzes SQL operations to generate pattern-appropriate migration tests with 8 migration-specific rules. Integration tests: generates wired-up tests via phpunit-integration-test-generation for controller, message-handler, indexer, DAL-flow, and multi-service patterns (defers to unit generation when the SUT is unit-shape); 8 quality rules plus a placement smoke check via phpunit-integration-test-reviewing; a separate user-invoked phpunit-integration-to-unit-migrating skill audits placement with 8 deep-reasoning rules and migrates load-bearing-free tests to the unit suite. Workflow-based team review: wave-orchestrated agents coordinated through a shared blackboard (no agent-to-agent messaging), with adversarial red team and defense rounds. Bundles test-rules MCP server for Shopware compliance rules. Orchestrator runs inline fix loop (max 4 iterations) with oscillation detection. Optional dev-tooling plugin enables PHPStan/PHPUnit/ECS validation in the fix loop.",
   "author": {
     "name": "Shopware Labs"

plugins/test-writing/AGENTS.md

@@ -76,7 +76,9 @@ plugins/test-writing/
     │   ├── SKILL.md
     │   ├── workflow/team-review.workflow.mjs    # committed parameterized Workflow script (reads its manifest from `const manifest = args;`)
     │   ├── workflow/build-run-script.sh         # splices the on-disk manifest into a flat run-script; launched via scriptPath (no args)
-    │   └── references/{input-resolution,workflow-design,agent-guardrails,reviewer-allocation,red-team-context,consensus-and-verdicts,report-format,error-handling}.md
+    │   ├── workflow/verify-method-counts.sh     # deterministic Phase-1 gate: re-counts test methods, corrects manifest entries
+    │   ├── workflow/verify-finding-evidence.sh  # deterministic Phase-5 gate: demotes findings whose `current` block is not in the file
+    │   └── references/{input-resolution,workflow-design,agent-guardrails,reviewer-allocation,red-team-context,consensus-and-verdicts,report-format,error-handling,fix-application}.md
     ├── phpunit-migration-test-generation/
     │   ├── SKILL.md
     │   ├── references/{source-analysis,output-format}.md