diff --git a/.github/scripts/sarif.sh b/.github/scripts/sarif.sh new file mode 100644 index 0000000000..60428651ae --- /dev/null +++ b/.github/scripts/sarif.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +sarif_source_modules() { + grep -oE '\.\./[^<]+' ddk-parent/pom.xml \ + | sed -E 's#.*\.\./([^<]+)#\1#' \ + | while IFS= read -r module; do + if [ -f "$module/META-INF/MANIFEST.MF" ] && [ -d "$module/src" ]; then + echo "$module" + fi + done +} + +validate_sarif() { + local report=$1 analyzer=${2:?expected analyzer required} + if ! jq -se --arg analyzer "$analyzer" ' + def valid_base($run; $seen): + . as $id | type == "string" and ($seen | index($id) | not) + and ($run.originalUriBaseIds[$id] | type == "object") + and ($run.originalUriBaseIds[$id] | + if has("uriBaseId") then .uriBaseId | valid_base($run; $seen + [$id]) + else (.uri | type == "string") end); + def optional_array($key): (has($key) | not) or (.[$key] | type == "array"); + length == 1 and (.[0] | + type == "object" and .version == "2.1.0" + and (.runs | type == "array" and length > 0) + and all(.runs[]; + (.tool.driver.name == $analyzer) + and (.results | type == "array") + and (. as $run | all(.. | objects | select(has("uriBaseId")); .uriBaseId | valid_base($run; []))) + and optional_array("invocations") + and ($analyzer == "Checkstyle" or (.invocations | type == "array" and length > 0)) + and all(.results[]?; + type == "object" and (.message | type == "object")) + and all(.invocations[]?; + type == "object" + and .executionSuccessful == true + and optional_array("toolExecutionNotifications") + and optional_array("toolConfigurationNotifications") + and all(.toolExecutionNotifications[]?, .toolConfigurationNotifications[]?; + type == "object" and .level != "error")))) + ' "$report" >/dev/null; then + echo "::error::${report} is missing, invalid SARIF, or reports unsuccessful analysis." + return 1 + fi +} + +merge_sarif() { + local report=$1 output=$2 modules=$3 analyzer=${4:?expected analyzer required} module + local inputs=() + for module in $modules; do + validate_sarif "${module}/target/${report}" "$analyzer" || return 1 + inputs+=("${module}/target/${report}") + done + if [ ${#inputs[@]} -eq 0 ]; then + echo "::error::No expected modules supplied for ${report}." + return 1 + fi + mkdir -p "$(dirname "$output")" + jq -s --arg root "${GITHUB_WORKSPACE:-$(pwd -P)}" ' + def file_path: + (if startswith("file://localhost/") then ltrimstr("file://localhost") + elif startswith("file:/") then sub("^file:/+"; "/") + elif startswith("/") then . else error("Expected a file URI") end) as $path + | reduce ($path | split("/"))[] as $part ([]; + if $part == "" or $part == "." then . + elif $part == ".." then + if length > 0 then .[:-1] else error("Source path escapes filesystem root") end + else . + [$part] end) + | "/" + join("/") + (if ($path | endswith("/")) and length > 0 then "/" else "" end); + def resolve_uri($run; $root): + . as $location + | (if has("uriBaseId") then $run.originalUriBaseIds[.uriBaseId] | resolve_uri($run; $root) + else $root end) as $base + | ($location.uri // "") as $uri + | if ($uri | startswith("/") or startswith("file:/")) then $uri | file_path + elif ($uri | test("^[A-Za-z][A-Za-z0-9+.-]*:")) then error("Unsupported source URI scheme") + elif $uri == "" then $base + else (($base | sub("[^/]*$"; "")) + $uri) | file_path end; + def repository_locations($root): + . as $run | del(.originalUriBaseIds) + | walk(if type == "object" and (has("uri") or has("uriBaseId")) then + (resolve_uri($run; $root)) as $absolute + | if ($absolute | startswith($root)) then + .uri = ($absolute | ltrimstr($root)) | del(.uriBaseId) + else error("Source location is outside the repository: " + $absolute) end + else . end); + def identical: + unique | if length == 1 then .[0] else error("Conflicting SARIF metadata") end; + def descriptors: + group_by(.id) | map(identical); + def compatible_run: + if ((keys - ["tool", "results", "invocations", "originalUriBaseIds", "taxonomies"]) | length) > 0 + or any(.. | objects; has("index") or has("invocationIndex")) + then error("Unsupported run metadata or indexed reference; update the merger") else . end; + def resolve_rules: + .tool.driver.rules as $rules + | .results |= map(if has("ruleIndex") then + .ruleIndex as $index + | if ($index | type) != "number" or $index < 0 or ($index | floor) != $index + or $rules[$index].id == null + or (has("ruleId") and .ruleId != $rules[$index].id) + then error("Invalid ruleIndex") + else .ruleId = $rules[$index].id | del(.ruleIndex) end + else . end); + ($root | split("/") | map(@uri) | join("/") | rtrimstr("/") + "/") as $root_uri + | [.[].runs[] | compatible_run | resolve_rules | repository_locations($root_uri)] as $runs + | { + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + version: "2.1.0", + runs: [{ + tool: ([$runs[].tool | del(.driver.rules)] | identical + | .driver.rules = ([$runs[].tool.driver.rules[]?] | descriptors)), + taxonomies: ([$runs[].taxonomies[]?] | group_by([.name, .guid]) + | map(. as $group | map(del(.taxa)) | identical + | .taxa = ([$group[].taxa[]?] | descriptors))), + results: [$runs[].results[]?], + invocations: [$runs[].invocations[]?] + }] + } + ' "${inputs[@]}" > "$output" +} + +cpd_count() { + local report=$1 valid count + if [ ! -s "$report" ]; then + echo "::error::${report} is missing or empty." >&2 + return 1 + fi + valid=$(xmllint --nonet --xpath \ + 'boolean(/*[local-name()="pmd-cpd"] and not(//*[local-name()="error"]) and not(/*/*[local-name()!="duplication" and local-name()!="file"]))' \ + "$report") || return 1 + if [ "$valid" != "true" ]; then + echo "::error::${report} is not a successful CPD report." >&2 + return 1 + fi + count=$(xmllint --nonet --xpath 'count(/*/*[local-name()="duplication"])' "$report") || return 1 + printf '%s\n' "$count" +} diff --git a/.github/tests/analysis/README.md b/.github/tests/analysis/README.md new file mode 100644 index 0000000000..5213c2b85c --- /dev/null +++ b/.github/tests/analysis/README.md @@ -0,0 +1,53 @@ +# Analysis regression checks + +Run `bash .github/tests/analysis/run.sh` from any directory. Requires Bash, +Mike Farah yq v4, jq, and xmllint (Debian/Ubuntu: `libxml2-utils`). The verify +workflow runs the same suite. Tests use temporary directories and extract the +merge and gate blocks directly from the current workflow. + +The clean fixtures were emitted by Checkstyle 14.1.0, PMD 7.27.0, and SpotBugs +4.10.4. The error fixtures use those renderers with injected processing failures; +PMD omits the notification level, so its unsuccessful invocation is essential. +Checkstyle emits no invocation metadata on clean runs. The workflow therefore +also preserves the exit status of each Maven invocation with `--fail-at-end`. + +The merger supports the fields emitted by these configured producers: tool, +results, invocations, originalUriBaseIds, and taxonomies. It resolves URI-base chains and rewrites artifact locations to repository-relative +URIs, resolves result rule indices before unioning rule descriptors, retains taxonomy +descriptors, and rejects conflicting descriptors or unsupported run/indexed +metadata rather than silently dropping it. It is not a general SARIF merger. + +CPD fixtures were generated by PMD 7.27.0 CpdAnalysis with a normal Java source, +two duplicate sources, and an unterminated string (a real lexical error). The +SpotBugs finding fixture comes from an actual Java 21 analysis of a null +dereference. Machine-specific absolute source roots are normalized to /checkout. + +Run `bash .github/tests/analysis/mutations.sh` to verify that deliberate removal +of completion, notification, XML, URI, merged-report and process-exit protections +is detected. This suite also runs automatically in the verify workflow. + +The suite exercises valid findings separately from report failures, with a clean +sibling present. It also checks completion metadata, parser failures, raw and +merged reports, scoped report lists, CPD XML errors and structural counts, chained +URI bases, encoded paths, secondary locations, rule identities and severity. + + +`scope.sh` exercises the actual scope script against `fixtures/scope.bundle`, a +5.4 KB Git bundle containing signed fixture commits. It covers single and multiple +changed modules, docs-only and source-less changes, mixed changes, shared config, +workflow changes, deletions and moves. Module a depends on b. The tests check skip +injection, idempotence, report lists, reactor arguments, empty scopes and failures. +The bundle preserves actual Git diff behavior without requiring signing keys or +creating unsigned commits in CI. Regenerate it by creating signed scenario commits +in a temporary repository and bundling their branch refs with `git bundle create`. +The base SARIF head has no scope script; this suite runs on the descendant heads. + + +The fork annotation probe demonstrated that preserving arbitrary `uriBaseId` +values was insufficient: GitHub accepted the report but stored package-relative +paths that did not match the source tree. The merger now resolves file URIs and +base chains before emitting repository-relative artifact URIs without a base ID. +The repository root is `GITHUB_WORKSPACE`, or the current directory locally. +Encoded paths and dot segments are covered; locations outside this checkout or +using unsupported schemes fail explicitly. This is intentional for these source +analyzers, which report repository sources. diff --git a/.github/tests/analysis/fixtures/checkstyle-clean.json b/.github/tests/analysis/fixtures/checkstyle-clean.json new file mode 100644 index 0000000000..3b8ec573e9 --- /dev/null +++ b/.github/tests/analysis/fixtures/checkstyle-clean.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "downloadUri": "https://github.com/checkstyle/checkstyle/releases/", + "fullName": "Checkstyle", + "informationUri": "https://checkstyle.org/", + "language": "en", + "name": "Checkstyle", + "organization": "Checkstyle", + "rules": [ + + ], + "semanticVersion": "14.1.0", + "version": "14.1.0" + } + }, + "results": [ + + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/cpd-clean.xml b/.github/tests/analysis/fixtures/cpd-clean.xml new file mode 100644 index 0000000000..32e7629be7 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-clean.xml @@ -0,0 +1,4 @@ + + + + diff --git a/.github/tests/analysis/fixtures/cpd-error.xml b/.github/tests/analysis/fixtures/cpd-error.xml new file mode 100644 index 0000000000..ea6c6202c3 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-error.xml @@ -0,0 +1,22 @@ + + + + net.sourceforge.pmd.lang.ast.LexException: Lexical error in file 'b/src/Bad.java' at line 1, column 38: <EOF> after : "\"unterminated; }" (in lexical state DEFAULT) + at net.sourceforge.pmd.lang.ast.InternalApiBridge.newLexException(InternalApiBridge.java:25) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:2386) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:22) + at net.sourceforge.pmd.cpd.impl.BaseTokenFilter.getNextToken(BaseTokenFilter.java:44) + at net.sourceforge.pmd.cpd.impl.CpdLexerBase.tokenize(CpdLexerBase.java:40) + at net.sourceforge.pmd.cpd.CpdLexer.tokenize(CpdLexer.java:29) + at net.sourceforge.pmd.cpd.CpdAnalysis.doTokenize(CpdAnalysis.java:150) + at net.sourceforge.pmd.cpd.CpdAnalysis.tokenizeFiles(CpdAnalysis.java:225) + at net.sourceforge.pmd.cpd.CpdAnalysis.findMatches(CpdAnalysis.java:198) + at net.sourceforge.pmd.cpd.CpdAnalysis.performAnalysis(CpdAnalysis.java:164) + at CpdProbe.main(CpdProbe.java:15) + at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) + at java.base/java.lang.reflect.Method.invoke(Method.java:580) + at jdk.compiler/com.sun.tools.javac.launcher.Main.execute(Main.java:484) + at jdk.compiler/com.sun.tools.javac.launcher.Main.run(Main.java:208) + at jdk.compiler/com.sun.tools.javac.launcher.Main.main(Main.java:135) + + diff --git a/.github/tests/analysis/fixtures/cpd-finding.xml b/.github/tests/analysis/fixtures/cpd-finding.xml new file mode 100644 index 0000000000..9ae55681e1 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-finding.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/.github/tests/analysis/fixtures/pmd-clean.json b/.github/tests/analysis/fixtures/pmd-clean.json new file mode 100644 index 0000000000..d92eadb63a --- /dev/null +++ b/.github/tests/analysis/fixtures/pmd-clean.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "PMD", + "version": "7.27.0", + "informationUri": "https://docs.pmd-code.org/latest/", + "rules": [] + } + }, + "results": [], + "invocations": [ + { + "executionSuccessful": true, + "toolConfigurationNotifications": [], + "toolExecutionNotifications": [] + } + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/pmd-error.json b/.github/tests/analysis/fixtures/pmd-error.json new file mode 100644 index 0000000000..e65d56146b --- /dev/null +++ b/.github/tests/analysis/fixtures/pmd-error.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "PMD", + "version": "7.27.0", + "informationUri": "https://docs.pmd-code.org/latest/", + "rules": [] + } + }, + "results": [], + "invocations": [ + { + "executionSuccessful": false, + "toolConfigurationNotifications": [], + "toolExecutionNotifications": [ + { + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "file://src/Bad.java" + } + } + } + ], + "message": { + "text": "RuntimeException: synthetic analyzer failure" + }, + "exception": { + "message": "java.lang.RuntimeException: synthetic analyzer failure\n\tat Probe.lambda$main$0(Probe.java:7)\n\tat net.sourceforge.pmd.util.BaseResultProducingCloseable.using(BaseResultProducingCloseable.java:66)\n\tat net.sourceforge.pmd.reporting.Report.buildReport(Report.java:262)\n\tat Probe.main(Probe.java:7)\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:580)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.execute(Main.java:484)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.run(Main.java:208)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.main(Main.java:135)\n" + } + } + ] + } + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/scope.bundle b/.github/tests/analysis/fixtures/scope.bundle new file mode 100644 index 0000000000..12d84f0fe1 Binary files /dev/null and b/.github/tests/analysis/fixtures/scope.bundle differ diff --git a/.github/tests/analysis/fixtures/spotbugs-clean.json b/.github/tests/analysis/fixtures/spotbugs-clean.json new file mode 100644 index 0000000000..1e6ebd918f --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-clean.json @@ -0,0 +1 @@ +{"version":"2.1.0","$schema":"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json","runs":[{"tool":{"extensions":[{"version":"4.10.4","name":"edu.umd.cs.findbugs.plugins.core","shortDescription":{"text":"Core SpotBugs plugin"},"informationUri":"https://github.com/spotbugs","organization":"SpotBugs project"}],"driver":{"name":"SpotBugs","version":"4.10.4","language":"en","informationUri":"https://spotbugs.github.io/","rules":[],"supportedTaxonomies":[{"name":"CWE","guid":"b8c54a32-de19-51d2-9a08-f0abfbaa7310"}]}},"invocations":[{"exitCode":0,"exitCodeDescription":"SUCCESS","executionSuccessful":true}],"results":[],"originalUriBaseIds":{},"taxonomies":[]}]} \ No newline at end of file diff --git a/.github/tests/analysis/fixtures/spotbugs-error.json b/.github/tests/analysis/fixtures/spotbugs-error.json new file mode 100644 index 0000000000..6f67129348 --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-error.json @@ -0,0 +1 @@ +{"version":"2.1.0","$schema":"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json","runs":[{"tool":{"extensions":[{"version":"4.10.4","name":"edu.umd.cs.findbugs.plugins.core","shortDescription":{"text":"Core SpotBugs plugin"},"informationUri":"https://github.com/spotbugs","organization":"SpotBugs project"}],"driver":{"name":"SpotBugs","version":"4.10.4","language":"en","informationUri":"https://spotbugs.github.io/","rules":[],"supportedTaxonomies":[{"name":"CWE","guid":"b8c54a32-de19-51d2-9a08-f0abfbaa7310"}]}},"invocations":[{"exitCode":4,"exitCodeDescription":"ERROR","executionSuccessful":false,"toolExecutionNotifications":[{"descriptor":{"id":"spotbugs-error-0"},"message":{"text":"Synthetic detector failure"},"level":"error"}]}],"results":[],"originalUriBaseIds":{},"taxonomies":[]}]} \ No newline at end of file diff --git a/.github/tests/analysis/fixtures/spotbugs-finding.json b/.github/tests/analysis/fixtures/spotbugs-finding.json new file mode 100644 index 0000000000..eb04cfcd2b --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-finding.json @@ -0,0 +1,155 @@ +{ + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "extensions": [ + { + "version": "4.10.4", + "name": "edu.umd.cs.findbugs.plugins.core", + "shortDescription": { + "text": "Core SpotBugs plugin" + }, + "informationUri": "https://github.com/spotbugs", + "organization": "SpotBugs project" + } + ], + "driver": { + "name": "SpotBugs", + "version": "4.10.4", + "language": "en", + "informationUri": "https://spotbugs.github.io/", + "rules": [ + { + "id": "NP_ALWAYS_NULL", + "shortDescription": { + "text": "Null pointer dereference." + }, + "fullDescription": { + "text": "A null pointer is dereferenced here.\u00a0 This will lead to a NullPointerException \nwhen the code is executed." + }, + "messageStrings": { + "default": { + "text": "Null pointer dereference of {0} in {1}." + } + }, + "helpUri": "https://spotbugs.readthedocs.io/en/latest/bugDescriptions.html#NP_ALWAYS_NULL", + "properties": { + "tags": [ + "CORRECTNESS" + ] + } + }, + { + "id": "NP_LOAD_OF_KNOWN_NULL_VALUE", + "shortDescription": { + "text": "Load of known null value." + }, + "fullDescription": { + "text": "The variable referenced at this point is known to be null due to an earlier \ncheck against null. Although this is valid, it might be a mistake (perhaps you \nintended to refer to a different variable, or perhaps the earlier check to see \nif the variable is null should have been a check to see if it was non-null)." + }, + "messageStrings": { + "default": { + "text": "Load of known null value in {0}." + } + }, + "helpUri": "https://spotbugs.readthedocs.io/en/latest/bugDescriptions.html#NP_LOAD_OF_KNOWN_NULL_VALUE", + "properties": { + "tags": [ + "STYLE" + ] + } + } + ], + "supportedTaxonomies": [ + { + "name": "CWE", + "guid": "b8c54a32-de19-51d2-9a08-f0abfbaa7310" + } + ] + } + }, + "invocations": [ + { + "exitCode": 1, + "exitCodeDescription": "BUGS FOUND", + "executionSuccessful": true + } + ], + "results": [ + { + "ruleId": "NP_ALWAYS_NULL", + "ruleIndex": 0, + "message": { + "id": "default", + "text": "Null pointer dereference", + "arguments": [ + "s", + "com.example.Example.main(String[])" + ] + }, + "level": "error", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "com/example/Example.java", + "uriBaseId": "2139602823" + }, + "region": { + "startLine": 2 + } + }, + "logicalLocations": [ + { + "name": "s", + "kind": "variable", + "fullyQualifiedName": "com.example.Example.main(String[])#s" + } + ] + } + ] + }, + { + "ruleId": "NP_LOAD_OF_KNOWN_NULL_VALUE", + "ruleIndex": 1, + "message": { + "id": "default", + "text": "Load of known null value", + "arguments": [ + "com.example.Example.main(String[])" + ] + }, + "level": "note", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "com/example/Example.java", + "uriBaseId": "2139602823" + }, + "region": { + "startLine": 2 + } + }, + "logicalLocations": [ + { + "name": "s", + "kind": "variable", + "fullyQualifiedName": "com.example.Example.main(String[])#s" + } + ] + } + ] + } + ], + "originalUriBaseIds": { + "2139602823": { + "uri": "file:///checkout/module/src/" + } + }, + "taxonomies": [] + } + ] +} diff --git a/.github/tests/analysis/mutations.sh b/.github/tests/analysis/mutations.sh new file mode 100644 index 0000000000..e5a8fd62f0 --- /dev/null +++ b/.github/tests/analysis/mutations.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +repo=$(cd "$(dirname "$0")/../../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mutant="$scratch/repo" +mutation() { + local label=$1 file=$2 needle=$3 replacement=$4 case_filter=$5 content + rm -rf "$mutant" + mkdir -p "$mutant/.github/scripts" "$mutant/.github/workflows" "$mutant/.github/tests" + cp -R "$repo/.github/tests/analysis" "$mutant/.github/tests/" + cp "$repo/.github/scripts/sarif.sh" "$mutant/.github/scripts/" + cp "$repo/.github/workflows/verify.yml" "$mutant/.github/workflows/" + content=$(cat "$mutant/$file") + if [[ "$content" != *"$needle"* ]]; then echo "Mutation anchor missing: $label"; exit 1; fi + content=${content//"$needle"/"$replacement"} + printf '%s\n' "$content" > "$mutant/$file" + if CASE_FILTER="$case_filter" bash "$mutant/.github/tests/analysis/run.sh" > "$scratch/$label.log" 2>&1; then + echo "FAIL: regression escaped detection: $label"; exit 1 + fi + grep -q 'FAIL:' "$scratch/$label.log" + echo "PASS: detected $label" +} +helper=.github/scripts/sarif.sh +workflow=.github/workflows/verify.yml +mutation execution-status "$helper" 'and .executionSuccessful == true' 'and true' spotbugs/execution-false +mutation error-notification "$helper" '.level != "error"' 'true' pmd/execution-error +mutation completion-metadata "$helper" 'and ($analyzer == "Checkstyle" or (.invocations | type == "array" and length > 0))' 'and true' spotbugs/missing-invocations +mutation cpd-parser "$helper" 'local report=$1 valid count' 'local report=$1 valid count; echo 0; return 0' cpd/truncated +mutation cpd-count "$helper" "'count(/*/*[local-name()=\"duplication\"])'" "'0'" cpd/minified +mutation uri-base-reference "$helper" '.uri = ($absolute | ltrimstr($root)) | del(.uriBaseId)' 'del(.uriBaseId)' spotbugs/locations-and-rules +mutation merged-validation "$workflow" 'validate_sarif .sarif-merged/spotbugs.sarif SpotBugs' ':' spotbugs/merged-execution-false +mutation maven-failure-status "$workflow" '--fail-at-end' '--fail-never' spotbugs/clean diff --git a/.github/tests/analysis/run.sh b/.github/tests/analysis/run.sh new file mode 100644 index 0000000000..bb2c7351c9 --- /dev/null +++ b/.github/tests/analysis/run.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +trap 'echo "FAIL: ${tool:-harness}/${case:-setup} line $LINENO"' ERR +repo=$(cd "$(dirname "$0")/../../.." && pwd) +fixtures="$repo/.github/tests/analysis/fixtures" +for dependency in bash jq yq xmllint; do command -v "$dependency" >/dev/null; done +yq --version | grep -Eq 'version v?4\.' +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +for job in lint spotbugs; do + for block in merge gate; do + if [ "$block" = merge ]; then prefix='Merge per-module'; else prefix='Gate on'; fi + JOB=$job PREFIX=$prefix yq -r '.jobs[strenv(JOB)].steps[] | select(.name | test("^" + strenv(PREFIX))) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/$job-$block.sh" + test -s "$scratch/$job-$block.sh" + bash -n "$scratch/$job-$block.sh" + done +done +# Every analyzer invocation must preserve a nonzero process exit status. +yq -o=json '.jobs' "$repo/.github/workflows/verify.yml" | jq -e ' + [.. | objects | .run? // empty | select(test("mvn .*--batch-mode"))] as $commands + | ($commands | length >= 3) and all($commands[]; contains("--fail-at-end") and (contains("--fail-never") | not))' >/dev/null +passed=0 +fresh() { + folder="$scratch/case" + rm -rf "$folder" + mkdir -p "$folder/ddk-parent/target" "$folder/.github/scripts" + cp "$repo/.github/scripts/sarif.sh" "$folder/.github/scripts/" + printf '\n../a\n../b\n\n' > "$folder/ddk-parent/pom.xml" + printf '\n' > "$folder/ddk-parent/target/checkstyle-result.xml" + for module in a b; do + mkdir -p "$folder/$module/target" "$folder/$module/src" "$folder/$module/META-INF" + touch "$folder/$module/META-INF/MANIFEST.MF" + cp "$fixtures/spotbugs-clean.json" "$folder/$module/target/spotbugsSarif.json" + cp "$fixtures/pmd-clean.json" "$folder/$module/target/pmd.sarif.json" + cp "$fixtures/checkstyle-clean.json" "$folder/$module/target/checkstyle-result.xml" + printf '\n' > "$folder/$module/target/cpd.xml" + done +} +edit() { jq "$1" "$target" > "$target.tmp"; mv "$target.tmp" "$target"; } +invoke() { + local block=$1 + (cd "$folder" && GITHUB_WORKSPACE="${FIXTURE_ROOT:-$folder}" bash -e "$scratch/$job-$block.sh") > "$scratch/$block.log" 2>&1 +} +expect() { + local block=$1 wanted=$2 status=0 + invoke "$block" || status=$? + if { [ "$wanted" = pass ] && [ "$status" -ne 0 ]; } || { [ "$wanted" = fail ] && [ "$status" -eq 0 ]; }; then + echo "FAIL: $tool/$case $block expected $wanted, exit $status" + cat "$scratch/$block.log" + exit 1 + fi +} +for tool in spotbugs pmd checkstyle; do + job=lint + case "$tool" in + spotbugs) job=spotbugs; filename=spotbugsSarif.json ;; + pmd) filename=pmd.sarif.json ;; + checkstyle) filename=checkstyle-result.xml ;; + esac + for case in clean finding execution-false execution-error configuration-error warning missing empty whitespace truncated \ + two-documents empty-runs missing-results null-results bad-message wrong-tool missing-invocations empty-invocations \ + null-invocations later-run-failure merged-truncated merged-missing merged-execution-false scoped-report-list real-error real-finding; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + unset FIXTURE_ROOT + fresh + target="$folder/b/target/$filename" + wanted=fail; merge_wanted=fail + case "$case" in + clean) wanted=pass; merge_wanted=pass ;; + finding) edit '.runs[0].results=[{ruleId:"fixture",message:{text:"Deliberate finding"}}]'; merge_wanted=pass ;; + execution-false) edit '.runs[0].invocations=[{executionSuccessful:false}]' ;; + execution-error) edit '.runs[0].invocations=[{executionSuccessful:true,toolExecutionNotifications:[{level:"error",message:{text:"error"}}]}]' ;; + configuration-error) edit '.runs[0].invocations=[{executionSuccessful:true,toolConfigurationNotifications:[{level:"error",message:{text:"error"}}]}]' ;; + warning) edit '.runs[0].invocations=[{executionSuccessful:true,toolExecutionNotifications:[{level:"warning",message:{text:"warning"}}]}]'; wanted=pass; merge_wanted=pass ;; + missing) rm "$target" ;; + empty) : > "$target" ;; + whitespace) printf ' \n\t' > "$target" ;; + truncated) printf '{"version":' > "$target" ;; + two-documents) cat "$target" "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + empty-runs) edit '.runs=[]' ;; + missing-results) edit 'del(.runs[0].results)' ;; + null-results) edit '.runs[0].results=null' ;; + bad-message) edit '.runs[0].results=[{message:null}]' ;; + wrong-tool) edit '.runs[0].tool.driver.name="Other"' ;; + missing-invocations) edit 'del(.runs[0].invocations)'; if [ "$tool" = checkstyle ]; then wanted=pass; merge_wanted=pass; fi ;; + empty-invocations) edit '.runs[0].invocations=[]'; if [ "$tool" = checkstyle ]; then wanted=pass; merge_wanted=pass; fi ;; + null-invocations) edit '.runs[0].invocations=null' ;; + later-run-failure) edit '.runs += [.runs[0]] | .runs[1].invocations=[{executionSuccessful:false}]' ;; + merged-truncated|merged-missing|merged-execution-false) merge_wanted=pass ;; + scoped-report-list) printf '{' > "$target"; export SPOTBUGS_EXPECT_REPORTS=a LINT_EXPECT_REPORTS=a; wanted=pass; merge_wanted=pass ;; + real-finding) if [ "$tool" != spotbugs ]; then continue; fi; cp "$fixtures/spotbugs-finding.json" "$target"; export FIXTURE_ROOT=/checkout; merge_wanted=pass ;; + real-error) if [ "$tool" = checkstyle ]; then continue; fi; cp "$fixtures/$tool-error.json" "$target" ;; + esac + expect merge "$merge_wanted" + if [ "$case" = merged-truncated ]; then printf '{' > "$folder/.sarif-merged/$tool.sarif"; fi + if [ "$case" = merged-missing ]; then rm "$folder/.sarif-merged/$tool.sarif"; fi + if [ "$case" = merged-execution-false ]; then + target="$folder/.sarif-merged/$tool.sarif" + edit '.runs[0].invocations=[{executionSuccessful:false}]' + fi + expect gate "$wanted" + if [ "$case" = finding ]; then grep -q 'found violations' "$scratch/gate.log"; fi + unset SPOTBUGS_EXPECT_REPORTS LINT_EXPECT_REPORTS + passed=$((passed + 1)) + echo "PASS: $tool/$case" + done +done +tool=cpd; job=lint +for case in clean finding minified multiple comment cdata missing empty truncated not-xml wrong-root error namespaced-error unexpected-child real-clean real-finding real-error; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + target="$folder/b/target/cpd.xml" + wanted=fail + case "$case" in + real-clean) cp "$fixtures/cpd-clean.xml" "$target"; wanted=pass ;; + real-finding) cp "$fixtures/cpd-finding.xml" "$target" ;; + real-error) cp "$fixtures/cpd-error.xml" "$target" ;; + clean) wanted=pass ;; + finding) printf '\n\n' > "$target" ;; + minified) printf '' > "$target" ;; + multiple) printf '' > "$target" ;; + comment) printf '' > "$target"; wanted=pass ;; + cdata) printf ']]>' > "$target"; wanted=pass ;; + missing) rm "$target" ;; + empty) : > "$target" ;; + truncated) printf '\n' > "$target" ;; + not-xml) printf 'analysis crashed\n' > "$target" ;; + wrong-root) printf '' > "$target" ;; + error) printf '' > "$target" ;; + namespaced-error) printf '' > "$target" ;; + unexpected-child) printf '' > "$target" ;; + esac + expect merge pass + expect gate "$wanted" + case "$case" in + finding|minified) grep -q 'CPD duplications: 1' "$scratch/gate.log" ;; + multiple) grep -q 'CPD duplications: 2' "$scratch/gate.log" ;; + esac + passed=$((passed + 1)); echo "PASS: cpd/$case" +done +# Exercise collisions, chained bases, encoded paths, secondary locations and different rule indices. +for tool in spotbugs pmd checkstyle; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/locations-and-rules" ]; then continue; fi + fresh + case "$tool" in + spotbugs) job=spotbugs; filename=spotbugsSarif.json ;; + pmd) job=lint; filename=pmd.sarif.json ;; + checkstyle) job=lint; filename=checkstyle-result.xml ;; + esac + for module in a b; do + target="$folder/$module/target/$filename" + jq --arg module "$module" ' + .runs[0] |= (.originalUriBaseIds={ROOT:{uri:"file:///checkout/"},SRC:{uri:($module+"/src/"),uriBaseId:"ROOT"}} + | .tool.driver.rules=[{id:$module,shortDescription:{text:$module}}] + | .results=[{ruleId:$module,ruleIndex:0,level:"warning",message:{text:("finding-"+$module)}, + locations:[{physicalLocation:{artifactLocation:{uri:"Space%20Name.java",uriBaseId:"SRC"}}}], + relatedLocations:[{id:1,physicalLocation:{artifactLocation:{uri:"Related.java",uriBaseId:"SRC"}}}]}])' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done + case=locations-and-rules + export FIXTURE_ROOT=/checkout + expect merge pass + expect gate fail + jq -e '.runs[0] as $run | ($run.results | length==2) + and ([$run.tool.driver.rules[].id] | sort == ["a","b"]) + and all($run.results[]; . as $result + | .level=="warning" and .message.text==("finding-"+.ruleId) + and (has("ruleIndex") | not) + and all(.locations[], .relatedLocations[]; + .physicalLocation.artifactLocation as $loc + | ($loc | has("uriBaseId") | not) + and ($loc.uri == ($result.ruleId+"/src/Space%20Name.java") + or $loc.uri == ($result.ruleId+"/src/Related.java"))))' \ + "$folder/.sarif-merged/$tool.sarif" >/dev/null + unset FIXTURE_ROOT + passed=$((passed + 1)); echo "PASS: $tool/$case" +done +# GitHub consumes repository-relative artifact URIs, not arbitrary URI-base identifiers. +for case in relative-uri absolute-uri file-uri dot-segments encoded-root outside-repository unknown-scheme; do + tool=spotbugs; job=spotbugs + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + export FIXTURE_ROOT=/checkout + target="$folder/b/target/spotbugsSarif.json" + wanted=pass + expected='a/src/Example.java' + case "$case" in + relative-uri) uri='a/src/Example.java' ;; + absolute-uri) uri='/checkout/a/src/Example.java' ;; + file-uri) uri='file:/checkout/a/src/Example.java' ;; + dot-segments) uri='file:///checkout/a/other/../src/./Example.java' ;; + encoded-root) export FIXTURE_ROOT='/checkout space'; uri='file:///checkout%20space/a/src/Space%20Name.java'; expected='a/src/Space%20Name.java' ;; + outside-repository) uri='file:///unrelated/src/Example.java'; wanted=fail ;; + unknown-scheme) uri='https://example.invalid/Example.java'; wanted=fail ;; + esac + jq --arg uri "$uri" '.runs[0].results=[{message:{text:"location probe"},locations:[{physicalLocation:{artifactLocation:{uri:$uri}}}]}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + expect merge "$wanted" + if [ "$wanted" = pass ]; then + expect gate fail + jq -e --arg expected "$expected" '.runs[0].results[0].locations[0].physicalLocation.artifactLocation | .uri==$expected and (has("uriBaseId")|not)' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + fi + unset FIXTURE_ROOT + passed=$((passed + 1)); echo "PASS: $tool/$case" +done + +# Metadata that cannot be combined without loss must fail explicitly. +for case in taxonomies conflicting-rules rule-index-only wrong-rule-index unsupported-run dangling-base cyclic-base; do + tool=spotbugs; job=spotbugs + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + target="$folder/b/target/spotbugsSarif.json" + wanted=fail + case "$case" in + taxonomies) + for module in a b; do + target="$folder/$module/target/spotbugsSarif.json" + jq --arg module "$module" '.runs[0].taxonomies=[{name:"CWE",guid:"shared",taxa:[{id:$module,shortDescription:{text:$module}}]}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done + wanted=pass ;; + conflicting-rules) + for module in a b; do + target="$folder/$module/target/spotbugsSarif.json" + jq --arg module "$module" '.runs[0].tool.driver.rules=[{id:"shared",shortDescription:{text:$module}}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done ;; + rule-index-only) + edit '.runs[0] |= (.tool.driver.rules=[{id:"resolved"}] | .results=[{ruleIndex:0,message:{text:"indexed rule"}}])' + wanted=pass ;; + wrong-rule-index) + edit '.runs[0] |= (.tool.driver.rules=[{id:"first"}] | .results=[{ruleId:"other",ruleIndex:0,message:{text:"mismatch"}}])' ;; + unsupported-run) edit '.runs[0].artifacts=[{location:{uri:"src/Example.java"}}]' ;; + dangling-base) edit '.runs[0].originalUriBaseIds={SRC:{uri:"src/",uriBaseId:"MISSING"}}' ;; + cyclic-base) edit '.runs[0].originalUriBaseIds={SRC:{uri:"src/",uriBaseId:"SRC"}}' ;; + esac + expect merge "$wanted" + if [ "$case" = taxonomies ]; then + expect gate pass + jq -e '.runs[0].taxonomies | length==1 and (.[0].taxa | map(.id) | sort==["a","b"])' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + elif [ "$case" = rule-index-only ]; then + expect gate fail + jq -e '.runs[0].results[0] | .ruleId=="resolved" and (has("ruleIndex") | not)' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + fi + passed=$((passed + 1)); echo "PASS: $tool/$case" +done + +# Maven failures remain failures even if usable reports already exist. +if [ -z "${CASE_FILTER:-}" ]; then + fresh + mkdir "$scratch/bin" + cat > "$scratch/bin/mvn" <<'STUB' +#!/usr/bin/env bash +printf 'process reached\n' > "$PROCESS_MARKER" +exit 42 +STUB + chmod +x "$scratch/bin/mvn" + for name in 'PMD + Checkstyle reports (SARIF)' 'CPD report (separate invocation — no SARIF support)' 'SpotBugs report (SARIF)'; do + NAME="$name" yq -r '.jobs[].steps[] | select(.name == strenv(NAME)) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/process.sh" + rm -f "$scratch/process-marker" + status=0 + (cd "$folder" && PATH="$scratch/bin:$PATH" PROCESS_MARKER="$scratch/process-marker" bash -e "$scratch/process.sh") || status=$? + test "$status" -eq 42 + test -s "$scratch/process-marker" + passed=$((passed + 1)); echo "PASS: process failure/$name" + done +fi +test "$passed" -gt 0 +echo "$passed analysis regression cases passed." diff --git a/.github/tests/analysis/scope.sh b/.github/tests/analysis/scope.sh new file mode 100644 index 0000000000..d9ccfe4f98 --- /dev/null +++ b/.github/tests/analysis/scope.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +trap 'echo "FAIL: scope ${mode:-setup}/${scenario:-setup} line $LINENO"' ERR +repo=$(cd "$(dirname "$0")/../../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +script=compute-spotbugs-skip.sh +modes=spotbugs +if [ ! -f "$repo/.github/scripts/$script" ] && [ ! -f "$repo/.github/scripts/compute-analysis-skip.sh" ]; then + echo 'Scope tests: this head uses the full reactor.' + exit 0 +fi +if [ -f "$repo/.github/scripts/compute-analysis-skip.sh" ]; then + script=compute-analysis-skip.sh + modes='spotbugs lint' +fi +for mode in $modes; do + prefix=$(printf '%s' "$mode" | tr '[:lower:]' '[:upper:]') + for block in merge gate; do + if [ "$block" = merge ]; then starts='Merge per-module'; else starts='Gate on'; fi + JOB=$mode PREFIX=$starts yq -r '.jobs[strenv(JOB)].steps[] | select(.name | test("^" + strenv(PREFIX))) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/$block.sh" + done + for scenario in partial two-modules docs-only source-less mixed shared-config workflow deletion cross-module-move; do + folder="$scratch/$mode-$scenario" + git clone --quiet --no-checkout "$repo/.github/tests/analysis/fixtures/scope.bundle" "$folder" + git -C "$folder" checkout --quiet --detach "origin/$scenario" + base=$(git -C "$folder" rev-parse HEAD^) + environment="$folder/environment" + mkdir -p "$folder/.github/scripts" + cp "$repo/.github/scripts/$script" "$repo/.github/scripts/sarif.sh" "$folder/.github/scripts/" + unset SPOTBUGS_KEPT SPOTBUGS_EXPECT_REPORTS SPOTBUGS_SCOPE_ARGS LINT_KEPT LINT_EXPECT_REPORTS LINT_SCOPE_ARGS + args=("$base") + if [ "$script" = compute-analysis-skip.sh ]; then args+=("$mode"); fi + for iteration in 1 2; do + (cd "$folder" && GITHUB_ENV="$environment" bash ".github/scripts/$script" "${args[@]}") > "$scratch/scope.log" 2>&1 + git -C "$folder" diff > "$scratch/injection-$iteration.diff" + done + cmp "$scratch/injection-1.diff" "$scratch/injection-2.diff" + if [ -f "$environment" ]; then + while IFS= read -r assignment; do export "$assignment"; done < "$environment" + fi + case "$scenario" in + partial|mixed|deletion) wanted='a' ;; + two-modules|shared-config|workflow|cross-module-move) wanted='a b' ;; + docs-only|source-less) wanted='' ;; + esac + key="${prefix}_KEPT"; kept=${!key:-all} + key="${prefix}_EXPECT_REPORTS"; expected=${!key:-a b} + if [ -z "$wanted" ]; then test "$kept" = 0; else test "$kept" != 0; test "$expected" = "$wanted"; fi + full=false + if [ "$scenario" = shared-config ] || [ "$scenario" = workflow ]; then full=true; fi + props=spotbugs.skip + if [ "$mode" = lint ]; then props='pmd.skip cpd.skip checkstyle.skip'; fi + for module in a b brand feature ddk-target; do + skip=true + case "$scenario:$module" in + partial:a|two-modules:a|two-modules:b|source-less:brand|mixed:a|mixed:brand|deletion:a|cross-module-move:a|cross-module-move:b) skip=false ;; + esac + if [ "$full" = true ]; then skip=false; fi + for prop in $props; do + count=$(grep -Fc "<$prop>true" "$folder/$module/pom.xml" || true) + if [ "$skip" = true ]; then test "$count" -eq 1; else test "$count" -eq 0; fi + done + done + key="${prefix}_SCOPE_ARGS"; scope_args=${!key:-} + if grep -q 'SCOPE_ARGS=' "$repo/.github/scripts/$script"; then + if [ -n "$wanted" ] && [ "$full" = false ]; then + case "$scenario" in + mixed) selected='../a,../brand' ;; + two-modules|cross-module-move) selected='../a,../b' ;; + *) selected='../a' ;; + esac + test "$scope_args" = "-pl ../ddk-target,$selected -am" + else test -z "$scope_args"; fi + fi + for module in $wanted; do + mkdir -p "$folder/$module/target" + cp "$repo/.github/tests/analysis/fixtures/spotbugs-clean.json" "$folder/$module/target/spotbugsSarif.json" + cp "$repo/.github/tests/analysis/fixtures/pmd-clean.json" "$folder/$module/target/pmd.sarif.json" + cp "$repo/.github/tests/analysis/fixtures/checkstyle-clean.json" "$folder/$module/target/checkstyle-result.xml" + cp "$repo/.github/tests/analysis/fixtures/cpd-clean.xml" "$folder/$module/target/cpd.xml" + done + for block in merge gate; do (cd "$folder" && bash -e "$scratch/$block.sh") > "$scratch/$block.log" 2>&1; done + if [ -z "$wanted" ]; then test ! -d "$folder/.sarif-merged"; fi + if [ -n "$wanted" ]; then + tools=spotbugs + if [ "$mode" = lint ]; then tools='pmd checkstyle'; fi + for tool in $tools; do + case "$tool" in spotbugs) filename=spotbugsSarif.json ;; pmd) filename=pmd.sarif.json ;; checkstyle) filename=checkstyle-result.xml ;; esac + for fault in missing invalid failed finding; do + target="$folder/a/target/$filename" + cp "$repo/.github/tests/analysis/fixtures/$tool-clean.json" "$target" + case "$fault" in + missing) rm "$target" ;; + invalid) printf '{' > "$target" ;; + failed) jq '.runs[0].invocations=[{executionSuccessful:false}]' "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + finding) jq '.runs[0].results=[{message:{text:"scoped finding"}}]' "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + esac + for block in merge gate; do + status=0 + (cd "$folder" && bash -e "$scratch/$block.sh") > "$scratch/$block.log" 2>&1 || status=$? + if [ "$fault:$block" = finding:merge ]; then test "$status" -eq 0; else test "$status" -ne 0; fi + done + done + cp "$repo/.github/tests/analysis/fixtures/$tool-clean.json" "$folder/a/target/$filename" + done + fi + echo "PASS: scope $mode/$scenario" + done + unset SPOTBUGS_KEPT SPOTBUGS_EXPECT_REPORTS SPOTBUGS_SCOPE_ARGS LINT_KEPT LINT_EXPECT_REPORTS LINT_SCOPE_ARGS + if (cd "$folder" && GITHUB_ENV="$scratch/invalid-base-env" bash ".github/scripts/$script" does-not-exist "$mode") > "$scratch/invalid-base.log" 2>&1; then + echo 'FAIL: invalid base was accepted'; exit 1 + fi + test ! -s "$scratch/invalid-base-env" + echo "PASS: scope $mode/invalid-base" +done diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2322f216bc..4e16cd0eb8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,8 +1,38 @@ name: verify on: pull_request: + +# Code Scanning needs write access to upload SARIF results for inline annotations. +permissions: + contents: read + security-events: write + jobs: - pmd: + analysis-regression: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install validation dependencies + run: sudo apt-get update && sudo apt-get install --yes jq libxml2-utils + - name: Check report gates and merge semantics + run: | + bash .github/tests/analysis/run.sh + bash .github/tests/analysis/mutations.sh + bash .github/tests/analysis/scope.sh + + # Fast, early-fail lint lane: PMD + Checkstyle (+ CPD). Turns red in a few + # minutes on any violation, independent of the long build below, so a stray + # PMD/Checkstyle issue is reported immediately rather than after `verify`. + # + # `compile` is in the same invocation as the analysis goals: PMD's + # type-resolving rules (e.g. InvalidLogMessageFormat on the SLF4J + # trailing-Throwable idiom) need Tycho's aux-classpath, which a fresh `mvn` + # does not inherit from a prior step's target/classes. + # + # Preserve Maven failures and validate every expected report before counting findings. + lint: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -12,10 +42,98 @@ jobs: java-version: '21' - name: Set up Workspace Environment Variable run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: PMD Check - run: mvn pmd:pmd pmd:cpd pmd:check pmd:cpd-check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - checkstyle: + - name: Restore Maven dependency cache + # Restore-only, mirroring snapshot.yml's producer cache exactly (path and + # key are hashed into the cache version — see the maven-verify step). + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-publish-${{ hashFiles('**/pom.xml', '**/*.target') }} + restore-keys: ${{ runner.os }}-maven-publish- + + - name: Install XML report validator + run: sudo apt-get update && sudo apt-get install --yes libxml2-utils + + - name: PMD + Checkstyle reports (SARIF) + # PMD: SarifRenderer FQCN — emits pmd.sarif.json AND keeps pmd.xml. + # Checkstyle: output.format=sarif — SARIF content in checkstyle-result.xml. + # CPD is excluded here: the global -Dformat flag uses PMD's Renderer + # hierarchy and would ClassCastException CPD's CPDReportRenderer. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:pmd checkstyle:checkstyle \ + -Dformat=net.sourceforge.pmd.renderers.SarifRenderer \ + -Dcheckstyle.output.format=sarif + + - name: CPD report (separate invocation — no SARIF support) + # CPD has no SARIF renderer; emits cpd.xml only. Run standalone so the + # PMD -Dformat flag isn't in scope. + # NOTE: the CPD token threshold is governed by pmd.cpd.min in + # ddk-parent/pom.xml. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:cpd-check + + - name: Merge per-module SARIFs (PMD + Checkstyle) + if: always() + # Merge only expected module reports into one run per analyzer. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + merge_sarif pmd.sarif.json .sarif-merged/pmd.sarif "${LINT_EXPECT_REPORTS:-$source_modules}" PMD + merge_sarif checkstyle-result.xml .sarif-merged/checkstyle.sarif "${LINT_EXPECT_REPORTS:-$source_modules}" Checkstyle + + - name: Gate on PMD / CPD / Checkstyle violations + # Require successful reports from every expected module before counting findings. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + cpd_reports=() + for mod in ${LINT_EXPECT_REPORTS:-$source_modules}; do + validate_sarif "${mod}/target/pmd.sarif.json" PMD + validate_sarif "${mod}/target/checkstyle-result.xml" Checkstyle + cpd_reports+=("${mod}/target/cpd.xml") + done + validate_sarif .sarif-merged/pmd.sarif PMD + validate_sarif .sarif-merged/checkstyle.sarif Checkstyle + if [ ${#cpd_reports[@]} -eq 0 ]; then + echo "::error::No expected CPD reports — the analysis silently failed." + exit 1 + fi + sarif_total=$(jq -s '[.[].runs[].results[]] | length' \ + .sarif-merged/pmd.sarif .sarif-merged/checkstyle.sarif) + cpd_total=0 + for report in "${cpd_reports[@]}"; do + count=$(cpd_count "$report") + cpd_total=$((cpd_total + count)) + done + echo "PMD/Checkstyle SARIF violations: $sarif_total" + echo "CPD duplications: $cpd_total" + if [ "$sarif_total" != "0" ] || [ "$cpd_total" != "0" ]; then + echo "::error::Static analysis found violations (PMD/CPD/Checkstyle)." + exit 1 + fi + + - name: Upload PMD/Checkstyle SARIF to Code Scanning + if: always() + # Annotation-only, never the gate: a fork PR gets a read-only token and + # upload-sarif 403s, which must not red an otherwise-clean lane. + continue-on-error: true + uses: github/codeql-action/upload-sarif@7fd177fa680c9881b53cdab4d346d32574c9f7f4 # v3.35.4 + with: + sarif_file: .sarif-merged + category: lint + + # SpotBugs is the slow critical-path analysis (the experiments' durable + # finding), so it runs in its own parallel lane and never delays `lint`. + spotbugs: runs-on: ubuntu-24.04 + env: + MAVEN_OPTS: -Xmx4g steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 @@ -24,14 +142,69 @@ jobs: java-version: '21' - name: Set up Workspace Environment Variable run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: Checkstyle Check - run: mvn checkstyle:checkstyle checkstyle:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + - name: Restore Maven dependency cache + # Restore-only, mirroring snapshot.yml's producer cache exactly (path and + # key are hashed into the cache version — see the maven-verify step). + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-publish-${{ hashFiles('**/pom.xml', '**/*.target') }} + restore-keys: ${{ runner.os }}-maven-publish- + + - name: SpotBugs report (SARIF) + # sarifOutput=true emits spotbugsSarif.json (also writes spotbugsXml.xml). + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + spotbugs:spotbugs \ + -Dspotbugs.sarifOutput=true + + - name: Merge per-module SpotBugs SARIFs + if: always() + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + merge_sarif spotbugsSarif.json .sarif-merged/spotbugs.sarif "${SPOTBUGS_EXPECT_REPORTS:-$source_modules}" SpotBugs + + - name: Gate on SpotBugs violations + # Require successful reports from every expected module before counting findings. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + for mod in ${SPOTBUGS_EXPECT_REPORTS:-$source_modules}; do + validate_sarif "${mod}/target/spotbugsSarif.json" SpotBugs + done + validate_sarif .sarif-merged/spotbugs.sarif SpotBugs + sb_total=$(jq '[.runs[].results[]] | length' .sarif-merged/spotbugs.sarif) + echo "SpotBugs SARIF violations: $sb_total" + if [ "$sb_total" != "0" ]; then + echo "::error::SpotBugs found violations." + exit 1 + fi + + - name: Upload SpotBugs SARIF to Code Scanning + if: always() + # Annotation-only, never the gate: a fork PR gets a read-only token and + # upload-sarif 403s, which must not red an otherwise-clean lane. + continue-on-error: true + uses: github/codeql-action/upload-sarif@7fd177fa680c9881b53cdab4d346d32574c9f7f4 # v3.35.4 + with: + sarif_file: .sarif-merged + category: spotbugs + line-endings: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check LF line endings run: bash .github/scripts/check-line-endings.sh + + # Build + tests only. Static analysis now lives in the `lint` and `spotbugs` + # jobs, so the redundant checkstyle/pmd/spotbugs goals are dropped from here — + # this is the wall-clock long pole and no longer re-runs analysis. + # No `-T 2C`: tests are not known to pass reliably under reactor parallelism. maven-verify: runs-on: ubuntu-24.04 steps: @@ -74,9 +247,7 @@ jobs: # all downloaded artifacts cached. run: rm -rf ~/.m2/repository/.cache/tycho/https/dsldevkit.github.io ~/.m2/repository/.cache/tycho/https/ddk.tools.avaloq.com - name: Build with Maven within a virtual X Server Environment - # Run pmd:pmd and pmd:cpd first to generate reports for all modules, then run pmd:check and pmd:cpd-check - # This ensures all violations are collected and reported before the build fails - run: xvfb-run mvn clean verify checkstyle:check pmd:pmd pmd:cpd pmd:check pmd:cpd-check spotbugs:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + run: xvfb-run mvn clean verify -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - name: Fail on missing surefire reports if: always() run: bash .github/scripts/check-surefire-reports.sh diff --git a/docs/ci-measurement-protocol.md b/docs/ci-measurement-protocol.md new file mode 100644 index 0000000000..470392f488 --- /dev/null +++ b/docs/ci-measurement-protocol.md @@ -0,0 +1,65 @@ +# CI timing-measurement protocol + +How to get **trustworthy** numbers for a CI-shape change. Written because the +earlier experiment program (2026-05) produced numbers that can't be trusted: +single samples, taken during an Eclipse p2 mirror-flaky window, against a CI +shape that has since changed. Don't repeat that. + +## What we're measuring + +- **Wall-clock** per run = the slowest job (what a developer waits for). +- **Per-job duration** = the analysis bottleneck (settles e.g. spotbugs-vs-maven-verify). +- **Failure latency** = time-to-red on a *planted* lint violation — the metric the + early-fail goal actually cares about. Measure it separately. + +Wall-clock is the headline, but per-job + failure-latency are what tell you *why*. + +## Noise floor (observed on this repo's `verify` runs) + +Per-job spread across recent runs — any change must beat this to be real signal: + +| Job | median | spread (±) | +|---|---|---| +| line-endings | ~5s | ±3s | +| checkstyle (old shape) | ~115s | ±~104s (≈90%!) | +| pmd (old shape) | ~191s | ±63s | +| maven-verify | ~869s (~14.5m) | ±~93s (≈11%) | + +**Decision rule:** accept candidate B as faster than A only if +`median(B) < median(A) − 2 × IQR(A)`. For maven-verify a real win must exceed ~100s; +for checkstyle, ~200s. Anything smaller is noise. + +## Protocol + +1. **Pre-flight — confirm mirrors are healthy.** Run one throwaway build; if + target-platform resolution stalls or errors, **stop** — the window is bad + (this is what voided the 2026-05-09 numbers). Measure only on a clean window. +2. **Warm the caches.** Run 3–5 discard builds first so `~/.m2` + `.cache/tycho` + are populated; cold-cache runs have a different (network-bound) profile. +3. **Sample.** N = 15–20 `workflow_dispatch` runs per candidate, back-to-back in a + ≤30-minute window. Run A and B **interleaved within ≤10 minutes** of each other + so they see the same mirror weather and runner-pool load. +4. **Report medians + IQR**, per job *and* total wall-clock. Never a single sample, + never the mean. +5. **Pin `ubuntu-24.04`** (not a matrix) for consistent runner hardware. +6. **Record per-sample metadata:** cache hit/miss, run start time, headSha. +7. **Failure latency:** plant a synthetic PMD/Checkstyle violation, measure how long + until the `lint` check goes red (independent of the build job). + +## Why not just trust the old experiment numbers + +- Single sample each (no repetition). +- 2026-05-09 mirror-flaky window → resolution time is contaminated and varies per job + even within one dispatch. +- Numbers disagree across rounds (sequential measured 21m one round, 33m another). +- `#1369` reshaped master's CI after the experiments ran, so their baseline is stale. + +## What is *not* a timing lever (validated) + +- **Local p2 mirror**: with a warm `~/.m2`/`.cache/tycho`, offline resolution is + ~equal to online (measured 5s vs 6s) — a mirror's steady-state speedup is ~0. + Its only value is cold-cache + flaky-mirror insurance; deferred per the rare-flake + rule. +- **`-Dtycho.mode=maven` on a gate pass**: marginal on a warm cache (resolution is + already seconds); and it *breaks* PMD type-resolving rules (strips the classpath → + false positives). Not used. diff --git a/docs/ci-static-analysis-design.md b/docs/ci-static-analysis-design.md new file mode 100644 index 0000000000..8e7decc74f --- /dev/null +++ b/docs/ci-static-analysis-design.md @@ -0,0 +1,127 @@ +# Static-analysis CI design: SARIF inline display + count-gate + +This documents *why* the static-analysis CI is shaped the way it is, so the +mechanics (verified against the plugin source) don't have to be re-discovered. + +## Goal + +Fast, complete, **early** PMD + Checkstyle feedback — a violation should fail CI +in minutes, on its own check, not after the ~15-minute build. SpotBugs (the slow +analysis) runs in its own parallel lane so it never delays the fast checks. All +of PMD, Checkstyle, and SpotBugs surface **inline** on the PR via SARIF + GitHub +Code Scanning (no custom annotator). + +## Job shape (`verify.yml`) + +Five independent parallel jobs (no `needs:`); wall-clock = the slowest job. + +| Job | Runs | Threads | Gate | +|---|---|---|---| +| `lint` | `compile` + `pmd:pmd` + `checkstyle:checkstyle` (SARIF) + `pmd:cpd-check` | `-T 2C` | validated SARIF result count + parsed CPD duplication count | +| `spotbugs` | `compile` + `spotbugs:spotbugs` (SARIF), `-Xmx4g` | `-T 2C` | jq count of SARIF results | +| `maven-verify` | `clean verify` (build + tests only) | none | Tycho-surefire | +| `analysis-regression` | report, merge, mutation and scope fixtures | none | extracted workflow blocks | +| `line-endings` | `git ls-files` check | n/a | shell | + +`maven-verify` **no longer re-runs** the analysis goals (now owned by `lint` / +`spotbugs`). Analysis jobs use `-T 2C`; the test job does not (tests are not known +reliable under reactor parallelism). + +## Report goals vs. check goals (code-verified at each plugin version tag) + +| Goal | Kind | `@Execute` fork | Runs analysis? | Writes | Fails build? | SARIF-capable? | +|---|---|---|---|---|---|---| +| `pmd:pmd` | report | — | yes | `pmd.xml` (+ `pmd.sarif.json` w/ `-Dformat=…SarifRenderer`) | no | **yes** | +| `pmd:check` | check | `@Execute(goal=pmd)` | yes (forked) | `pmd.xml` | yes (`> maxAllowedViolations`, dflt 0; `failurePriority` dflt 5 = all) | no (wants `pmd.xml`) | +| `pmd:cpd` | report | — | yes | `cpd.xml` | no | no (no CPD SARIF renderer) | +| `pmd:cpd-check` | check | `@Execute(goal=cpd)` | yes (forked) | `cpd.xml` | yes | no | +| `checkstyle:checkstyle` | report | — | yes | `checkstyle-result.xml` (XML, or SARIF w/ `-Dcheckstyle.output.format=sarif`) | no | **yes** | +| `checkstyle:check` | check | **none** (self-contained) | yes (internal, source-based) | `checkstyle-result.xml` (XML) | yes (severity ≥ `violationSeverity`) | no | +| `spotbugs:spotbugs` | report | — | yes (bytecode) | `spotbugsXml.xml` (+ `spotbugsSarif.json` w/ `-Dspotbugs.sarifOutput=true`) | no | **yes** | +| `spotbugs:check` | check | `@Execute(goal=spotbugs)` | yes (forked) | `spotbugsXml.xml` | yes (`bugCount > 0`) | no | + +The split is **observe vs. enforce**: report goals produce output (for the Maven +site / dashboards / SARIF consumers); analysis errors can still fail a report goal. Check goals bind to +`verify` and exist to fail the build. The `@Execute(goal=…)` on the PMD/CPD/SpotBugs +checks forks the report goal first (so `mvn pmd:check` is self-contained) — which +means they **re-run the analysis every time**. `checkstyle:check` is the lone +exception: no `@Execute`, runs Checkstyle internally in one pass. + +## Maven reactor failure flags + +| Flag | Reactor behavior | Multi-module report completeness | Suppresses violation failure? | +|---|---|---|---| +| `--fail-fast` (default) | halt at first failing module | downstream skipped → reports missing | no | +| `--fail-at-end` | build independent modules, fail at end | modules downstream of a failure still cascade-skipped → incomplete | no | +| `--fail-never` | suppresses every Maven failure | attempts remaining modules, but analysis can still fail to produce usable reports | yes | + +## Report goals, preserved process status, and validated reports + +Each SARIF producer runs once with `--fail-at-end`. Its process status remains +part of the job outcome, including a failure after it has written a report. +Independent modules continue; downstream modules may be skipped after a failure. +The merge requires every expected report, so incomplete analysis stays red even +when other modules produced clean reports. This trades report completeness on a +broken build for dependable failure reporting. + +The shared validator applies to SpotBugs, PMD and Checkstyle. It rejects malformed +JSON, unusable run/result structure, unsuccessful invocations and error execution +or configuration notifications. SpotBugs and PMD must include nonempty completion +metadata; Checkstyle's renderer legitimately omits it. Maven's nonzero exit status +therefore remains essential. CPD XML is parsed, checked for processing errors and +counted structurally, with Maven failures preserved too. + + +### Why not the `:check` goals +- They `@Execute`-fork a **second** analysis on top of the `pmd:pmd` we already run + for SARIF — pure waste. +- The forked analysis is only correct with the full `compile` + classpath; run cheaply + (`-Dtycho.mode=maven` / no compile) it loses the classpath and **false-positives** + (e.g. PMD `InvalidLogMessageFormat` flags the SLF4J trailing-`Throwable` idiom because + it can't resolve the last arg as a `Throwable`). +- `pmd:check` is **incompatible with `-Dformat=sarif`**: its fork writes `pmd.sarif.json`, + but the check looks for `pmd.xml` → "unable to find report." + +### count-gate == `:check` fidelity (for this project's config) +Counting **all** SARIF results equals what `:check` would fail on, because: PMD has no +`failurePriority` override (default 5 = all priorities), Checkstyle is globally +`severity=warning` with no info-level results, and SpotBugs uses `threshold=Low` with no +`failThreshold`/`maxAllowedViolations` override. Three config-drift caveats would make +the count-gate *stricter* than `:check` (over-fail, never under-fail), each guard-worthy: +1. PMD `pmd.failurePriority` set below 5. +2. A Checkstyle rule emitting `info`/`note` severity. +3. SpotBugs `failThreshold` or non-zero `maxAllowedViolations`. + +## Two operational rules + +- **`compile` must be full-reactor** (`-f ddk-parent/pom.xml`, no `-pl`). PMD's + type-resolving rules need the complete aux-classpath; a `-pl` subset produces + false positives (the trailing-`Throwable` case). +- **Merge SARIFs from SARIF files only.** Code Scanning accepts one run per category, + so per-module SARIFs are merged (jq) before upload. The `ddk-parent` aggregator emits + plain-XML `checkstyle-result.xml`; it is excluded by the expected source-module list. + Every expected module report must validate; invalid reports are never filtered out. + +## SARIF mechanics + +- PMD: `-Dformat=net.sourceforge.pmd.renderers.SarifRenderer` → `pmd.sarif.json`. +- Checkstyle: `-Dcheckstyle.output.format=sarif` → SARIF content in `checkstyle-result.xml`. +- SpotBugs: `-Dspotbugs.sarifOutput=true` → `spotbugsSarif.json`. +- All emit SARIF 2.1.0. Merge per tool → `github/codeql-action/upload-sarif` + (`permissions: security-events: write`). CPD has no SARIF renderer → parsed XML gate + (no inline display for CPD; acceptable, duplications are rare). + +## Merge preservation and regression checks + +The merger resolves each run's URI bases and emits repository-relative source +URIs, including secondary locations. A real fork upload showed that preserving +custom base IDs alone made GitHub store paths outside the actual source tree. It resolves result rule indices before unioning +rule descriptors, retains taxonomy descriptors, and rejects conflicts or run +metadata outside the configured producers' supported profile. The supported +profile and fixture provenance are documented in `.github/tests/analysis/README.md`. + +The fast regression job executes the actual workflow merge/gate blocks, checks +real producer fixtures and malformed reports, exercises real Git scope scenarios, +and proves that deliberate regressions are detected. Successful local fixtures +complement the real analyzer runs and GitHub upload checks; they do not establish +GitHub annotation rendering by themselves.