From 230afc4d11902623937c0752a8f2f91367056688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Dinis=20Ferreira?= Date: Sat, 30 May 2026 22:58:31 +0200 Subject: [PATCH] ci: fast static analysis with validated reports and inline SARIF Run PMD/Checkstyle/CPD and SpotBugs in separate analysis lanes. Preserve Maven failure status with --fail-at-end and validate every expected raw and merged report before counting findings. Require successful invocation metadata for PMD and SpotBugs; Checkstyle omits it, so retain its valid format while preserving the producer process status. Parse CPD XML and reject processing errors instead of counting lines that resemble findings. Resolve each run's URI base chains and emit repository-relative artifact URIs, including secondary locations. A real GitHub upload accepted preserved base IDs but recorded package-relative paths that did not match the source tree. Explicit relative URIs remove that consumer dependency. Resolve rule indices before combining descriptors, preserve SpotBugs taxonomies, and reject conflicting or unsupported metadata instead of discarding it. Exercise extracted workflow blocks with real producer reports and negative fixtures, and run targeted mutation checks in a fast CI job. Cover absent reports, malformed input, incomplete analysis, notification errors, merged output corruption, source locations, descriptors and producer exit status. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/sarif.sh | 138 +++++++++ .github/tests/analysis/README.md | 53 ++++ .../analysis/fixtures/checkstyle-clean.json | 26 ++ .github/tests/analysis/fixtures/cpd-clean.xml | 4 + .github/tests/analysis/fixtures/cpd-error.xml | 22 ++ .../tests/analysis/fixtures/cpd-finding.xml | 10 + .../tests/analysis/fixtures/pmd-clean.json | 24 ++ .../tests/analysis/fixtures/pmd-error.json | 42 +++ .github/tests/analysis/fixtures/scope.bundle | Bin 0 -> 5430 bytes .../analysis/fixtures/spotbugs-clean.json | 1 + .../analysis/fixtures/spotbugs-error.json | 1 + .../analysis/fixtures/spotbugs-finding.json | 155 ++++++++++ .github/tests/analysis/mutations.sh | 33 +++ .github/tests/analysis/run.sh | 271 ++++++++++++++++++ .github/tests/analysis/scope.sh | 117 ++++++++ .github/workflows/verify.yml | 189 +++++++++++- docs/ci-measurement-protocol.md | 65 +++++ docs/ci-static-analysis-design.md | 127 ++++++++ 18 files changed, 1269 insertions(+), 9 deletions(-) create mode 100644 .github/scripts/sarif.sh create mode 100644 .github/tests/analysis/README.md create mode 100644 .github/tests/analysis/fixtures/checkstyle-clean.json create mode 100644 .github/tests/analysis/fixtures/cpd-clean.xml create mode 100644 .github/tests/analysis/fixtures/cpd-error.xml create mode 100644 .github/tests/analysis/fixtures/cpd-finding.xml create mode 100644 .github/tests/analysis/fixtures/pmd-clean.json create mode 100644 .github/tests/analysis/fixtures/pmd-error.json create mode 100644 .github/tests/analysis/fixtures/scope.bundle create mode 100644 .github/tests/analysis/fixtures/spotbugs-clean.json create mode 100644 .github/tests/analysis/fixtures/spotbugs-error.json create mode 100644 .github/tests/analysis/fixtures/spotbugs-finding.json create mode 100644 .github/tests/analysis/mutations.sh create mode 100644 .github/tests/analysis/run.sh create mode 100644 .github/tests/analysis/scope.sh create mode 100644 docs/ci-measurement-protocol.md create mode 100644 docs/ci-static-analysis-design.md diff --git a/.github/scripts/sarif.sh b/.github/scripts/sarif.sh new file mode 100644 index 0000000000..60428651ae --- /dev/null +++ b/.github/scripts/sarif.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +sarif_source_modules() { + grep -oE '\.\./[^<]+' ddk-parent/pom.xml \ + | sed -E 's#.*\.\./([^<]+)#\1#' \ + | while IFS= read -r module; do + if [ -f "$module/META-INF/MANIFEST.MF" ] && [ -d "$module/src" ]; then + echo "$module" + fi + done +} + +validate_sarif() { + local report=$1 analyzer=${2:?expected analyzer required} + if ! jq -se --arg analyzer "$analyzer" ' + def valid_base($run; $seen): + . as $id | type == "string" and ($seen | index($id) | not) + and ($run.originalUriBaseIds[$id] | type == "object") + and ($run.originalUriBaseIds[$id] | + if has("uriBaseId") then .uriBaseId | valid_base($run; $seen + [$id]) + else (.uri | type == "string") end); + def optional_array($key): (has($key) | not) or (.[$key] | type == "array"); + length == 1 and (.[0] | + type == "object" and .version == "2.1.0" + and (.runs | type == "array" and length > 0) + and all(.runs[]; + (.tool.driver.name == $analyzer) + and (.results | type == "array") + and (. as $run | all(.. | objects | select(has("uriBaseId")); .uriBaseId | valid_base($run; []))) + and optional_array("invocations") + and ($analyzer == "Checkstyle" or (.invocations | type == "array" and length > 0)) + and all(.results[]?; + type == "object" and (.message | type == "object")) + and all(.invocations[]?; + type == "object" + and .executionSuccessful == true + and optional_array("toolExecutionNotifications") + and optional_array("toolConfigurationNotifications") + and all(.toolExecutionNotifications[]?, .toolConfigurationNotifications[]?; + type == "object" and .level != "error")))) + ' "$report" >/dev/null; then + echo "::error::${report} is missing, invalid SARIF, or reports unsuccessful analysis." + return 1 + fi +} + +merge_sarif() { + local report=$1 output=$2 modules=$3 analyzer=${4:?expected analyzer required} module + local inputs=() + for module in $modules; do + validate_sarif "${module}/target/${report}" "$analyzer" || return 1 + inputs+=("${module}/target/${report}") + done + if [ ${#inputs[@]} -eq 0 ]; then + echo "::error::No expected modules supplied for ${report}." + return 1 + fi + mkdir -p "$(dirname "$output")" + jq -s --arg root "${GITHUB_WORKSPACE:-$(pwd -P)}" ' + def file_path: + (if startswith("file://localhost/") then ltrimstr("file://localhost") + elif startswith("file:/") then sub("^file:/+"; "/") + elif startswith("/") then . else error("Expected a file URI") end) as $path + | reduce ($path | split("/"))[] as $part ([]; + if $part == "" or $part == "." then . + elif $part == ".." then + if length > 0 then .[:-1] else error("Source path escapes filesystem root") end + else . + [$part] end) + | "/" + join("/") + (if ($path | endswith("/")) and length > 0 then "/" else "" end); + def resolve_uri($run; $root): + . as $location + | (if has("uriBaseId") then $run.originalUriBaseIds[.uriBaseId] | resolve_uri($run; $root) + else $root end) as $base + | ($location.uri // "") as $uri + | if ($uri | startswith("/") or startswith("file:/")) then $uri | file_path + elif ($uri | test("^[A-Za-z][A-Za-z0-9+.-]*:")) then error("Unsupported source URI scheme") + elif $uri == "" then $base + else (($base | sub("[^/]*$"; "")) + $uri) | file_path end; + def repository_locations($root): + . as $run | del(.originalUriBaseIds) + | walk(if type == "object" and (has("uri") or has("uriBaseId")) then + (resolve_uri($run; $root)) as $absolute + | if ($absolute | startswith($root)) then + .uri = ($absolute | ltrimstr($root)) | del(.uriBaseId) + else error("Source location is outside the repository: " + $absolute) end + else . end); + def identical: + unique | if length == 1 then .[0] else error("Conflicting SARIF metadata") end; + def descriptors: + group_by(.id) | map(identical); + def compatible_run: + if ((keys - ["tool", "results", "invocations", "originalUriBaseIds", "taxonomies"]) | length) > 0 + or any(.. | objects; has("index") or has("invocationIndex")) + then error("Unsupported run metadata or indexed reference; update the merger") else . end; + def resolve_rules: + .tool.driver.rules as $rules + | .results |= map(if has("ruleIndex") then + .ruleIndex as $index + | if ($index | type) != "number" or $index < 0 or ($index | floor) != $index + or $rules[$index].id == null + or (has("ruleId") and .ruleId != $rules[$index].id) + then error("Invalid ruleIndex") + else .ruleId = $rules[$index].id | del(.ruleIndex) end + else . end); + ($root | split("/") | map(@uri) | join("/") | rtrimstr("/") + "/") as $root_uri + | [.[].runs[] | compatible_run | resolve_rules | repository_locations($root_uri)] as $runs + | { + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + version: "2.1.0", + runs: [{ + tool: ([$runs[].tool | del(.driver.rules)] | identical + | .driver.rules = ([$runs[].tool.driver.rules[]?] | descriptors)), + taxonomies: ([$runs[].taxonomies[]?] | group_by([.name, .guid]) + | map(. as $group | map(del(.taxa)) | identical + | .taxa = ([$group[].taxa[]?] | descriptors))), + results: [$runs[].results[]?], + invocations: [$runs[].invocations[]?] + }] + } + ' "${inputs[@]}" > "$output" +} + +cpd_count() { + local report=$1 valid count + if [ ! -s "$report" ]; then + echo "::error::${report} is missing or empty." >&2 + return 1 + fi + valid=$(xmllint --nonet --xpath \ + 'boolean(/*[local-name()="pmd-cpd"] and not(//*[local-name()="error"]) and not(/*/*[local-name()!="duplication" and local-name()!="file"]))' \ + "$report") || return 1 + if [ "$valid" != "true" ]; then + echo "::error::${report} is not a successful CPD report." >&2 + return 1 + fi + count=$(xmllint --nonet --xpath 'count(/*/*[local-name()="duplication"])' "$report") || return 1 + printf '%s\n' "$count" +} diff --git a/.github/tests/analysis/README.md b/.github/tests/analysis/README.md new file mode 100644 index 0000000000..5213c2b85c --- /dev/null +++ b/.github/tests/analysis/README.md @@ -0,0 +1,53 @@ +# Analysis regression checks + +Run `bash .github/tests/analysis/run.sh` from any directory. Requires Bash, +Mike Farah yq v4, jq, and xmllint (Debian/Ubuntu: `libxml2-utils`). The verify +workflow runs the same suite. Tests use temporary directories and extract the +merge and gate blocks directly from the current workflow. + +The clean fixtures were emitted by Checkstyle 14.1.0, PMD 7.27.0, and SpotBugs +4.10.4. The error fixtures use those renderers with injected processing failures; +PMD omits the notification level, so its unsuccessful invocation is essential. +Checkstyle emits no invocation metadata on clean runs. The workflow therefore +also preserves the exit status of each Maven invocation with `--fail-at-end`. + +The merger supports the fields emitted by these configured producers: tool, +results, invocations, originalUriBaseIds, and taxonomies. It resolves URI-base chains and rewrites artifact locations to repository-relative +URIs, resolves result rule indices before unioning rule descriptors, retains taxonomy +descriptors, and rejects conflicting descriptors or unsupported run/indexed +metadata rather than silently dropping it. It is not a general SARIF merger. + +CPD fixtures were generated by PMD 7.27.0 CpdAnalysis with a normal Java source, +two duplicate sources, and an unterminated string (a real lexical error). The +SpotBugs finding fixture comes from an actual Java 21 analysis of a null +dereference. Machine-specific absolute source roots are normalized to /checkout. + +Run `bash .github/tests/analysis/mutations.sh` to verify that deliberate removal +of completion, notification, XML, URI, merged-report and process-exit protections +is detected. This suite also runs automatically in the verify workflow. + +The suite exercises valid findings separately from report failures, with a clean +sibling present. It also checks completion metadata, parser failures, raw and +merged reports, scoped report lists, CPD XML errors and structural counts, chained +URI bases, encoded paths, secondary locations, rule identities and severity. + + +`scope.sh` exercises the actual scope script against `fixtures/scope.bundle`, a +5.4 KB Git bundle containing signed fixture commits. It covers single and multiple +changed modules, docs-only and source-less changes, mixed changes, shared config, +workflow changes, deletions and moves. Module a depends on b. The tests check skip +injection, idempotence, report lists, reactor arguments, empty scopes and failures. +The bundle preserves actual Git diff behavior without requiring signing keys or +creating unsigned commits in CI. Regenerate it by creating signed scenario commits +in a temporary repository and bundling their branch refs with `git bundle create`. +The base SARIF head has no scope script; this suite runs on the descendant heads. + + +The fork annotation probe demonstrated that preserving arbitrary `uriBaseId` +values was insufficient: GitHub accepted the report but stored package-relative +paths that did not match the source tree. The merger now resolves file URIs and +base chains before emitting repository-relative artifact URIs without a base ID. +The repository root is `GITHUB_WORKSPACE`, or the current directory locally. +Encoded paths and dot segments are covered; locations outside this checkout or +using unsupported schemes fail explicitly. This is intentional for these source +analyzers, which report repository sources. diff --git a/.github/tests/analysis/fixtures/checkstyle-clean.json b/.github/tests/analysis/fixtures/checkstyle-clean.json new file mode 100644 index 0000000000..3b8ec573e9 --- /dev/null +++ b/.github/tests/analysis/fixtures/checkstyle-clean.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "downloadUri": "https://github.com/checkstyle/checkstyle/releases/", + "fullName": "Checkstyle", + "informationUri": "https://checkstyle.org/", + "language": "en", + "name": "Checkstyle", + "organization": "Checkstyle", + "rules": [ + + ], + "semanticVersion": "14.1.0", + "version": "14.1.0" + } + }, + "results": [ + + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/cpd-clean.xml b/.github/tests/analysis/fixtures/cpd-clean.xml new file mode 100644 index 0000000000..32e7629be7 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-clean.xml @@ -0,0 +1,4 @@ + + + + diff --git a/.github/tests/analysis/fixtures/cpd-error.xml b/.github/tests/analysis/fixtures/cpd-error.xml new file mode 100644 index 0000000000..ea6c6202c3 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-error.xml @@ -0,0 +1,22 @@ + + + + net.sourceforge.pmd.lang.ast.LexException: Lexical error in file 'b/src/Bad.java' at line 1, column 38: <EOF> after : "\"unterminated; }" (in lexical state DEFAULT) + at net.sourceforge.pmd.lang.ast.InternalApiBridge.newLexException(InternalApiBridge.java:25) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:2386) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:22) + at net.sourceforge.pmd.cpd.impl.BaseTokenFilter.getNextToken(BaseTokenFilter.java:44) + at net.sourceforge.pmd.cpd.impl.CpdLexerBase.tokenize(CpdLexerBase.java:40) + at net.sourceforge.pmd.cpd.CpdLexer.tokenize(CpdLexer.java:29) + at net.sourceforge.pmd.cpd.CpdAnalysis.doTokenize(CpdAnalysis.java:150) + at net.sourceforge.pmd.cpd.CpdAnalysis.tokenizeFiles(CpdAnalysis.java:225) + at net.sourceforge.pmd.cpd.CpdAnalysis.findMatches(CpdAnalysis.java:198) + at net.sourceforge.pmd.cpd.CpdAnalysis.performAnalysis(CpdAnalysis.java:164) + at CpdProbe.main(CpdProbe.java:15) + at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) + at java.base/java.lang.reflect.Method.invoke(Method.java:580) + at jdk.compiler/com.sun.tools.javac.launcher.Main.execute(Main.java:484) + at jdk.compiler/com.sun.tools.javac.launcher.Main.run(Main.java:208) + at jdk.compiler/com.sun.tools.javac.launcher.Main.main(Main.java:135) + + diff --git a/.github/tests/analysis/fixtures/cpd-finding.xml b/.github/tests/analysis/fixtures/cpd-finding.xml new file mode 100644 index 0000000000..9ae55681e1 --- /dev/null +++ b/.github/tests/analysis/fixtures/cpd-finding.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/.github/tests/analysis/fixtures/pmd-clean.json b/.github/tests/analysis/fixtures/pmd-clean.json new file mode 100644 index 0000000000..d92eadb63a --- /dev/null +++ b/.github/tests/analysis/fixtures/pmd-clean.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "PMD", + "version": "7.27.0", + "informationUri": "https://docs.pmd-code.org/latest/", + "rules": [] + } + }, + "results": [], + "invocations": [ + { + "executionSuccessful": true, + "toolConfigurationNotifications": [], + "toolExecutionNotifications": [] + } + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/pmd-error.json b/.github/tests/analysis/fixtures/pmd-error.json new file mode 100644 index 0000000000..e65d56146b --- /dev/null +++ b/.github/tests/analysis/fixtures/pmd-error.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "PMD", + "version": "7.27.0", + "informationUri": "https://docs.pmd-code.org/latest/", + "rules": [] + } + }, + "results": [], + "invocations": [ + { + "executionSuccessful": false, + "toolConfigurationNotifications": [], + "toolExecutionNotifications": [ + { + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "file://src/Bad.java" + } + } + } + ], + "message": { + "text": "RuntimeException: synthetic analyzer failure" + }, + "exception": { + "message": "java.lang.RuntimeException: synthetic analyzer failure\n\tat Probe.lambda$main$0(Probe.java:7)\n\tat net.sourceforge.pmd.util.BaseResultProducingCloseable.using(BaseResultProducingCloseable.java:66)\n\tat net.sourceforge.pmd.reporting.Report.buildReport(Report.java:262)\n\tat Probe.main(Probe.java:7)\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:580)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.execute(Main.java:484)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.run(Main.java:208)\n\tat jdk.compiler/com.sun.tools.javac.launcher.Main.main(Main.java:135)\n" + } + } + ] + } + ] + } + ] +} diff --git a/.github/tests/analysis/fixtures/scope.bundle b/.github/tests/analysis/fixtures/scope.bundle new file mode 100644 index 0000000000000000000000000000000000000000..12d84f0fe1cfef5dcdcf4bb8512fe9b13490827a GIT binary patch literal 5430 zcmbuCc|26>|G*DY8f#fnDiSrAVdl(wO6W$GAu*OVF*9dIcFhp2T9qgXp>&ljDalf# zd}SFKnh9fso23Y8p^PYz@1*X%dfo2#_s8##d7VGz`8?12d4HDkIxBVdfI8kjAv*NX zK-iy&AdoR28Apa749FzoFa#O~Cg3m_3=wA1AOeF%#9}cd2FTE1Gg%yyT}&FxF=4QS zI2?395FF~ylsxt@5eys=WPl_*OvaL6B9@6^Kp>F}(rE++3&zl}bP}CKWZ~f1yTD9; zW{6KvAOgY>a5yp^OJ_hVIu;KxF)TWc0Fp>p5)LB4bP`Br;$RT>Q*aQ2gANMx-;03o zbS8^Q#4#9jEQtw`KpK`ohp-F+okqjrSTM*SVemu@d3H3*Cp;vS&Ez0h3_1gkXMrHd zBogV88^beKU=fG}A`3&L!!VN$GD*bQ!2v$uOc+6eNDMrPftYk0lMFFIEP()G$aoMY zFlj^#mdF6fL>8WjpB))YV~6h7)dmp2};D0Xbc7(L!`lE42?{{(-?FTmQ0XbVfN{q zpinl0iS}o5I0!7Bj>D2j3<$!rAOe#~B7iI!QDRAwI4l#(z<>lAjR^guJR~gWN6tA2 zGEE86cN_se1R}+D)G}N24rj`p2<-a^nU{@f#Z6)Ta6=2AC$qUhna1VtVTj08~_hqf4pAPQzfMpg{ z6goWFdVTO~|J0^s#uw3|><@~Q1=X=x)1VA~2Vs0O)`^pxl$4dY6f{U!vfS#aTbZA? z(&5#SL+I_lE3R(c|H?H(K#qCiR{3F47+k0RXR<<>}N<%Y`9<#a)&snsnAH-Ygj`E}T4~d3tAcd`QmDH*yEN!#;1_ zHld$sE7wKy=d3rCvX~DUW>i(}Sz(f)xWFV)*Rrz2Rjc&S(uTt&C0&bOg^b*vA6Oa5 zYS2M!HtxYJ{*tUgvm1H!M5(ml)!{V?M=18IuJ9tLDIwrIDBLWoONjBO)Ix zEWB|4^&gATV-dGYz{cl@W4phRK34oj+UP~r%znP2u53(?GZJa2s~DV{ z+v>G2Y2?PrPhIZQuiT6v(iS@7r0TrY=kx*%$q(eT3f0&f&ne34tI69mJ*&Ul=10L9 z3(cg$X%jk!$Ag!xpTNhqI`4DnEf17i#NJ*jR%frcMq7O1`JF}0oU~>6GOIQ=?zzNt zGvCsc;}Kn$HSS87#GAS$@~xb5j>NDc9(Ukw^dn9z4Nu+QFp_Lll7S?+R`M$$Ei;ws3Gp&eTRZkDkgnkDThUsV)=hF`~=PyTszB>PIn2dfn+rv3>8+3hsH!u%e#E!VI;No&?=& zk#23{siSOT+10}+-0FzwEB4}iKB_sv!d#nS-TI9FBS{I=G8Og>OGM1 zTf`~38S?W#$gu(us9(p!v{bZJbszr&H zZ~0?tu@%8VLERR&{fFhUPs3t=leD)7hS|!DwEBe2B{w^xYt+VO;Tq(IatGOej<}B+u$9l!<3`nZNgA#Q2@y+s`or^*2zp7SWcT z$81){njRGE+`e-dG(K^Mhjh1|rjr@~$LM;)pmv!?_~=lvm~Hr-I#Z8O zIe<95XNEe#Dto4~d8SQ#U45uV#B1wlL$$Z@_(F0GA8fBc)p13_#wuhTSjXkT>6_+W z`wKc^^~&3Z*O1lR+!4gVUOD%Mg!BVNRoskYouIp1QKm(YVc28iFD8|nE5-6p><+749Zd@pT- zi-fYbh!}&}1yaewv~xv#E}2l%!P9AJ2I*~3n z`S-ZY)7kxQ^KpvT2|s2j=e)bk-U7$Sf+LynySHs~3f#9PxY*X^NNl`caJgqI%kmj_uUtk2uO*J+EBEsaH6oOJXmjp@5J&>Wyp}{!JOdHPULIXO1Due-S18hc7AGR*Yb&-!(Y)Ye$L(rHFhU02KBO1)GWj= zk(E2uxBlLBi#;%}LT>$Ux75Q^aacfF_CKTi^D#obNF$$G|6}IlGAQ8Q%EI!&)W=_3 zym1?3WN@mg`RDngWs~VHk-JJZm?O?w{>RpDaP0%Duk`0HW!y_FTxCnw-2Zg{bzc^y z%>2_BSAG7aw|d>v<1IIDqBxsElP?}dQ{e0Su9o)=e1)7c-)+55Lk#Zl<* zZ?)$L_ZXfF@Ky+GT%oN!Ml{n{etm}%eJ5f=edoD`^J%TQ+xJ-ccywP%j5QkDPTcU4 z>RcH5ecn)_7g9j6STk_z#H-02FZo<_Xz6ILDorI4u&ZcHv@e^|Z#JKXfS$OJW3g6+b zv(4jNqygHlPa0S?E~{7~6`pEji?VY>Hw_4lHlgb6n*|QCZ0cw!K%bn~MGjBp{Iy=m zKzbx&?vI>$wyIcrIGU3p117)xo?z(su`o3h(;XJ@=|Nq5y_TY)0^i~=psuQPrQ(G*i(*7`?;0`QThY;{i(A$H(D zwx^(tn-nQ%!zcWPB4>;U_Z(q|MhQ@MEun@2N}a7lLyt6|11j#ymkCcTGCL_WtH*ke{%%ivf-n7wk?nCP!u@4$TVzuN!Z1V-RJsI zQ}6b7k(NKF%Lu zNKo+D`KdL$SIsC(xn;Sha{GK~z=ykP`d<_#KU***vb6Cnx5=tcVhI|*#C8qtlL5Lq z6r+kICFkD@>lAtv?q4SWj!g*32$BV&>IcR?Tx1-9s?H+ce@SFguG$;ZPz@T**Qow@2e zlOY1{*&N{2dnF&q4n+nt3rl*X0TmnR?rXDl;5D>-i$7&I>QOk|9{<_cN1nN3sHjxK z*R&4++LtQYJwW{AyiVX~AUQR%=pMA|w)xP!?$)EXOR2vjY#=G1^@GC4 zA*s_=;i*!~ffv)A(w)-h;?x?@`B%tai()vS?81_;yL& zY*SQpM#zC<5m~uw&4nL3d6g<|zaE>L%OiSodQi(%ljQr% zep|frk0y2(6&RORT=)0Xd}70=I66r3uir4IaZoBsQiwD6EHt<-!i4O7;n{nJnwxCq zvj4z3c(ojGCRg69U*^bKi8>-s_g!76+Sqoki3+gb%S!W<_9lEtyi>1wN?a6DEx!1} zy&7&|={FB}YlVeP)hIC%L}K?|izl)*N5ot?FBnKUPom z0f4ic5S9PdeW$*1sAg@J-Z_U#4+p(p5cQct7&@u>o9OdEhJ`2>tzIxB3Sm>}oqJw1 z2i7_6-tG2Ks;yA#+QWZNFE<|F$jRorg#x$hl38~55TfP((T}O0IJo!cc%HUNPX~fl zdu%aq3(n~WT|Cww%G+rbs8p@~+1=3m!1VEvwV%p04hSxfxLDNB2Tr@810*jcd9;l! z%Jy426rV^7`(=*n{VZCUh%@b!5qXlzCIn9uJ6(ueOD>H|}*}kZ<)e zDHqLnVMANo2md16lB2)uX|mBr_O0tt?z;}gd@OCSwhS|R(k}-9Lk5an_h#ncx2AUc zQ;VyifuVz|@LP`utdTi}HxP zfwKb@Y?*NZmTGXPi+Hg(H!n-1n|JnYPH*@_=$NEdO4GGg=Sz@&m~s01HIKKF{X8Uf zTz)o6!M|-Y)vy3JB=rqtR>$Y+KgD&zR}%Kr=&F;~bYwleG8ZT5__l3ICWBHj4u6*C zzI9d)5GaK0N|JwBIVy+pua3n0Fe8j1P@6a24_?JPkD{6WhuJV TX7Z4hrsW)*_FCE2b-Mor-{rsz literal 0 HcmV?d00001 diff --git a/.github/tests/analysis/fixtures/spotbugs-clean.json b/.github/tests/analysis/fixtures/spotbugs-clean.json new file mode 100644 index 0000000000..1e6ebd918f --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-clean.json @@ -0,0 +1 @@ +{"version":"2.1.0","$schema":"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json","runs":[{"tool":{"extensions":[{"version":"4.10.4","name":"edu.umd.cs.findbugs.plugins.core","shortDescription":{"text":"Core SpotBugs plugin"},"informationUri":"https://github.com/spotbugs","organization":"SpotBugs project"}],"driver":{"name":"SpotBugs","version":"4.10.4","language":"en","informationUri":"https://spotbugs.github.io/","rules":[],"supportedTaxonomies":[{"name":"CWE","guid":"b8c54a32-de19-51d2-9a08-f0abfbaa7310"}]}},"invocations":[{"exitCode":0,"exitCodeDescription":"SUCCESS","executionSuccessful":true}],"results":[],"originalUriBaseIds":{},"taxonomies":[]}]} \ No newline at end of file diff --git a/.github/tests/analysis/fixtures/spotbugs-error.json b/.github/tests/analysis/fixtures/spotbugs-error.json new file mode 100644 index 0000000000..6f67129348 --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-error.json @@ -0,0 +1 @@ +{"version":"2.1.0","$schema":"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json","runs":[{"tool":{"extensions":[{"version":"4.10.4","name":"edu.umd.cs.findbugs.plugins.core","shortDescription":{"text":"Core SpotBugs plugin"},"informationUri":"https://github.com/spotbugs","organization":"SpotBugs project"}],"driver":{"name":"SpotBugs","version":"4.10.4","language":"en","informationUri":"https://spotbugs.github.io/","rules":[],"supportedTaxonomies":[{"name":"CWE","guid":"b8c54a32-de19-51d2-9a08-f0abfbaa7310"}]}},"invocations":[{"exitCode":4,"exitCodeDescription":"ERROR","executionSuccessful":false,"toolExecutionNotifications":[{"descriptor":{"id":"spotbugs-error-0"},"message":{"text":"Synthetic detector failure"},"level":"error"}]}],"results":[],"originalUriBaseIds":{},"taxonomies":[]}]} \ No newline at end of file diff --git a/.github/tests/analysis/fixtures/spotbugs-finding.json b/.github/tests/analysis/fixtures/spotbugs-finding.json new file mode 100644 index 0000000000..eb04cfcd2b --- /dev/null +++ b/.github/tests/analysis/fixtures/spotbugs-finding.json @@ -0,0 +1,155 @@ +{ + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "extensions": [ + { + "version": "4.10.4", + "name": "edu.umd.cs.findbugs.plugins.core", + "shortDescription": { + "text": "Core SpotBugs plugin" + }, + "informationUri": "https://github.com/spotbugs", + "organization": "SpotBugs project" + } + ], + "driver": { + "name": "SpotBugs", + "version": "4.10.4", + "language": "en", + "informationUri": "https://spotbugs.github.io/", + "rules": [ + { + "id": "NP_ALWAYS_NULL", + "shortDescription": { + "text": "Null pointer dereference." + }, + "fullDescription": { + "text": "A null pointer is dereferenced here.\u00a0 This will lead to a NullPointerException \nwhen the code is executed." + }, + "messageStrings": { + "default": { + "text": "Null pointer dereference of {0} in {1}." + } + }, + "helpUri": "https://spotbugs.readthedocs.io/en/latest/bugDescriptions.html#NP_ALWAYS_NULL", + "properties": { + "tags": [ + "CORRECTNESS" + ] + } + }, + { + "id": "NP_LOAD_OF_KNOWN_NULL_VALUE", + "shortDescription": { + "text": "Load of known null value." + }, + "fullDescription": { + "text": "The variable referenced at this point is known to be null due to an earlier \ncheck against null. Although this is valid, it might be a mistake (perhaps you \nintended to refer to a different variable, or perhaps the earlier check to see \nif the variable is null should have been a check to see if it was non-null)." + }, + "messageStrings": { + "default": { + "text": "Load of known null value in {0}." + } + }, + "helpUri": "https://spotbugs.readthedocs.io/en/latest/bugDescriptions.html#NP_LOAD_OF_KNOWN_NULL_VALUE", + "properties": { + "tags": [ + "STYLE" + ] + } + } + ], + "supportedTaxonomies": [ + { + "name": "CWE", + "guid": "b8c54a32-de19-51d2-9a08-f0abfbaa7310" + } + ] + } + }, + "invocations": [ + { + "exitCode": 1, + "exitCodeDescription": "BUGS FOUND", + "executionSuccessful": true + } + ], + "results": [ + { + "ruleId": "NP_ALWAYS_NULL", + "ruleIndex": 0, + "message": { + "id": "default", + "text": "Null pointer dereference", + "arguments": [ + "s", + "com.example.Example.main(String[])" + ] + }, + "level": "error", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "com/example/Example.java", + "uriBaseId": "2139602823" + }, + "region": { + "startLine": 2 + } + }, + "logicalLocations": [ + { + "name": "s", + "kind": "variable", + "fullyQualifiedName": "com.example.Example.main(String[])#s" + } + ] + } + ] + }, + { + "ruleId": "NP_LOAD_OF_KNOWN_NULL_VALUE", + "ruleIndex": 1, + "message": { + "id": "default", + "text": "Load of known null value", + "arguments": [ + "com.example.Example.main(String[])" + ] + }, + "level": "note", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "com/example/Example.java", + "uriBaseId": "2139602823" + }, + "region": { + "startLine": 2 + } + }, + "logicalLocations": [ + { + "name": "s", + "kind": "variable", + "fullyQualifiedName": "com.example.Example.main(String[])#s" + } + ] + } + ] + } + ], + "originalUriBaseIds": { + "2139602823": { + "uri": "file:///checkout/module/src/" + } + }, + "taxonomies": [] + } + ] +} diff --git a/.github/tests/analysis/mutations.sh b/.github/tests/analysis/mutations.sh new file mode 100644 index 0000000000..e5a8fd62f0 --- /dev/null +++ b/.github/tests/analysis/mutations.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +repo=$(cd "$(dirname "$0")/../../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mutant="$scratch/repo" +mutation() { + local label=$1 file=$2 needle=$3 replacement=$4 case_filter=$5 content + rm -rf "$mutant" + mkdir -p "$mutant/.github/scripts" "$mutant/.github/workflows" "$mutant/.github/tests" + cp -R "$repo/.github/tests/analysis" "$mutant/.github/tests/" + cp "$repo/.github/scripts/sarif.sh" "$mutant/.github/scripts/" + cp "$repo/.github/workflows/verify.yml" "$mutant/.github/workflows/" + content=$(cat "$mutant/$file") + if [[ "$content" != *"$needle"* ]]; then echo "Mutation anchor missing: $label"; exit 1; fi + content=${content//"$needle"/"$replacement"} + printf '%s\n' "$content" > "$mutant/$file" + if CASE_FILTER="$case_filter" bash "$mutant/.github/tests/analysis/run.sh" > "$scratch/$label.log" 2>&1; then + echo "FAIL: regression escaped detection: $label"; exit 1 + fi + grep -q 'FAIL:' "$scratch/$label.log" + echo "PASS: detected $label" +} +helper=.github/scripts/sarif.sh +workflow=.github/workflows/verify.yml +mutation execution-status "$helper" 'and .executionSuccessful == true' 'and true' spotbugs/execution-false +mutation error-notification "$helper" '.level != "error"' 'true' pmd/execution-error +mutation completion-metadata "$helper" 'and ($analyzer == "Checkstyle" or (.invocations | type == "array" and length > 0))' 'and true' spotbugs/missing-invocations +mutation cpd-parser "$helper" 'local report=$1 valid count' 'local report=$1 valid count; echo 0; return 0' cpd/truncated +mutation cpd-count "$helper" "'count(/*/*[local-name()=\"duplication\"])'" "'0'" cpd/minified +mutation uri-base-reference "$helper" '.uri = ($absolute | ltrimstr($root)) | del(.uriBaseId)' 'del(.uriBaseId)' spotbugs/locations-and-rules +mutation merged-validation "$workflow" 'validate_sarif .sarif-merged/spotbugs.sarif SpotBugs' ':' spotbugs/merged-execution-false +mutation maven-failure-status "$workflow" '--fail-at-end' '--fail-never' spotbugs/clean diff --git a/.github/tests/analysis/run.sh b/.github/tests/analysis/run.sh new file mode 100644 index 0000000000..bb2c7351c9 --- /dev/null +++ b/.github/tests/analysis/run.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +trap 'echo "FAIL: ${tool:-harness}/${case:-setup} line $LINENO"' ERR +repo=$(cd "$(dirname "$0")/../../.." && pwd) +fixtures="$repo/.github/tests/analysis/fixtures" +for dependency in bash jq yq xmllint; do command -v "$dependency" >/dev/null; done +yq --version | grep -Eq 'version v?4\.' +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +for job in lint spotbugs; do + for block in merge gate; do + if [ "$block" = merge ]; then prefix='Merge per-module'; else prefix='Gate on'; fi + JOB=$job PREFIX=$prefix yq -r '.jobs[strenv(JOB)].steps[] | select(.name | test("^" + strenv(PREFIX))) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/$job-$block.sh" + test -s "$scratch/$job-$block.sh" + bash -n "$scratch/$job-$block.sh" + done +done +# Every analyzer invocation must preserve a nonzero process exit status. +yq -o=json '.jobs' "$repo/.github/workflows/verify.yml" | jq -e ' + [.. | objects | .run? // empty | select(test("mvn .*--batch-mode"))] as $commands + | ($commands | length >= 3) and all($commands[]; contains("--fail-at-end") and (contains("--fail-never") | not))' >/dev/null +passed=0 +fresh() { + folder="$scratch/case" + rm -rf "$folder" + mkdir -p "$folder/ddk-parent/target" "$folder/.github/scripts" + cp "$repo/.github/scripts/sarif.sh" "$folder/.github/scripts/" + printf '\n../a\n../b\n\n' > "$folder/ddk-parent/pom.xml" + printf '\n' > "$folder/ddk-parent/target/checkstyle-result.xml" + for module in a b; do + mkdir -p "$folder/$module/target" "$folder/$module/src" "$folder/$module/META-INF" + touch "$folder/$module/META-INF/MANIFEST.MF" + cp "$fixtures/spotbugs-clean.json" "$folder/$module/target/spotbugsSarif.json" + cp "$fixtures/pmd-clean.json" "$folder/$module/target/pmd.sarif.json" + cp "$fixtures/checkstyle-clean.json" "$folder/$module/target/checkstyle-result.xml" + printf '\n' > "$folder/$module/target/cpd.xml" + done +} +edit() { jq "$1" "$target" > "$target.tmp"; mv "$target.tmp" "$target"; } +invoke() { + local block=$1 + (cd "$folder" && GITHUB_WORKSPACE="${FIXTURE_ROOT:-$folder}" bash -e "$scratch/$job-$block.sh") > "$scratch/$block.log" 2>&1 +} +expect() { + local block=$1 wanted=$2 status=0 + invoke "$block" || status=$? + if { [ "$wanted" = pass ] && [ "$status" -ne 0 ]; } || { [ "$wanted" = fail ] && [ "$status" -eq 0 ]; }; then + echo "FAIL: $tool/$case $block expected $wanted, exit $status" + cat "$scratch/$block.log" + exit 1 + fi +} +for tool in spotbugs pmd checkstyle; do + job=lint + case "$tool" in + spotbugs) job=spotbugs; filename=spotbugsSarif.json ;; + pmd) filename=pmd.sarif.json ;; + checkstyle) filename=checkstyle-result.xml ;; + esac + for case in clean finding execution-false execution-error configuration-error warning missing empty whitespace truncated \ + two-documents empty-runs missing-results null-results bad-message wrong-tool missing-invocations empty-invocations \ + null-invocations later-run-failure merged-truncated merged-missing merged-execution-false scoped-report-list real-error real-finding; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + unset FIXTURE_ROOT + fresh + target="$folder/b/target/$filename" + wanted=fail; merge_wanted=fail + case "$case" in + clean) wanted=pass; merge_wanted=pass ;; + finding) edit '.runs[0].results=[{ruleId:"fixture",message:{text:"Deliberate finding"}}]'; merge_wanted=pass ;; + execution-false) edit '.runs[0].invocations=[{executionSuccessful:false}]' ;; + execution-error) edit '.runs[0].invocations=[{executionSuccessful:true,toolExecutionNotifications:[{level:"error",message:{text:"error"}}]}]' ;; + configuration-error) edit '.runs[0].invocations=[{executionSuccessful:true,toolConfigurationNotifications:[{level:"error",message:{text:"error"}}]}]' ;; + warning) edit '.runs[0].invocations=[{executionSuccessful:true,toolExecutionNotifications:[{level:"warning",message:{text:"warning"}}]}]'; wanted=pass; merge_wanted=pass ;; + missing) rm "$target" ;; + empty) : > "$target" ;; + whitespace) printf ' \n\t' > "$target" ;; + truncated) printf '{"version":' > "$target" ;; + two-documents) cat "$target" "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + empty-runs) edit '.runs=[]' ;; + missing-results) edit 'del(.runs[0].results)' ;; + null-results) edit '.runs[0].results=null' ;; + bad-message) edit '.runs[0].results=[{message:null}]' ;; + wrong-tool) edit '.runs[0].tool.driver.name="Other"' ;; + missing-invocations) edit 'del(.runs[0].invocations)'; if [ "$tool" = checkstyle ]; then wanted=pass; merge_wanted=pass; fi ;; + empty-invocations) edit '.runs[0].invocations=[]'; if [ "$tool" = checkstyle ]; then wanted=pass; merge_wanted=pass; fi ;; + null-invocations) edit '.runs[0].invocations=null' ;; + later-run-failure) edit '.runs += [.runs[0]] | .runs[1].invocations=[{executionSuccessful:false}]' ;; + merged-truncated|merged-missing|merged-execution-false) merge_wanted=pass ;; + scoped-report-list) printf '{' > "$target"; export SPOTBUGS_EXPECT_REPORTS=a LINT_EXPECT_REPORTS=a; wanted=pass; merge_wanted=pass ;; + real-finding) if [ "$tool" != spotbugs ]; then continue; fi; cp "$fixtures/spotbugs-finding.json" "$target"; export FIXTURE_ROOT=/checkout; merge_wanted=pass ;; + real-error) if [ "$tool" = checkstyle ]; then continue; fi; cp "$fixtures/$tool-error.json" "$target" ;; + esac + expect merge "$merge_wanted" + if [ "$case" = merged-truncated ]; then printf '{' > "$folder/.sarif-merged/$tool.sarif"; fi + if [ "$case" = merged-missing ]; then rm "$folder/.sarif-merged/$tool.sarif"; fi + if [ "$case" = merged-execution-false ]; then + target="$folder/.sarif-merged/$tool.sarif" + edit '.runs[0].invocations=[{executionSuccessful:false}]' + fi + expect gate "$wanted" + if [ "$case" = finding ]; then grep -q 'found violations' "$scratch/gate.log"; fi + unset SPOTBUGS_EXPECT_REPORTS LINT_EXPECT_REPORTS + passed=$((passed + 1)) + echo "PASS: $tool/$case" + done +done +tool=cpd; job=lint +for case in clean finding minified multiple comment cdata missing empty truncated not-xml wrong-root error namespaced-error unexpected-child real-clean real-finding real-error; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + target="$folder/b/target/cpd.xml" + wanted=fail + case "$case" in + real-clean) cp "$fixtures/cpd-clean.xml" "$target"; wanted=pass ;; + real-finding) cp "$fixtures/cpd-finding.xml" "$target" ;; + real-error) cp "$fixtures/cpd-error.xml" "$target" ;; + clean) wanted=pass ;; + finding) printf '\n\n' > "$target" ;; + minified) printf '' > "$target" ;; + multiple) printf '' > "$target" ;; + comment) printf '' > "$target"; wanted=pass ;; + cdata) printf ']]>' > "$target"; wanted=pass ;; + missing) rm "$target" ;; + empty) : > "$target" ;; + truncated) printf '\n' > "$target" ;; + not-xml) printf 'analysis crashed\n' > "$target" ;; + wrong-root) printf '' > "$target" ;; + error) printf '' > "$target" ;; + namespaced-error) printf '' > "$target" ;; + unexpected-child) printf '' > "$target" ;; + esac + expect merge pass + expect gate "$wanted" + case "$case" in + finding|minified) grep -q 'CPD duplications: 1' "$scratch/gate.log" ;; + multiple) grep -q 'CPD duplications: 2' "$scratch/gate.log" ;; + esac + passed=$((passed + 1)); echo "PASS: cpd/$case" +done +# Exercise collisions, chained bases, encoded paths, secondary locations and different rule indices. +for tool in spotbugs pmd checkstyle; do + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/locations-and-rules" ]; then continue; fi + fresh + case "$tool" in + spotbugs) job=spotbugs; filename=spotbugsSarif.json ;; + pmd) job=lint; filename=pmd.sarif.json ;; + checkstyle) job=lint; filename=checkstyle-result.xml ;; + esac + for module in a b; do + target="$folder/$module/target/$filename" + jq --arg module "$module" ' + .runs[0] |= (.originalUriBaseIds={ROOT:{uri:"file:///checkout/"},SRC:{uri:($module+"/src/"),uriBaseId:"ROOT"}} + | .tool.driver.rules=[{id:$module,shortDescription:{text:$module}}] + | .results=[{ruleId:$module,ruleIndex:0,level:"warning",message:{text:("finding-"+$module)}, + locations:[{physicalLocation:{artifactLocation:{uri:"Space%20Name.java",uriBaseId:"SRC"}}}], + relatedLocations:[{id:1,physicalLocation:{artifactLocation:{uri:"Related.java",uriBaseId:"SRC"}}}]}])' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done + case=locations-and-rules + export FIXTURE_ROOT=/checkout + expect merge pass + expect gate fail + jq -e '.runs[0] as $run | ($run.results | length==2) + and ([$run.tool.driver.rules[].id] | sort == ["a","b"]) + and all($run.results[]; . as $result + | .level=="warning" and .message.text==("finding-"+.ruleId) + and (has("ruleIndex") | not) + and all(.locations[], .relatedLocations[]; + .physicalLocation.artifactLocation as $loc + | ($loc | has("uriBaseId") | not) + and ($loc.uri == ($result.ruleId+"/src/Space%20Name.java") + or $loc.uri == ($result.ruleId+"/src/Related.java"))))' \ + "$folder/.sarif-merged/$tool.sarif" >/dev/null + unset FIXTURE_ROOT + passed=$((passed + 1)); echo "PASS: $tool/$case" +done +# GitHub consumes repository-relative artifact URIs, not arbitrary URI-base identifiers. +for case in relative-uri absolute-uri file-uri dot-segments encoded-root outside-repository unknown-scheme; do + tool=spotbugs; job=spotbugs + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + export FIXTURE_ROOT=/checkout + target="$folder/b/target/spotbugsSarif.json" + wanted=pass + expected='a/src/Example.java' + case "$case" in + relative-uri) uri='a/src/Example.java' ;; + absolute-uri) uri='/checkout/a/src/Example.java' ;; + file-uri) uri='file:/checkout/a/src/Example.java' ;; + dot-segments) uri='file:///checkout/a/other/../src/./Example.java' ;; + encoded-root) export FIXTURE_ROOT='/checkout space'; uri='file:///checkout%20space/a/src/Space%20Name.java'; expected='a/src/Space%20Name.java' ;; + outside-repository) uri='file:///unrelated/src/Example.java'; wanted=fail ;; + unknown-scheme) uri='https://example.invalid/Example.java'; wanted=fail ;; + esac + jq --arg uri "$uri" '.runs[0].results=[{message:{text:"location probe"},locations:[{physicalLocation:{artifactLocation:{uri:$uri}}}]}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + expect merge "$wanted" + if [ "$wanted" = pass ]; then + expect gate fail + jq -e --arg expected "$expected" '.runs[0].results[0].locations[0].physicalLocation.artifactLocation | .uri==$expected and (has("uriBaseId")|not)' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + fi + unset FIXTURE_ROOT + passed=$((passed + 1)); echo "PASS: $tool/$case" +done + +# Metadata that cannot be combined without loss must fail explicitly. +for case in taxonomies conflicting-rules rule-index-only wrong-rule-index unsupported-run dangling-base cyclic-base; do + tool=spotbugs; job=spotbugs + if [ -n "${CASE_FILTER:-}" ] && [ "$CASE_FILTER" != "$tool/$case" ]; then continue; fi + fresh + target="$folder/b/target/spotbugsSarif.json" + wanted=fail + case "$case" in + taxonomies) + for module in a b; do + target="$folder/$module/target/spotbugsSarif.json" + jq --arg module "$module" '.runs[0].taxonomies=[{name:"CWE",guid:"shared",taxa:[{id:$module,shortDescription:{text:$module}}]}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done + wanted=pass ;; + conflicting-rules) + for module in a b; do + target="$folder/$module/target/spotbugsSarif.json" + jq --arg module "$module" '.runs[0].tool.driver.rules=[{id:"shared",shortDescription:{text:$module}}]' "$target" > "$target.tmp" + mv "$target.tmp" "$target" + done ;; + rule-index-only) + edit '.runs[0] |= (.tool.driver.rules=[{id:"resolved"}] | .results=[{ruleIndex:0,message:{text:"indexed rule"}}])' + wanted=pass ;; + wrong-rule-index) + edit '.runs[0] |= (.tool.driver.rules=[{id:"first"}] | .results=[{ruleId:"other",ruleIndex:0,message:{text:"mismatch"}}])' ;; + unsupported-run) edit '.runs[0].artifacts=[{location:{uri:"src/Example.java"}}]' ;; + dangling-base) edit '.runs[0].originalUriBaseIds={SRC:{uri:"src/",uriBaseId:"MISSING"}}' ;; + cyclic-base) edit '.runs[0].originalUriBaseIds={SRC:{uri:"src/",uriBaseId:"SRC"}}' ;; + esac + expect merge "$wanted" + if [ "$case" = taxonomies ]; then + expect gate pass + jq -e '.runs[0].taxonomies | length==1 and (.[0].taxa | map(.id) | sort==["a","b"])' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + elif [ "$case" = rule-index-only ]; then + expect gate fail + jq -e '.runs[0].results[0] | .ruleId=="resolved" and (has("ruleIndex") | not)' "$folder/.sarif-merged/spotbugs.sarif" >/dev/null + fi + passed=$((passed + 1)); echo "PASS: $tool/$case" +done + +# Maven failures remain failures even if usable reports already exist. +if [ -z "${CASE_FILTER:-}" ]; then + fresh + mkdir "$scratch/bin" + cat > "$scratch/bin/mvn" <<'STUB' +#!/usr/bin/env bash +printf 'process reached\n' > "$PROCESS_MARKER" +exit 42 +STUB + chmod +x "$scratch/bin/mvn" + for name in 'PMD + Checkstyle reports (SARIF)' 'CPD report (separate invocation — no SARIF support)' 'SpotBugs report (SARIF)'; do + NAME="$name" yq -r '.jobs[].steps[] | select(.name == strenv(NAME)) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/process.sh" + rm -f "$scratch/process-marker" + status=0 + (cd "$folder" && PATH="$scratch/bin:$PATH" PROCESS_MARKER="$scratch/process-marker" bash -e "$scratch/process.sh") || status=$? + test "$status" -eq 42 + test -s "$scratch/process-marker" + passed=$((passed + 1)); echo "PASS: process failure/$name" + done +fi +test "$passed" -gt 0 +echo "$passed analysis regression cases passed." diff --git a/.github/tests/analysis/scope.sh b/.github/tests/analysis/scope.sh new file mode 100644 index 0000000000..d9ccfe4f98 --- /dev/null +++ b/.github/tests/analysis/scope.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +trap 'echo "FAIL: scope ${mode:-setup}/${scenario:-setup} line $LINENO"' ERR +repo=$(cd "$(dirname "$0")/../../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +script=compute-spotbugs-skip.sh +modes=spotbugs +if [ ! -f "$repo/.github/scripts/$script" ] && [ ! -f "$repo/.github/scripts/compute-analysis-skip.sh" ]; then + echo 'Scope tests: this head uses the full reactor.' + exit 0 +fi +if [ -f "$repo/.github/scripts/compute-analysis-skip.sh" ]; then + script=compute-analysis-skip.sh + modes='spotbugs lint' +fi +for mode in $modes; do + prefix=$(printf '%s' "$mode" | tr '[:lower:]' '[:upper:]') + for block in merge gate; do + if [ "$block" = merge ]; then starts='Merge per-module'; else starts='Gate on'; fi + JOB=$mode PREFIX=$starts yq -r '.jobs[strenv(JOB)].steps[] | select(.name | test("^" + strenv(PREFIX))) | .run' \ + "$repo/.github/workflows/verify.yml" > "$scratch/$block.sh" + done + for scenario in partial two-modules docs-only source-less mixed shared-config workflow deletion cross-module-move; do + folder="$scratch/$mode-$scenario" + git clone --quiet --no-checkout "$repo/.github/tests/analysis/fixtures/scope.bundle" "$folder" + git -C "$folder" checkout --quiet --detach "origin/$scenario" + base=$(git -C "$folder" rev-parse HEAD^) + environment="$folder/environment" + mkdir -p "$folder/.github/scripts" + cp "$repo/.github/scripts/$script" "$repo/.github/scripts/sarif.sh" "$folder/.github/scripts/" + unset SPOTBUGS_KEPT SPOTBUGS_EXPECT_REPORTS SPOTBUGS_SCOPE_ARGS LINT_KEPT LINT_EXPECT_REPORTS LINT_SCOPE_ARGS + args=("$base") + if [ "$script" = compute-analysis-skip.sh ]; then args+=("$mode"); fi + for iteration in 1 2; do + (cd "$folder" && GITHUB_ENV="$environment" bash ".github/scripts/$script" "${args[@]}") > "$scratch/scope.log" 2>&1 + git -C "$folder" diff > "$scratch/injection-$iteration.diff" + done + cmp "$scratch/injection-1.diff" "$scratch/injection-2.diff" + if [ -f "$environment" ]; then + while IFS= read -r assignment; do export "$assignment"; done < "$environment" + fi + case "$scenario" in + partial|mixed|deletion) wanted='a' ;; + two-modules|shared-config|workflow|cross-module-move) wanted='a b' ;; + docs-only|source-less) wanted='' ;; + esac + key="${prefix}_KEPT"; kept=${!key:-all} + key="${prefix}_EXPECT_REPORTS"; expected=${!key:-a b} + if [ -z "$wanted" ]; then test "$kept" = 0; else test "$kept" != 0; test "$expected" = "$wanted"; fi + full=false + if [ "$scenario" = shared-config ] || [ "$scenario" = workflow ]; then full=true; fi + props=spotbugs.skip + if [ "$mode" = lint ]; then props='pmd.skip cpd.skip checkstyle.skip'; fi + for module in a b brand feature ddk-target; do + skip=true + case "$scenario:$module" in + partial:a|two-modules:a|two-modules:b|source-less:brand|mixed:a|mixed:brand|deletion:a|cross-module-move:a|cross-module-move:b) skip=false ;; + esac + if [ "$full" = true ]; then skip=false; fi + for prop in $props; do + count=$(grep -Fc "<$prop>true" "$folder/$module/pom.xml" || true) + if [ "$skip" = true ]; then test "$count" -eq 1; else test "$count" -eq 0; fi + done + done + key="${prefix}_SCOPE_ARGS"; scope_args=${!key:-} + if grep -q 'SCOPE_ARGS=' "$repo/.github/scripts/$script"; then + if [ -n "$wanted" ] && [ "$full" = false ]; then + case "$scenario" in + mixed) selected='../a,../brand' ;; + two-modules|cross-module-move) selected='../a,../b' ;; + *) selected='../a' ;; + esac + test "$scope_args" = "-pl ../ddk-target,$selected -am" + else test -z "$scope_args"; fi + fi + for module in $wanted; do + mkdir -p "$folder/$module/target" + cp "$repo/.github/tests/analysis/fixtures/spotbugs-clean.json" "$folder/$module/target/spotbugsSarif.json" + cp "$repo/.github/tests/analysis/fixtures/pmd-clean.json" "$folder/$module/target/pmd.sarif.json" + cp "$repo/.github/tests/analysis/fixtures/checkstyle-clean.json" "$folder/$module/target/checkstyle-result.xml" + cp "$repo/.github/tests/analysis/fixtures/cpd-clean.xml" "$folder/$module/target/cpd.xml" + done + for block in merge gate; do (cd "$folder" && bash -e "$scratch/$block.sh") > "$scratch/$block.log" 2>&1; done + if [ -z "$wanted" ]; then test ! -d "$folder/.sarif-merged"; fi + if [ -n "$wanted" ]; then + tools=spotbugs + if [ "$mode" = lint ]; then tools='pmd checkstyle'; fi + for tool in $tools; do + case "$tool" in spotbugs) filename=spotbugsSarif.json ;; pmd) filename=pmd.sarif.json ;; checkstyle) filename=checkstyle-result.xml ;; esac + for fault in missing invalid failed finding; do + target="$folder/a/target/$filename" + cp "$repo/.github/tests/analysis/fixtures/$tool-clean.json" "$target" + case "$fault" in + missing) rm "$target" ;; + invalid) printf '{' > "$target" ;; + failed) jq '.runs[0].invocations=[{executionSuccessful:false}]' "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + finding) jq '.runs[0].results=[{message:{text:"scoped finding"}}]' "$target" > "$target.tmp"; mv "$target.tmp" "$target" ;; + esac + for block in merge gate; do + status=0 + (cd "$folder" && bash -e "$scratch/$block.sh") > "$scratch/$block.log" 2>&1 || status=$? + if [ "$fault:$block" = finding:merge ]; then test "$status" -eq 0; else test "$status" -ne 0; fi + done + done + cp "$repo/.github/tests/analysis/fixtures/$tool-clean.json" "$folder/a/target/$filename" + done + fi + echo "PASS: scope $mode/$scenario" + done + unset SPOTBUGS_KEPT SPOTBUGS_EXPECT_REPORTS SPOTBUGS_SCOPE_ARGS LINT_KEPT LINT_EXPECT_REPORTS LINT_SCOPE_ARGS + if (cd "$folder" && GITHUB_ENV="$scratch/invalid-base-env" bash ".github/scripts/$script" does-not-exist "$mode") > "$scratch/invalid-base.log" 2>&1; then + echo 'FAIL: invalid base was accepted'; exit 1 + fi + test ! -s "$scratch/invalid-base-env" + echo "PASS: scope $mode/invalid-base" +done diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2322f216bc..4e16cd0eb8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,8 +1,38 @@ name: verify on: pull_request: + +# Code Scanning needs write access to upload SARIF results for inline annotations. +permissions: + contents: read + security-events: write + jobs: - pmd: + analysis-regression: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install validation dependencies + run: sudo apt-get update && sudo apt-get install --yes jq libxml2-utils + - name: Check report gates and merge semantics + run: | + bash .github/tests/analysis/run.sh + bash .github/tests/analysis/mutations.sh + bash .github/tests/analysis/scope.sh + + # Fast, early-fail lint lane: PMD + Checkstyle (+ CPD). Turns red in a few + # minutes on any violation, independent of the long build below, so a stray + # PMD/Checkstyle issue is reported immediately rather than after `verify`. + # + # `compile` is in the same invocation as the analysis goals: PMD's + # type-resolving rules (e.g. InvalidLogMessageFormat on the SLF4J + # trailing-Throwable idiom) need Tycho's aux-classpath, which a fresh `mvn` + # does not inherit from a prior step's target/classes. + # + # Preserve Maven failures and validate every expected report before counting findings. + lint: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -12,10 +42,98 @@ jobs: java-version: '21' - name: Set up Workspace Environment Variable run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: PMD Check - run: mvn pmd:pmd pmd:cpd pmd:check pmd:cpd-check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - checkstyle: + - name: Restore Maven dependency cache + # Restore-only, mirroring snapshot.yml's producer cache exactly (path and + # key are hashed into the cache version — see the maven-verify step). + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-publish-${{ hashFiles('**/pom.xml', '**/*.target') }} + restore-keys: ${{ runner.os }}-maven-publish- + + - name: Install XML report validator + run: sudo apt-get update && sudo apt-get install --yes libxml2-utils + + - name: PMD + Checkstyle reports (SARIF) + # PMD: SarifRenderer FQCN — emits pmd.sarif.json AND keeps pmd.xml. + # Checkstyle: output.format=sarif — SARIF content in checkstyle-result.xml. + # CPD is excluded here: the global -Dformat flag uses PMD's Renderer + # hierarchy and would ClassCastException CPD's CPDReportRenderer. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:pmd checkstyle:checkstyle \ + -Dformat=net.sourceforge.pmd.renderers.SarifRenderer \ + -Dcheckstyle.output.format=sarif + + - name: CPD report (separate invocation — no SARIF support) + # CPD has no SARIF renderer; emits cpd.xml only. Run standalone so the + # PMD -Dformat flag isn't in scope. + # NOTE: the CPD token threshold is governed by pmd.cpd.min in + # ddk-parent/pom.xml. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:cpd-check + + - name: Merge per-module SARIFs (PMD + Checkstyle) + if: always() + # Merge only expected module reports into one run per analyzer. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + merge_sarif pmd.sarif.json .sarif-merged/pmd.sarif "${LINT_EXPECT_REPORTS:-$source_modules}" PMD + merge_sarif checkstyle-result.xml .sarif-merged/checkstyle.sarif "${LINT_EXPECT_REPORTS:-$source_modules}" Checkstyle + + - name: Gate on PMD / CPD / Checkstyle violations + # Require successful reports from every expected module before counting findings. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + cpd_reports=() + for mod in ${LINT_EXPECT_REPORTS:-$source_modules}; do + validate_sarif "${mod}/target/pmd.sarif.json" PMD + validate_sarif "${mod}/target/checkstyle-result.xml" Checkstyle + cpd_reports+=("${mod}/target/cpd.xml") + done + validate_sarif .sarif-merged/pmd.sarif PMD + validate_sarif .sarif-merged/checkstyle.sarif Checkstyle + if [ ${#cpd_reports[@]} -eq 0 ]; then + echo "::error::No expected CPD reports — the analysis silently failed." + exit 1 + fi + sarif_total=$(jq -s '[.[].runs[].results[]] | length' \ + .sarif-merged/pmd.sarif .sarif-merged/checkstyle.sarif) + cpd_total=0 + for report in "${cpd_reports[@]}"; do + count=$(cpd_count "$report") + cpd_total=$((cpd_total + count)) + done + echo "PMD/Checkstyle SARIF violations: $sarif_total" + echo "CPD duplications: $cpd_total" + if [ "$sarif_total" != "0" ] || [ "$cpd_total" != "0" ]; then + echo "::error::Static analysis found violations (PMD/CPD/Checkstyle)." + exit 1 + fi + + - name: Upload PMD/Checkstyle SARIF to Code Scanning + if: always() + # Annotation-only, never the gate: a fork PR gets a read-only token and + # upload-sarif 403s, which must not red an otherwise-clean lane. + continue-on-error: true + uses: github/codeql-action/upload-sarif@7fd177fa680c9881b53cdab4d346d32574c9f7f4 # v3.35.4 + with: + sarif_file: .sarif-merged + category: lint + + # SpotBugs is the slow critical-path analysis (the experiments' durable + # finding), so it runs in its own parallel lane and never delays `lint`. + spotbugs: runs-on: ubuntu-24.04 + env: + MAVEN_OPTS: -Xmx4g steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 @@ -24,14 +142,69 @@ jobs: java-version: '21' - name: Set up Workspace Environment Variable run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: Checkstyle Check - run: mvn checkstyle:checkstyle checkstyle:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + - name: Restore Maven dependency cache + # Restore-only, mirroring snapshot.yml's producer cache exactly (path and + # key are hashed into the cache version — see the maven-verify step). + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-publish-${{ hashFiles('**/pom.xml', '**/*.target') }} + restore-keys: ${{ runner.os }}-maven-publish- + + - name: SpotBugs report (SARIF) + # sarifOutput=true emits spotbugsSarif.json (also writes spotbugsXml.xml). + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + spotbugs:spotbugs \ + -Dspotbugs.sarifOutput=true + + - name: Merge per-module SpotBugs SARIFs + if: always() + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + merge_sarif spotbugsSarif.json .sarif-merged/spotbugs.sarif "${SPOTBUGS_EXPECT_REPORTS:-$source_modules}" SpotBugs + + - name: Gate on SpotBugs violations + # Require successful reports from every expected module before counting findings. + run: | + set -euo pipefail + source .github/scripts/sarif.sh + source_modules=$(sarif_source_modules) + for mod in ${SPOTBUGS_EXPECT_REPORTS:-$source_modules}; do + validate_sarif "${mod}/target/spotbugsSarif.json" SpotBugs + done + validate_sarif .sarif-merged/spotbugs.sarif SpotBugs + sb_total=$(jq '[.runs[].results[]] | length' .sarif-merged/spotbugs.sarif) + echo "SpotBugs SARIF violations: $sb_total" + if [ "$sb_total" != "0" ]; then + echo "::error::SpotBugs found violations." + exit 1 + fi + + - name: Upload SpotBugs SARIF to Code Scanning + if: always() + # Annotation-only, never the gate: a fork PR gets a read-only token and + # upload-sarif 403s, which must not red an otherwise-clean lane. + continue-on-error: true + uses: github/codeql-action/upload-sarif@7fd177fa680c9881b53cdab4d346d32574c9f7f4 # v3.35.4 + with: + sarif_file: .sarif-merged + category: spotbugs + line-endings: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check LF line endings run: bash .github/scripts/check-line-endings.sh + + # Build + tests only. Static analysis now lives in the `lint` and `spotbugs` + # jobs, so the redundant checkstyle/pmd/spotbugs goals are dropped from here — + # this is the wall-clock long pole and no longer re-runs analysis. + # No `-T 2C`: tests are not known to pass reliably under reactor parallelism. maven-verify: runs-on: ubuntu-24.04 steps: @@ -74,9 +247,7 @@ jobs: # all downloaded artifacts cached. run: rm -rf ~/.m2/repository/.cache/tycho/https/dsldevkit.github.io ~/.m2/repository/.cache/tycho/https/ddk.tools.avaloq.com - name: Build with Maven within a virtual X Server Environment - # Run pmd:pmd and pmd:cpd first to generate reports for all modules, then run pmd:check and pmd:cpd-check - # This ensures all violations are collected and reported before the build fails - run: xvfb-run mvn clean verify checkstyle:check pmd:pmd pmd:cpd pmd:check pmd:cpd-check spotbugs:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + run: xvfb-run mvn clean verify -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - name: Fail on missing surefire reports if: always() run: bash .github/scripts/check-surefire-reports.sh diff --git a/docs/ci-measurement-protocol.md b/docs/ci-measurement-protocol.md new file mode 100644 index 0000000000..470392f488 --- /dev/null +++ b/docs/ci-measurement-protocol.md @@ -0,0 +1,65 @@ +# CI timing-measurement protocol + +How to get **trustworthy** numbers for a CI-shape change. Written because the +earlier experiment program (2026-05) produced numbers that can't be trusted: +single samples, taken during an Eclipse p2 mirror-flaky window, against a CI +shape that has since changed. Don't repeat that. + +## What we're measuring + +- **Wall-clock** per run = the slowest job (what a developer waits for). +- **Per-job duration** = the analysis bottleneck (settles e.g. spotbugs-vs-maven-verify). +- **Failure latency** = time-to-red on a *planted* lint violation — the metric the + early-fail goal actually cares about. Measure it separately. + +Wall-clock is the headline, but per-job + failure-latency are what tell you *why*. + +## Noise floor (observed on this repo's `verify` runs) + +Per-job spread across recent runs — any change must beat this to be real signal: + +| Job | median | spread (±) | +|---|---|---| +| line-endings | ~5s | ±3s | +| checkstyle (old shape) | ~115s | ±~104s (≈90%!) | +| pmd (old shape) | ~191s | ±63s | +| maven-verify | ~869s (~14.5m) | ±~93s (≈11%) | + +**Decision rule:** accept candidate B as faster than A only if +`median(B) < median(A) − 2 × IQR(A)`. For maven-verify a real win must exceed ~100s; +for checkstyle, ~200s. Anything smaller is noise. + +## Protocol + +1. **Pre-flight — confirm mirrors are healthy.** Run one throwaway build; if + target-platform resolution stalls or errors, **stop** — the window is bad + (this is what voided the 2026-05-09 numbers). Measure only on a clean window. +2. **Warm the caches.** Run 3–5 discard builds first so `~/.m2` + `.cache/tycho` + are populated; cold-cache runs have a different (network-bound) profile. +3. **Sample.** N = 15–20 `workflow_dispatch` runs per candidate, back-to-back in a + ≤30-minute window. Run A and B **interleaved within ≤10 minutes** of each other + so they see the same mirror weather and runner-pool load. +4. **Report medians + IQR**, per job *and* total wall-clock. Never a single sample, + never the mean. +5. **Pin `ubuntu-24.04`** (not a matrix) for consistent runner hardware. +6. **Record per-sample metadata:** cache hit/miss, run start time, headSha. +7. **Failure latency:** plant a synthetic PMD/Checkstyle violation, measure how long + until the `lint` check goes red (independent of the build job). + +## Why not just trust the old experiment numbers + +- Single sample each (no repetition). +- 2026-05-09 mirror-flaky window → resolution time is contaminated and varies per job + even within one dispatch. +- Numbers disagree across rounds (sequential measured 21m one round, 33m another). +- `#1369` reshaped master's CI after the experiments ran, so their baseline is stale. + +## What is *not* a timing lever (validated) + +- **Local p2 mirror**: with a warm `~/.m2`/`.cache/tycho`, offline resolution is + ~equal to online (measured 5s vs 6s) — a mirror's steady-state speedup is ~0. + Its only value is cold-cache + flaky-mirror insurance; deferred per the rare-flake + rule. +- **`-Dtycho.mode=maven` on a gate pass**: marginal on a warm cache (resolution is + already seconds); and it *breaks* PMD type-resolving rules (strips the classpath → + false positives). Not used. diff --git a/docs/ci-static-analysis-design.md b/docs/ci-static-analysis-design.md new file mode 100644 index 0000000000..8e7decc74f --- /dev/null +++ b/docs/ci-static-analysis-design.md @@ -0,0 +1,127 @@ +# Static-analysis CI design: SARIF inline display + count-gate + +This documents *why* the static-analysis CI is shaped the way it is, so the +mechanics (verified against the plugin source) don't have to be re-discovered. + +## Goal + +Fast, complete, **early** PMD + Checkstyle feedback — a violation should fail CI +in minutes, on its own check, not after the ~15-minute build. SpotBugs (the slow +analysis) runs in its own parallel lane so it never delays the fast checks. All +of PMD, Checkstyle, and SpotBugs surface **inline** on the PR via SARIF + GitHub +Code Scanning (no custom annotator). + +## Job shape (`verify.yml`) + +Five independent parallel jobs (no `needs:`); wall-clock = the slowest job. + +| Job | Runs | Threads | Gate | +|---|---|---|---| +| `lint` | `compile` + `pmd:pmd` + `checkstyle:checkstyle` (SARIF) + `pmd:cpd-check` | `-T 2C` | validated SARIF result count + parsed CPD duplication count | +| `spotbugs` | `compile` + `spotbugs:spotbugs` (SARIF), `-Xmx4g` | `-T 2C` | jq count of SARIF results | +| `maven-verify` | `clean verify` (build + tests only) | none | Tycho-surefire | +| `analysis-regression` | report, merge, mutation and scope fixtures | none | extracted workflow blocks | +| `line-endings` | `git ls-files` check | n/a | shell | + +`maven-verify` **no longer re-runs** the analysis goals (now owned by `lint` / +`spotbugs`). Analysis jobs use `-T 2C`; the test job does not (tests are not known +reliable under reactor parallelism). + +## Report goals vs. check goals (code-verified at each plugin version tag) + +| Goal | Kind | `@Execute` fork | Runs analysis? | Writes | Fails build? | SARIF-capable? | +|---|---|---|---|---|---|---| +| `pmd:pmd` | report | — | yes | `pmd.xml` (+ `pmd.sarif.json` w/ `-Dformat=…SarifRenderer`) | no | **yes** | +| `pmd:check` | check | `@Execute(goal=pmd)` | yes (forked) | `pmd.xml` | yes (`> maxAllowedViolations`, dflt 0; `failurePriority` dflt 5 = all) | no (wants `pmd.xml`) | +| `pmd:cpd` | report | — | yes | `cpd.xml` | no | no (no CPD SARIF renderer) | +| `pmd:cpd-check` | check | `@Execute(goal=cpd)` | yes (forked) | `cpd.xml` | yes | no | +| `checkstyle:checkstyle` | report | — | yes | `checkstyle-result.xml` (XML, or SARIF w/ `-Dcheckstyle.output.format=sarif`) | no | **yes** | +| `checkstyle:check` | check | **none** (self-contained) | yes (internal, source-based) | `checkstyle-result.xml` (XML) | yes (severity ≥ `violationSeverity`) | no | +| `spotbugs:spotbugs` | report | — | yes (bytecode) | `spotbugsXml.xml` (+ `spotbugsSarif.json` w/ `-Dspotbugs.sarifOutput=true`) | no | **yes** | +| `spotbugs:check` | check | `@Execute(goal=spotbugs)` | yes (forked) | `spotbugsXml.xml` | yes (`bugCount > 0`) | no | + +The split is **observe vs. enforce**: report goals produce output (for the Maven +site / dashboards / SARIF consumers); analysis errors can still fail a report goal. Check goals bind to +`verify` and exist to fail the build. The `@Execute(goal=…)` on the PMD/CPD/SpotBugs +checks forks the report goal first (so `mvn pmd:check` is self-contained) — which +means they **re-run the analysis every time**. `checkstyle:check` is the lone +exception: no `@Execute`, runs Checkstyle internally in one pass. + +## Maven reactor failure flags + +| Flag | Reactor behavior | Multi-module report completeness | Suppresses violation failure? | +|---|---|---|---| +| `--fail-fast` (default) | halt at first failing module | downstream skipped → reports missing | no | +| `--fail-at-end` | build independent modules, fail at end | modules downstream of a failure still cascade-skipped → incomplete | no | +| `--fail-never` | suppresses every Maven failure | attempts remaining modules, but analysis can still fail to produce usable reports | yes | + +## Report goals, preserved process status, and validated reports + +Each SARIF producer runs once with `--fail-at-end`. Its process status remains +part of the job outcome, including a failure after it has written a report. +Independent modules continue; downstream modules may be skipped after a failure. +The merge requires every expected report, so incomplete analysis stays red even +when other modules produced clean reports. This trades report completeness on a +broken build for dependable failure reporting. + +The shared validator applies to SpotBugs, PMD and Checkstyle. It rejects malformed +JSON, unusable run/result structure, unsuccessful invocations and error execution +or configuration notifications. SpotBugs and PMD must include nonempty completion +metadata; Checkstyle's renderer legitimately omits it. Maven's nonzero exit status +therefore remains essential. CPD XML is parsed, checked for processing errors and +counted structurally, with Maven failures preserved too. + + +### Why not the `:check` goals +- They `@Execute`-fork a **second** analysis on top of the `pmd:pmd` we already run + for SARIF — pure waste. +- The forked analysis is only correct with the full `compile` + classpath; run cheaply + (`-Dtycho.mode=maven` / no compile) it loses the classpath and **false-positives** + (e.g. PMD `InvalidLogMessageFormat` flags the SLF4J trailing-`Throwable` idiom because + it can't resolve the last arg as a `Throwable`). +- `pmd:check` is **incompatible with `-Dformat=sarif`**: its fork writes `pmd.sarif.json`, + but the check looks for `pmd.xml` → "unable to find report." + +### count-gate == `:check` fidelity (for this project's config) +Counting **all** SARIF results equals what `:check` would fail on, because: PMD has no +`failurePriority` override (default 5 = all priorities), Checkstyle is globally +`severity=warning` with no info-level results, and SpotBugs uses `threshold=Low` with no +`failThreshold`/`maxAllowedViolations` override. Three config-drift caveats would make +the count-gate *stricter* than `:check` (over-fail, never under-fail), each guard-worthy: +1. PMD `pmd.failurePriority` set below 5. +2. A Checkstyle rule emitting `info`/`note` severity. +3. SpotBugs `failThreshold` or non-zero `maxAllowedViolations`. + +## Two operational rules + +- **`compile` must be full-reactor** (`-f ddk-parent/pom.xml`, no `-pl`). PMD's + type-resolving rules need the complete aux-classpath; a `-pl` subset produces + false positives (the trailing-`Throwable` case). +- **Merge SARIFs from SARIF files only.** Code Scanning accepts one run per category, + so per-module SARIFs are merged (jq) before upload. The `ddk-parent` aggregator emits + plain-XML `checkstyle-result.xml`; it is excluded by the expected source-module list. + Every expected module report must validate; invalid reports are never filtered out. + +## SARIF mechanics + +- PMD: `-Dformat=net.sourceforge.pmd.renderers.SarifRenderer` → `pmd.sarif.json`. +- Checkstyle: `-Dcheckstyle.output.format=sarif` → SARIF content in `checkstyle-result.xml`. +- SpotBugs: `-Dspotbugs.sarifOutput=true` → `spotbugsSarif.json`. +- All emit SARIF 2.1.0. Merge per tool → `github/codeql-action/upload-sarif` + (`permissions: security-events: write`). CPD has no SARIF renderer → parsed XML gate + (no inline display for CPD; acceptable, duplications are rare). + +## Merge preservation and regression checks + +The merger resolves each run's URI bases and emits repository-relative source +URIs, including secondary locations. A real fork upload showed that preserving +custom base IDs alone made GitHub store paths outside the actual source tree. It resolves result rule indices before unioning +rule descriptors, retains taxonomy descriptors, and rejects conflicts or run +metadata outside the configured producers' supported profile. The supported +profile and fixture provenance are documented in `.github/tests/analysis/README.md`. + +The fast regression job executes the actual workflow merge/gate blocks, checks +real producer fixtures and malformed reports, exercises real Git scope scenarios, +and proves that deliberate regressions are detected. Successful local fixtures +complement the real analyzer runs and GitHub upload checks; they do not establish +GitHub annotation rendering by themselves.