From 2f3c1c964a44e3cb80f182ee1d38fc341e7aad63 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 3 Sep 2026 18:21:14 +0100 Subject: [PATCH 1/4] Add an internal `job-status` input to the `init` Action Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- init/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/init/action.yml b/init/action.yml index 1b64e8d2a3..7787a0a071 100644 --- a/init/action.yml +++ b/init/action.yml @@ -164,6 +164,13 @@ inputs: [Internal] The ID of the check run, as provided by the Actions runtime environment. Do not set this value manually. default: ${{ job.check_run_id }} required: false + job-status: + description: >- + [Internal] The status of the job, as provided by the Actions runtime environment. This is how the + post step learns whether the job as a whole succeeded, failed, or was cancelled. Do not set this + value manually. + default: ${{ job.status }} + required: false outputs: codeql-path: description: The path of the CodeQL binary used for analysis From 38dd4a088a2d38c5d48640c3506dc86215880d92 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 3 Sep 2026 18:21:46 +0100 Subject: [PATCH 2/4] Don't record an overlay status when the job was cancelled Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 18 +++++- src/init-action-post-helper.test.ts | 95 +++++++++++++++++++++++++++++ src/init-action-post-helper.ts | 34 ++++++++++- src/init-action-post.ts | 7 +++ 4 files changed, 150 insertions(+), 4 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 497e44d9d3..7666a9d2b7 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -162277,8 +162277,8 @@ async function tryUploadSarifIfRunFailed(config, repositoryNwo, features, logger return createFailedUploadFailedSarifResult(e); } } -async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLogs2, codeql, config, repositoryNwo, features, logger) { - await recordOverlayStatus(codeql, config, features, logger); +async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLogs2, codeql, config, repositoryNwo, features, jobStatus, logger) { + await recordOverlayStatus(codeql, config, features, jobStatus, logger); const uploadFailedSarifResult = await tryUploadSarifIfRunFailed( config, repositoryNwo, @@ -162340,10 +162340,20 @@ async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLog } return uploadFailedSarifResult; } -async function recordOverlayStatus(codeql, config, features, logger) { +function didCodeQlReportError() { + const jobStatus = process.env["CODEQL_ACTION_JOB_STATUS" /* JOB_STATUS */]; + return jobStatus === "JOB_STATUS_FAILURE" /* FailureStatus */ || jobStatus === "JOB_STATUS_CONFIGURATION_ERROR" /* ConfigErrorStatus */; +} +async function recordOverlayStatus(codeql, config, features, jobStatus, logger) { if (config.overlayDatabaseMode !== "overlay-base" /* OverlayBase */ || process.env["CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY" /* ANALYZE_DID_COMPLETE_SUCCESSFULLY */] === "true" || !await features.getValue("overlay_analysis_status_save" /* OverlayAnalysisStatusSave */)) { return; } + if (jobStatus?.trim().toLowerCase() === "cancelled" && !didCodeQlReportError()) { + logger.info( + "Not recording an improved incremental analysis failure for this job because the workflow run was cancelled." + ); + return; + } const checkRunIdInput = getOptionalInput("check-run-id"); const checkRunId = checkRunIdInput !== void 0 ? parseInt(checkRunIdInput, 10) : void 0; const overlayStatus = createOverlayStatus( @@ -162445,6 +162455,7 @@ async function run4(startedAt) { let uploadFailedSarifResult; let dependencyCachingUsage; try { + const jobStatus2 = getOptionalInput("job-status"); restoreInputs(); const gitHubVersion = await getGitHubVersion(); checkGitHubVersionInRange(gitHubVersion, logger); @@ -162469,6 +162480,7 @@ async function run4(startedAt) { config, repositoryNwo, features, + jobStatus2, logger ); if (await isAnalyzingDefaultBranch() && config.dependencyCachingEnabled !== "none" /* None */) { diff --git a/src/init-action-post-helper.test.ts b/src/init-action-post-helper.test.ts index f24cc5e4e4..8f2868ef78 100644 --- a/src/init-action-post-helper.test.ts +++ b/src/init-action-post-helper.test.ts @@ -15,6 +15,7 @@ import { getRunnerLogger } from "./logging"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; import * as overlayStatus from "./overlay/status"; import { parseRepositoryNwo } from "./repository"; +import { JobStatus } from "./status-report"; import { createFeatures, createTestConfig, @@ -58,6 +59,7 @@ test.serial("init-post action with debug mode off", async (t) => { createTestConfig({ debugMode: false }), parseRepositoryNwo("github/codeql-action"), createFeatures([]), + "success", getRunnerLogger(true), ); @@ -80,6 +82,7 @@ test.serial("init-post action with debug mode on", async (t) => { createTestConfig({ debugMode: true }), parseRepositoryNwo("github/codeql-action"), createFeatures([]), + "success", getRunnerLogger(true), ); @@ -375,6 +378,7 @@ test.serial( }), parseRepositoryNwo("github/codeql-action"), createFeatures([Feature.OverlayAnalysisStatusSave]), + "success", getRunnerLogger(true), ); @@ -443,6 +447,7 @@ test.serial( }), parseRepositoryNwo("github/codeql-action"), createFeatures([]), + "success", getRunnerLogger(true), ); @@ -480,6 +485,7 @@ test.serial("does not save overlay status when build successful", async (t) => { }), parseRepositoryNwo("github/codeql-action"), createFeatures([Feature.OverlayAnalysisStatusSave]), + "success", getRunnerLogger(true), ); @@ -517,6 +523,7 @@ test.serial( }), parseRepositoryNwo("github/codeql-action"), createFeatures([]), + "success", getRunnerLogger(true), ); @@ -528,6 +535,94 @@ test.serial( }, ); +/** + * Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, for a job + * that the Actions runtime environment reports as cancelled. + */ +async function testCancelledOverlayJob({ + jobStatus = "cancelled", + codeQlReportedError = false, +}: { + jobStatus?: string; + codeQlReportedError?: boolean; +} = {}) { + return await util.withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY]; + if (codeQlReportedError) { + process.env[EnvVar.JOB_STATUS] = JobStatus.FailureStatus; + } else { + delete process.env[EnvVar.JOB_STATUS]; + } + + sinon.stub(util, "checkDiskUsage").resolves({ + numAvailableBytes: 100 * NUM_BYTES_PER_GIB, + numTotalBytes: 200 * NUM_BYTES_PER_GIB, + }); + + const saveOverlayStatusStub = sinon + .stub(overlayStatus, "saveOverlayStatus") + .resolves(true); + + await initActionPostHelper.uploadFailureInfo( + sinon.spy(), + sinon.spy(), + codeql.createStubCodeQL({}), + createTestConfig({ + debugMode: false, + languages: ["javascript"], + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + }), + parseRepositoryNwo("github/codeql-action"), + createFeatures([Feature.OverlayAnalysisStatusSave]), + jobStatus, + getRunnerLogger(true), + ); + + return { saveOverlayStatusStub }; + }); +} + +test.serial( + "does not save overlay status when the job was cancelled", + async (t) => { + const { saveOverlayStatusStub } = await testCancelledOverlayJob(); + + t.true( + saveOverlayStatusStub.notCalled, + "a cancellation tells us nothing about whether the analysis would have succeeded", + ); + }, +); + +test.serial( + "saves overlay status when the job failed rather than being cancelled", + async (t) => { + const { saveOverlayStatusStub } = await testCancelledOverlayJob({ + jobStatus: "failure", + }); + + t.true( + saveOverlayStatusStub.calledOnce, + "only cancellations are treated as unrelated to the analysis", + ); + }, +); + +test.serial( + "saves overlay status when a CodeQL Action reported an error before the run was cancelled", + async (t) => { + const { saveOverlayStatusStub } = await testCancelledOverlayJob({ + codeQlReportedError: true, + }); + + t.true( + saveOverlayStatusStub.calledOnce, + "the analysis genuinely failed, even though the run was later cancelled", + ); + }, +); + function createTestWorkflow( steps: workflow.WorkflowJobStep[], ): workflow.Workflow { diff --git a/src/init-action-post-helper.ts b/src/init-action-post-helper.ts index 7b7b056a1c..72c9a62367 100644 --- a/src/init-action-post-helper.ts +++ b/src/init-action-post-helper.ts @@ -316,6 +316,7 @@ export async function tryUploadSarifIfRunFailed( * @param config The CodeQL Action configuration. * @param repositoryNwo The name and owner of the repository. * @param features Information about enabled features. + * @param jobStatus The status of the job, as reported by the Actions runtime environment. * @param logger The logger to use. * @returns The results of uploading the SARIF file for the failure. */ @@ -331,9 +332,10 @@ export async function uploadFailureInfo( config: Config, repositoryNwo: RepositoryNwo, features: FeatureEnablement, + jobStatus: string | undefined, logger: Logger, ): Promise { - await recordOverlayStatus(codeql, config, features, logger); + await recordOverlayStatus(codeql, config, features, jobStatus, logger); const uploadFailedSarifResult = await tryUploadSarifIfRunFailed( config, @@ -412,6 +414,21 @@ export async function uploadFailureInfo( return uploadFailedSarifResult; } +/** + * Whether one of the CodeQL Actions reported an error for this job, which means the analysis + * genuinely failed. + * + * Note that the converse does not hold: an Action that is terminated abruptly, or that fails before + * it can gather telemetry, does not get to report anything. + */ +function didCodeQlReportError(): boolean { + const jobStatus = process.env[EnvVar.JOB_STATUS]; + return ( + jobStatus === JobStatus.FailureStatus || + jobStatus === JobStatus.ConfigErrorStatus + ); +} + /** * If overlay base database creation was attempted but the analysis did not complete * successfully, save the failure status to the Actions cache so that subsequent runs @@ -421,6 +438,7 @@ async function recordOverlayStatus( codeql: CodeQL, config: Config, features: FeatureEnablement, + jobStatus: string | undefined, logger: Logger, ) { if ( @@ -431,6 +449,20 @@ async function recordOverlayStatus( return; } + // A cancelled run tells us nothing about whether the analysis would have succeeded, so recording + // a failure would disable overlay analysis needlessly. Note that we still record a failure if one + // of our own Actions reported an error before the run was cancelled. + if ( + jobStatus?.trim().toLowerCase() === "cancelled" && + !didCodeQlReportError() + ) { + logger.info( + "Not recording an improved incremental analysis failure for this job because the workflow " + + "run was cancelled.", + ); + return; + } + const checkRunIdInput = actionsUtil.getOptionalInput("check-run-id"); const checkRunId = checkRunIdInput !== undefined ? parseInt(checkRunIdInput, 10) : undefined; diff --git a/src/init-action-post.ts b/src/init-action-post.ts index 2261b56ea6..6d6b653e62 100644 --- a/src/init-action-post.ts +++ b/src/init-action-post.ts @@ -8,6 +8,7 @@ import * as core from "@actions/core"; import { restoreInputs, + getOptionalInput, getTemporaryDirectory, printDebugLogs, } from "./actions-util"; @@ -55,6 +56,11 @@ async function run(startedAt: Date) { | undefined; let dependencyCachingUsage: DependencyCachingUsageReport | undefined; try { + // Read the job status before restoring inputs, since it is provided by the Actions runtime + // environment for this step and would otherwise be overwritten by the value that the `init` + // Action saw, which is always a success. + const jobStatus = getOptionalInput("job-status"); + // Restore inputs from `init` Action. restoreInputs(); @@ -84,6 +90,7 @@ async function run(startedAt: Date) { config, repositoryNwo, features, + jobStatus, logger, ); From 657964c39f90c08ebf0224f0e2e264eab1d5fc15 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Fri, 4 Sep 2026 17:31:24 +0100 Subject: [PATCH 3/4] Read the job status from a `ReadOnlyEnv` in `recordOverlayStatus` Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 15 ++++++++------- src/init-action-post-helper.test.ts | 25 +++++++++++++++++++------ src/init-action-post-helper.ts | 15 +++++++++------ src/init-action-post.ts | 3 ++- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 7666a9d2b7..67cd84b780 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -162277,8 +162277,8 @@ async function tryUploadSarifIfRunFailed(config, repositoryNwo, features, logger return createFailedUploadFailedSarifResult(e); } } -async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLogs2, codeql, config, repositoryNwo, features, jobStatus, logger) { - await recordOverlayStatus(codeql, config, features, jobStatus, logger); +async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLogs2, codeql, config, repositoryNwo, features, jobStatus, env, logger) { + await recordOverlayStatus(codeql, config, features, jobStatus, env, logger); const uploadFailedSarifResult = await tryUploadSarifIfRunFailed( config, repositoryNwo, @@ -162340,15 +162340,15 @@ async function uploadFailureInfo(uploadAllAvailableDebugArtifacts, printDebugLog } return uploadFailedSarifResult; } -function didCodeQlReportError() { - const jobStatus = process.env["CODEQL_ACTION_JOB_STATUS" /* JOB_STATUS */]; +function didCodeQlReportError(env) { + const jobStatus = env.getOptional("CODEQL_ACTION_JOB_STATUS" /* JOB_STATUS */); return jobStatus === "JOB_STATUS_FAILURE" /* FailureStatus */ || jobStatus === "JOB_STATUS_CONFIGURATION_ERROR" /* ConfigErrorStatus */; } -async function recordOverlayStatus(codeql, config, features, jobStatus, logger) { - if (config.overlayDatabaseMode !== "overlay-base" /* OverlayBase */ || process.env["CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY" /* ANALYZE_DID_COMPLETE_SUCCESSFULLY */] === "true" || !await features.getValue("overlay_analysis_status_save" /* OverlayAnalysisStatusSave */)) { +async function recordOverlayStatus(codeql, config, features, jobStatus, env, logger) { + if (config.overlayDatabaseMode !== "overlay-base" /* OverlayBase */ || env.getOptional("CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY" /* ANALYZE_DID_COMPLETE_SUCCESSFULLY */) === "true" || !await features.getValue("overlay_analysis_status_save" /* OverlayAnalysisStatusSave */)) { return; } - if (jobStatus?.trim().toLowerCase() === "cancelled" && !didCodeQlReportError()) { + if (jobStatus?.trim().toLowerCase() === "cancelled" && !didCodeQlReportError(env)) { logger.info( "Not recording an improved incremental analysis failure for this job because the workflow run was cancelled." ); @@ -162481,6 +162481,7 @@ async function run4(startedAt) { repositoryNwo, features, jobStatus2, + getEnv(), logger ); if (await isAnalyzingDefaultBranch() && config.dependencyCachingEnabled !== "none" /* None */) { diff --git a/src/init-action-post-helper.test.ts b/src/init-action-post-helper.test.ts index 8f2868ef78..95ec27f4b3 100644 --- a/src/init-action-post-helper.test.ts +++ b/src/init-action-post-helper.test.ts @@ -20,6 +20,7 @@ import { createFeatures, createTestConfig, DEFAULT_ACTIONS_VARS, + getTestEnv, makeMacro, makeVersionInfo, RecordingLogger, @@ -60,6 +61,7 @@ test.serial("init-post action with debug mode off", async (t) => { parseRepositoryNwo("github/codeql-action"), createFeatures([]), "success", + getTestEnv(), getRunnerLogger(true), ); @@ -83,6 +85,7 @@ test.serial("init-post action with debug mode on", async (t) => { parseRepositoryNwo("github/codeql-action"), createFeatures([]), "success", + getTestEnv(), getRunnerLogger(true), ); @@ -379,6 +382,7 @@ test.serial( parseRepositoryNwo("github/codeql-action"), createFeatures([Feature.OverlayAnalysisStatusSave]), "success", + getTestEnv(), getRunnerLogger(true), ); @@ -448,6 +452,7 @@ test.serial( parseRepositoryNwo("github/codeql-action"), createFeatures([]), "success", + getTestEnv(), getRunnerLogger(true), ); @@ -462,8 +467,13 @@ test.serial( test.serial("does not save overlay status when build successful", async (t) => { return await util.withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); - // Mark analyze as having completed successfully. + // Mark analyze as having completed successfully. `tryUploadSarifIfRunFailed` reads this from + // the process environment, while `recordOverlayStatus` reads it from the environment it is + // given. process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] = "true"; + const env = getTestEnv({ + [EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY]: "true", + }); sinon.stub(util, "checkDiskUsage").resolves({ numAvailableBytes: 100 * NUM_BYTES_PER_GIB, @@ -486,6 +496,7 @@ test.serial("does not save overlay status when build successful", async (t) => { parseRepositoryNwo("github/codeql-action"), createFeatures([Feature.OverlayAnalysisStatusSave]), "success", + env, getRunnerLogger(true), ); @@ -524,6 +535,7 @@ test.serial( parseRepositoryNwo("github/codeql-action"), createFeatures([]), "success", + getTestEnv(), getRunnerLogger(true), ); @@ -549,11 +561,11 @@ async function testCancelledOverlayJob({ return await util.withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY]; - if (codeQlReportedError) { - process.env[EnvVar.JOB_STATUS] = JobStatus.FailureStatus; - } else { - delete process.env[EnvVar.JOB_STATUS]; - } + const env = getTestEnv( + codeQlReportedError + ? { [EnvVar.JOB_STATUS]: JobStatus.FailureStatus } + : {}, + ); sinon.stub(util, "checkDiskUsage").resolves({ numAvailableBytes: 100 * NUM_BYTES_PER_GIB, @@ -576,6 +588,7 @@ async function testCancelledOverlayJob({ parseRepositoryNwo("github/codeql-action"), createFeatures([Feature.OverlayAnalysisStatusSave]), jobStatus, + env, getRunnerLogger(true), ); diff --git a/src/init-action-post-helper.ts b/src/init-action-post-helper.ts index 72c9a62367..4363fc7558 100644 --- a/src/init-action-post-helper.ts +++ b/src/init-action-post-helper.ts @@ -18,7 +18,7 @@ import { sanitizeArtifactName, } from "./debug-artifacts"; import * as dependencyCaching from "./dependency-caching"; -import { EnvVar } from "./environment"; +import { EnvVar, ReadOnlyEnv } from "./environment"; import { Feature, FeatureEnablement } from "./feature-flags"; import { Logger } from "./logging"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; @@ -317,6 +317,7 @@ export async function tryUploadSarifIfRunFailed( * @param repositoryNwo The name and owner of the repository. * @param features Information about enabled features. * @param jobStatus The status of the job, as reported by the Actions runtime environment. + * @param env The environment to read variables from. * @param logger The logger to use. * @returns The results of uploading the SARIF file for the failure. */ @@ -333,9 +334,10 @@ export async function uploadFailureInfo( repositoryNwo: RepositoryNwo, features: FeatureEnablement, jobStatus: string | undefined, + env: ReadOnlyEnv, logger: Logger, ): Promise { - await recordOverlayStatus(codeql, config, features, jobStatus, logger); + await recordOverlayStatus(codeql, config, features, jobStatus, env, logger); const uploadFailedSarifResult = await tryUploadSarifIfRunFailed( config, @@ -421,8 +423,8 @@ export async function uploadFailureInfo( * Note that the converse does not hold: an Action that is terminated abruptly, or that fails before * it can gather telemetry, does not get to report anything. */ -function didCodeQlReportError(): boolean { - const jobStatus = process.env[EnvVar.JOB_STATUS]; +function didCodeQlReportError(env: ReadOnlyEnv): boolean { + const jobStatus = env.getOptional(EnvVar.JOB_STATUS); return ( jobStatus === JobStatus.FailureStatus || jobStatus === JobStatus.ConfigErrorStatus @@ -439,11 +441,12 @@ async function recordOverlayStatus( config: Config, features: FeatureEnablement, jobStatus: string | undefined, + env: ReadOnlyEnv, logger: Logger, ) { if ( config.overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase || - process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] === "true" || + env.getOptional(EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY) === "true" || !(await features.getValue(Feature.OverlayAnalysisStatusSave)) ) { return; @@ -454,7 +457,7 @@ async function recordOverlayStatus( // of our own Actions reported an error before the run was cancelled. if ( jobStatus?.trim().toLowerCase() === "cancelled" && - !didCodeQlReportError() + !didCodeQlReportError(env) ) { logger.info( "Not recording an improved incremental analysis failure for this job because the workflow " + diff --git a/src/init-action-post.ts b/src/init-action-post.ts index 6d6b653e62..749020ac64 100644 --- a/src/init-action-post.ts +++ b/src/init-action-post.ts @@ -21,7 +21,7 @@ import { DependencyCachingUsageReport, getDependencyCacheUsage, } from "./dependency-caching"; -import { EnvVar } from "./environment"; +import { EnvVar, getEnv } from "./environment"; import { initFeatures } from "./feature-flags"; import * as gitUtils from "./git-utils"; import * as initActionPostHelper from "./init-action-post-helper"; @@ -91,6 +91,7 @@ async function run(startedAt: Date) { repositoryNwo, features, jobStatus, + getEnv(), logger, ); From a48f2d30771d5b2b049495b82e4e6b1eeeac932d Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Fri, 4 Sep 2026 17:33:02 +0100 Subject: [PATCH 4/4] Record an overlay status only for conclusive job statuses Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 13 +++++- src/init-action-post-helper.test.ts | 62 ++++++++++++++++++++++++----- src/init-action-post-helper.ts | 32 ++++++++++----- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 67cd84b780..24cf3af70d 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -162344,13 +162344,22 @@ function didCodeQlReportError(env) { const jobStatus = env.getOptional("CODEQL_ACTION_JOB_STATUS" /* JOB_STATUS */); return jobStatus === "JOB_STATUS_FAILURE" /* FailureStatus */ || jobStatus === "JOB_STATUS_CONFIGURATION_ERROR" /* ConfigErrorStatus */; } +function isConclusiveJobStatus(jobStatus) { + switch (jobStatus?.trim().toLowerCase()) { + case "failure": + case "success": + return true; + default: + return false; + } +} async function recordOverlayStatus(codeql, config, features, jobStatus, env, logger) { if (config.overlayDatabaseMode !== "overlay-base" /* OverlayBase */ || env.getOptional("CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY" /* ANALYZE_DID_COMPLETE_SUCCESSFULLY */) === "true" || !await features.getValue("overlay_analysis_status_save" /* OverlayAnalysisStatusSave */)) { return; } - if (jobStatus?.trim().toLowerCase() === "cancelled" && !didCodeQlReportError(env)) { + if (!isConclusiveJobStatus(jobStatus) && !didCodeQlReportError(env)) { logger.info( - "Not recording an improved incremental analysis failure for this job because the workflow run was cancelled." + `Not recording an improved incremental analysis failure for this job because the job status (${jobStatus ?? "unset"}) does not tell us whether the analysis itself failed.` ); return; } diff --git a/src/init-action-post-helper.test.ts b/src/init-action-post-helper.test.ts index 95ec27f4b3..fd1f489f7a 100644 --- a/src/init-action-post-helper.test.ts +++ b/src/init-action-post-helper.test.ts @@ -548,16 +548,16 @@ test.serial( ); /** - * Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, for a job - * that the Actions runtime environment reports as cancelled. + * Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, with the + * given job status from the Actions runtime environment. */ -async function testCancelledOverlayJob({ - jobStatus = "cancelled", +async function runOverlayPostStep({ + jobStatus, codeQlReportedError = false, }: { - jobStatus?: string; + jobStatus: string | undefined; codeQlReportedError?: boolean; -} = {}) { +}) { return await util.withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY]; @@ -599,7 +599,9 @@ async function testCancelledOverlayJob({ test.serial( "does not save overlay status when the job was cancelled", async (t) => { - const { saveOverlayStatusStub } = await testCancelledOverlayJob(); + const { saveOverlayStatusStub } = await runOverlayPostStep({ + jobStatus: "cancelled", + }); t.true( saveOverlayStatusStub.notCalled, @@ -608,24 +610,64 @@ test.serial( }, ); +test.serial( + "does not save overlay status when the job status is not recognised", + async (t) => { + const { saveOverlayStatusStub } = await runOverlayPostStep({ + jobStatus: "some-new-status", + }); + + t.true( + saveOverlayStatusStub.notCalled, + "a status we do not recognise tells us nothing about whether the analysis would have succeeded", + ); + }, +); + +test.serial( + "does not save overlay status when the job status is unavailable", + async (t) => { + const { saveOverlayStatusStub } = await runOverlayPostStep({ + jobStatus: undefined, + }); + + t.true( + saveOverlayStatusStub.notCalled, + "without a job status we cannot tell whether the analysis would have succeeded", + ); + }, +); + test.serial( "saves overlay status when the job failed rather than being cancelled", async (t) => { - const { saveOverlayStatusStub } = await testCancelledOverlayJob({ + const { saveOverlayStatusStub } = await runOverlayPostStep({ jobStatus: "failure", }); t.true( saveOverlayStatusStub.calledOnce, - "only cancellations are treated as unrelated to the analysis", + "a failed job indicates that the analysis itself failed", ); }, ); +test.serial("saves overlay status when the job succeeded", async (t) => { + const { saveOverlayStatusStub } = await runOverlayPostStep({ + jobStatus: "success", + }); + + t.true( + saveOverlayStatusStub.calledOnce, + "the analysis did not complete successfully even though the job as a whole succeeded", + ); +}); + test.serial( "saves overlay status when a CodeQL Action reported an error before the run was cancelled", async (t) => { - const { saveOverlayStatusStub } = await testCancelledOverlayJob({ + const { saveOverlayStatusStub } = await runOverlayPostStep({ + jobStatus: "cancelled", codeQlReportedError: true, }); diff --git a/src/init-action-post-helper.ts b/src/init-action-post-helper.ts index 4363fc7558..0e6dae13aa 100644 --- a/src/init-action-post-helper.ts +++ b/src/init-action-post-helper.ts @@ -431,6 +431,22 @@ function didCodeQlReportError(env: ReadOnlyEnv): boolean { ); } +/** + * Whether the job status tells us anything about whether the analysis itself would have succeeded. + * + * We check for the statuses we know to be meaningful rather than excluding the ones that are not, + * so that a status we do not recognise is treated as inconclusive. + */ +function isConclusiveJobStatus(jobStatus: string | undefined): boolean { + switch (jobStatus?.trim().toLowerCase()) { + case "failure": + case "success": + return true; + default: + return false; + } +} + /** * If overlay base database creation was attempted but the analysis did not complete * successfully, save the failure status to the Actions cache so that subsequent runs @@ -452,16 +468,14 @@ async function recordOverlayStatus( return; } - // A cancelled run tells us nothing about whether the analysis would have succeeded, so recording - // a failure would disable overlay analysis needlessly. Note that we still record a failure if one - // of our own Actions reported an error before the run was cancelled. - if ( - jobStatus?.trim().toLowerCase() === "cancelled" && - !didCodeQlReportError(env) - ) { + // Only record a failure when the job outcome tells us something about the analysis. A cancelled + // job, or a status we do not recognise, says nothing about whether the analysis would have + // succeeded, so recording a failure would disable overlay analysis needlessly. We still record + // one if a CodeQL Action reported an error before the job ended. + if (!isConclusiveJobStatus(jobStatus) && !didCodeQlReportError(env)) { logger.info( - "Not recording an improved incremental analysis failure for this job because the workflow " + - "run was cancelled.", + "Not recording an improved incremental analysis failure for this job because the job " + + `status (${jobStatus ?? "unset"}) does not tell us whether the analysis itself failed.`, ); return; }