From 032d4cdd96cc0bbf0dd02ae994583f25d64a961e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 03:03:06 +0000 Subject: [PATCH] chore: sync actions from gh-aw@v0.87.10 --- setup/js/add_reaction_and_edit_comment.cjs | 57 ++-- setup/js/check_cooldown.cjs | 141 +++++++++ setup/js/check_stop_time.cjs | 77 ++++- setup/js/codex_harness.cjs | 89 ++++++ setup/js/copilot_sdk_driver.cjs | 26 +- setup/js/copilot_sdk_session.cjs | 30 +- setup/js/copilot_sdk_tool_config.cjs | 238 ++++++++++++++ setup/js/copilot_sdk_web_fetch.cjs | 297 ++++++++++++++++++ .../awf-v0.28.7-aic-token-usage.jsonl | 5 + setup/js/model_costs.cjs | 11 +- setup/js/mount_mcp_as_cli.cjs | 48 +-- setup/js/parse_mcp_gateway_log.cjs | 247 ++++++++++++--- setup/js/parse_token_usage.cjs | 33 +- setup/js/pi_agent_core_driver.cjs | 13 +- setup/js/pi_models_json.cjs | 40 ++- setup/js/pi_provider.cjs | 7 +- setup/js/send_otlp_span.cjs | 110 ++++++- setup/js/start_mcp_gateway.cjs | 22 +- .../copilot_sdk_web_fetch_contract.json | 18 ++ setup/js/trace_graders.cjs | 3 +- setup/sh/check_mcp_servers.sh | 8 +- setup/sh/start_mcp_gateway.sh | 24 +- setup/sh/stop_mcp_gateway.sh | 10 +- 23 files changed, 1384 insertions(+), 170 deletions(-) create mode 100644 setup/js/check_cooldown.cjs create mode 100644 setup/js/copilot_sdk_tool_config.cjs create mode 100644 setup/js/copilot_sdk_web_fetch.cjs create mode 100644 setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl create mode 100644 setup/js/testdata/copilot_sdk_web_fetch_contract.json diff --git a/setup/js/add_reaction_and_edit_comment.cjs b/setup/js/add_reaction_and_edit_comment.cjs index 7ab23af7..f7840aa2 100644 --- a/setup/js/add_reaction_and_edit_comment.cjs +++ b/setup/js/add_reaction_and_edit_comment.cjs @@ -18,6 +18,21 @@ const VALID_REACTIONS = Object.freeze(Object.keys(REACTION_MAP)); * @typedef {{ route: string, params: Record }} RestEndpoint */ +/** + * Validate a required field extracted from an event payload, calling setFailed if missing. + * @param {unknown} value - The extracted value + * @param {string} fieldName - Human-readable field name for the error message + * @param {string} errorCode - Error code prefix (ERR_NOT_FOUND or ERR_VALIDATION) + * @returns {boolean} true if valid, false if missing (setFailed already called) + */ +function requireEventField(value, fieldName, errorCode) { + if (value == null) { + core.setFailed(`${errorCode}: ${fieldName} not found in event payload`); + return false; + } + return true; +} + /** * @param {unknown} endpoint * @param {string} endpointName @@ -59,10 +74,7 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { switch (eventName) { case "issues": { const issueNumber = payload?.issue?.number; - if (!issueNumber) { - core.setFailed(`${ERR_NOT_FOUND}: Issue number not found in event payload`); - return null; - } + if (!requireEventField(issueNumber, "Issue number", ERR_NOT_FOUND)) return null; return { reactionEndpoint: { route: "POST /repos/{owner}/{repo}/issues/{issue_number}/reactions", params: { owner, repo, issue_number: issueNumber } }, commentUpdateEndpoint: { route: "POST /repos/{owner}/{repo}/issues/{issue_number}/comments", params: { owner, repo, issue_number: issueNumber } }, @@ -72,14 +84,8 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { case "issue_comment": { const commentId = payload?.comment?.id; const issueNumber = payload?.issue?.number; - if (!commentId) { - core.setFailed(`${ERR_VALIDATION}: Comment ID not found in event payload`); - return null; - } - if (!issueNumber) { - core.setFailed(`${ERR_NOT_FOUND}: Issue number not found in event payload`); - return null; - } + if (!requireEventField(commentId, "Comment ID", ERR_VALIDATION)) return null; + if (!requireEventField(issueNumber, "Issue number", ERR_NOT_FOUND)) return null; return { reactionEndpoint: { route: "POST /repos/{owner}/{repo}/issues/comments/{comment_id}/reactions", params: { owner, repo, comment_id: commentId } }, // Create new comment on the issue itself, not on the comment @@ -89,10 +95,7 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { case "pull_request": { const prNumber = payload?.pull_request?.number; - if (!prNumber) { - core.setFailed(`${ERR_NOT_FOUND}: Pull request number not found in event payload`); - return null; - } + if (!requireEventField(prNumber, "Pull request number", ERR_NOT_FOUND)) return null; // PRs are "issues" for the reactions endpoint return { reactionEndpoint: { route: "POST /repos/{owner}/{repo}/issues/{issue_number}/reactions", params: { owner, repo, issue_number: prNumber } }, @@ -103,14 +106,8 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { case "pull_request_review_comment": { const reviewCommentId = payload?.comment?.id; const prNumber = payload?.pull_request?.number; - if (!reviewCommentId) { - core.setFailed(`${ERR_VALIDATION}: Review comment ID not found in event payload`); - return null; - } - if (!prNumber) { - core.setFailed(`${ERR_NOT_FOUND}: Pull request number not found in event payload`); - return null; - } + if (!requireEventField(reviewCommentId, "Review comment ID", ERR_VALIDATION)) return null; + if (!requireEventField(prNumber, "Pull request number", ERR_NOT_FOUND)) return null; return { reactionEndpoint: { route: "POST /repos/{owner}/{repo}/pulls/comments/{comment_id}/reactions", params: { owner, repo, comment_id: reviewCommentId } }, // Create new comment on the PR itself (using issues endpoint since PRs are issues) @@ -120,10 +117,7 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { case "discussion": { const discussionNumber = payload?.discussion?.number; - if (!discussionNumber) { - core.setFailed(`${ERR_NOT_FOUND}: Discussion number not found in event payload`); - return null; - } + if (!requireEventField(discussionNumber, "Discussion number", ERR_NOT_FOUND)) return null; // Discussions use GraphQL API - get the node ID const discussionNodeId = await getDiscussionNodeId(owner, repo, discussionNumber); return { @@ -140,10 +134,7 @@ async function resolveEventEndpoints(eventName, owner, repo, payload) { return null; } const commentNodeId = payload?.comment?.node_id; - if (!commentNodeId) { - core.setFailed(`${ERR_NOT_FOUND}: Discussion comment node ID not found in event payload`); - return null; - } + if (!requireEventField(commentNodeId, "Discussion comment node ID", ERR_NOT_FOUND)) return null; return { reactionEndpoint: commentNodeId, // Store node ID for GraphQL commentUpdateEndpoint: `discussion_comment:${discussionNumber}:${commentId}`, // Special format @@ -305,4 +296,4 @@ async function addCommentWithWorkflowLink(endpoint, runUrl, eventName, invocatio } } -module.exports = { main, addCommentWithWorkflowLink, resolveEventEndpoints, VALID_REACTIONS, addReaction, addDiscussionReaction, expectRestEndpoint, parseDiscussionEndpoint }; +module.exports = { main, addCommentWithWorkflowLink, resolveEventEndpoints, VALID_REACTIONS, addReaction, addDiscussionReaction, expectRestEndpoint, parseDiscussionEndpoint, requireEventField }; diff --git a/setup/js/check_cooldown.cjs b/setup/js/check_cooldown.cjs new file mode 100644 index 00000000..433dd336 --- /dev/null +++ b/setup/js/check_cooldown.cjs @@ -0,0 +1,141 @@ +// @ts-check +/// + +const { getErrorMessage } = require("./error_helpers.cjs"); +const { fetchAndLogRateLimit } = require("./github_rate_limit_logger.cjs"); + +const MAX_WORKFLOW_RUN_PAGES = 5; +const MAX_AGENT_JOB_LOOKUPS = 50; + +async function resolveWorkflowId(githubClient, owner, repo, runId) { + const currentRun = await githubClient.rest.actions.getWorkflowRun({ + owner, + repo, + run_id: runId, + }); + const workflowId = currentRun.data.workflow_id; + if (!workflowId) { + throw new Error(`Cannot resolve workflow id for run ${runId}`); + } + return workflowId; +} + +function parseRunCompletedAt(run) { + // Prefer run_completed_at because it is the terminal timestamp; updated_at is + // only a compatibility fallback for API responses that do not expose it yet. + const completedAtMs = Date.parse(run.run_completed_at ?? run.updated_at ?? ""); + if (Number.isNaN(completedAtMs)) { + return null; + } + return { completedAtMs }; +} + +function agentJobStarted(job) { + return job?.name === "agent" && job.conclusion !== "skipped" && job.started_at; +} + +async function main() { + const { + repo: { owner, repo }, + runId, + } = context; + const cooldownSeconds = Number(process.env.GH_AW_COOLDOWN_SECONDS); + if (!Number.isFinite(cooldownSeconds) || !Number.isSafeInteger(cooldownSeconds) || cooldownSeconds < 300) { + throw new Error("Workflow cooldown must be an integer of at least 300 seconds"); + } + + const threshold = Date.now() - cooldownSeconds * 1000; + + try { + await fetchAndLogRateLimit(github, "check_cooldown_start"); + const workflowId = await resolveWorkflowId(github, owner, repo, runId); + core.info(`Checking ${cooldownSeconds}-second cooldown for workflow '${workflowId}'`); + + let page = 1; + const perPage = 100; + let agentJobLookups = 0; + let hitPageLimit = false; + + while (true) { + if (page > MAX_WORKFLOW_RUN_PAGES) { + hitPageLimit = true; + break; + } + const response = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: workflowId, + status: "completed", + per_page: perPage, + page, + }); + const runs = response.data.workflow_runs || []; + const candidateRuns = []; + + for (const run of runs) { + if (run.id === runId) { + continue; + } + + const parsedCompletion = parseRunCompletedAt(run); + if (!parsedCompletion) { + core.warning(`Skipping run ${run.id} with an invalid completion time`); + continue; + } + if (parsedCompletion.completedAtMs <= threshold) { + continue; + } + candidateRuns.push({ run, ...parsedCompletion }); + } + + // listWorkflowRuns is ordered by creation time, not completion time. Sort + // in-window runs by run_completed_at so the first executed agent run found + // is the newest completion among the inspected history. + candidateRuns.sort((left, right) => right.completedAtMs - left.completedAtMs); + for (const { run, completedAtMs } of candidateRuns) { + if (agentJobLookups >= MAX_AGENT_JOB_LOOKUPS) { + core.warning(`Cooldown check reached the ${MAX_AGENT_JOB_LOOKUPS}-run job lookup budget before confirming no agent execution within the cooldown`); + core.setOutput("cooldown_ok", "true"); + return; + } + agentJobLookups++; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: run.id, + filter: "latest", + per_page: 100, + }); + const agentExecuted = jobs.some(agentJobStarted); + if (!agentExecuted) { + continue; + } + + const remainingSeconds = Math.max(0, Math.ceil((completedAtMs + cooldownSeconds * 1000 - Date.now()) / 1000)); + core.warning(`Skipping agent execution because run ${run.id} completed within the cooldown period (${remainingSeconds} seconds remaining)`); + core.setOutput("cooldown_ok", "false"); + return; + } + + // Do not stop just because every run on this page is older than the + // threshold: listWorkflowRuns is not ordered by run_completed_at, so a + // later-created page can still contain a newer completion. + if (runs.length < perPage) { + break; + } + page++; + } + + if (hitPageLimit) { + core.warning(`Cooldown check stopped after scanning ${MAX_WORKFLOW_RUN_PAGES} workflow run pages`); + } + core.info("Cooldown passed; no recent completed run executed the agent job"); + core.setOutput("cooldown_ok", "true"); + } catch (error) { + core.warning(`Cooldown check failed: ${getErrorMessage(error)}`); + core.warning("Allowing agent execution because workflow run history could not be checked"); + core.setOutput("cooldown_ok", "true"); + } +} + +module.exports = { agentJobStarted, main, parseRunCompletedAt, resolveWorkflowId }; diff --git a/setup/js/check_stop_time.cjs b/setup/js/check_stop_time.cjs index 2ea0e184..d066c086 100644 --- a/setup/js/check_stop_time.cjs +++ b/setup/js/check_stop_time.cjs @@ -3,6 +3,66 @@ const { ERR_CONFIG, ERR_VALIDATION } = require("./error_codes.cjs"); const { writeDenialSummary } = require("./pre_activation_summary.cjs"); + +// Matches a relative time delta such as "+25h", "+3d", "+1w", "+1mo", "+1d12h". +// Mirrors pkg/workflow/time_delta.go's parseTimeDeltaForStopAfter: minutes are not +// supported since the minimum unit for stop-after is hours. +const TIME_DELTA_PATTERN = /(\d+)(mo|w|d|h)/g; + +/** @param {string} stopTime */ +function isRelativeStopTime(stopTime) { + return stopTime.startsWith("+"); +} + +/** @param {string} deltaStr */ +function parseTimeDeltaForStopAfter(deltaStr) { + const rest = deltaStr.slice(1); + if (!rest) { + throw new Error("empty time delta after '+'"); + } + + const matches = [...rest.matchAll(TIME_DELTA_PATTERN)]; + if (matches.length === 0) { + throw new Error(`invalid time delta format: +${rest}. Expected format like +25h, +3d, +1w, +1mo, +1d12h`); + } + + const consumed = matches.reduce((sum, match) => sum + match[0].length, 0); + if (consumed !== rest.length) { + throw new Error(`invalid time delta format: +${rest}. Extra characters detected`); + } + + const delta = { months: 0, weeks: 0, days: 0, hours: 0 }; + const seenUnits = new Set(); + for (const [, valueStr, unit] of matches) { + if (seenUnits.has(unit)) { + throw new Error(`duplicate unit '${unit}' in time delta: +${rest}`); + } + seenUnits.add(unit); + const value = parseInt(valueStr, 10); + if (unit === "mo") delta.months = value; + else if (unit === "w") delta.weeks = value; + else if (unit === "d") delta.days = value; + else if (unit === "h") delta.hours = value; + } + return delta; +} + +/** + * Resolves a relative stop-time delta (e.g. "+48h") to an absolute Date, relative to baseTime. + * Mirrors pkg/workflow/stop_after.go's resolveStopTime: months and days/weeks are applied + * together in a single calendar computation (so date-normalization overflow, e.g. Jan 31 + 1mo, + * is resolved consistently), then hours are added on top. + * @param {string} deltaStr + * @param {Date} baseTime + */ +function resolveRelativeStopTime(deltaStr, baseTime) { + const delta = parseTimeDeltaForStopAfter(deltaStr); + const totalDays = delta.weeks * 7 + delta.days; + return new Date( + Date.UTC(baseTime.getUTCFullYear(), baseTime.getUTCMonth() + delta.months, baseTime.getUTCDate() + totalDays, baseTime.getUTCHours() + delta.hours, baseTime.getUTCMinutes(), baseTime.getUTCSeconds(), baseTime.getUTCMilliseconds()) + ); +} + async function main() { const stopTime = process.env.GH_AW_STOP_TIME; const workflowName = process.env.GH_AW_WORKFLOW_NAME; @@ -19,8 +79,21 @@ async function main() { core.info(`Checking stop-time limit: ${stopTime}`); - // Parse the stop time (format: "YYYY-MM-DD HH:MM:SS") - const stopTimeDate = new Date(stopTime); + // Resolve the stop time. A GitHub Actions expression (e.g. "${{ inputs.stop-after }}") + // is passed through verbatim at compile time and evaluated by the runner before this + // step runs, so it may still be a relative delta (e.g. "+48h") rather than an already + // resolved absolute timestamp (format: "YYYY-MM-DD HH:MM:SS"). + let stopTimeDate; + if (isRelativeStopTime(stopTime)) { + try { + stopTimeDate = resolveRelativeStopTime(stopTime, new Date()); + } catch (err) { + core.setFailed(`${ERR_VALIDATION}: Invalid stop-time format: ${stopTime}. ${err instanceof Error ? err.message : String(err)}`); + return; + } + } else { + stopTimeDate = new Date(stopTime); + } if (Number.isNaN(stopTimeDate.getTime())) { core.setFailed(`${ERR_VALIDATION}: Invalid stop-time format: ${stopTime}. Expected format: YYYY-MM-DD HH:MM:SS`); diff --git a/setup/js/codex_harness.cjs b/setup/js/codex_harness.cjs index f40d03ac..1e5a0282 100644 --- a/setup/js/codex_harness.cjs +++ b/setup/js/codex_harness.cjs @@ -93,6 +93,61 @@ const SERVER_ERROR_PATTERN = /InternalServerError|ServiceUnavailableError|500 In // an identical rejection: retrying only re-bills the turns that succeeded before the failure point. const INVALID_REQUEST_ERROR_PATTERN = /invalid_request_error/i; +// Codex's `turn.failed` event nests the actual provider error as a JSON string inside +// `error.message` (sometimes doubly-nested, e.g. `error.message` -> `{"error": {...}}`). +// This is a specific, common form of "unsupported model" failure: the configured model does +// not support the `custom` tool type Codex uses for its `apply_patch`/freeform tool schema. +// The provider rejects the whole request before any work happens, surfacing as: +// {"error": {"message": "Invalid value: 'custom'", "type": "invalid_request_error", +// "param": "tools", "code": "unknown_parameter"}} +// This is a model-capability mismatch, not a malformed request, so it warrants a dedicated, +// more actionable message than the generic invalid_request_error handling below. + +/** + * Unwraps up to a few levels of Codex's nested provider error payload to find the + * innermost object that carries string `param`/`code` fields. + * @param {unknown} error + * @returns {{ param?: string, code?: string } | null} + */ +function extractNestedProviderErrorDetails(error) { + const candidates = [error]; + for (let visited = 0; visited < 8 && candidates.length > 0; visited++) { + const current = candidates.shift(); + if (!current || typeof current !== "object") continue; + /** @type {{ param?: unknown, code?: unknown, error?: unknown, message?: unknown, metadata?: unknown }} */ + const candidate = current; + if (typeof candidate.param === "string" && typeof candidate.code === "string") { + return { param: candidate.param, code: candidate.code }; + } + if (candidate.error && typeof candidate.error === "object") candidates.push(candidate.error); + if (typeof candidate.message === "string") { + const parsed = parseJsonOrUndefined(candidate.message); + if (parsed !== undefined) candidates.push(parsed); + } + if (candidate.metadata && typeof candidate.metadata === "object") { + /** @type {{ raw?: unknown }} */ + const metadata = candidate.metadata; + if (typeof metadata.raw === "string") { + const parsed = parseJsonOrUndefined(metadata.raw); + if (parsed !== undefined) candidates.push(parsed); + } + } + } + return null; +} + +/** + * @param {string} value + * @returns {unknown} + */ +function parseJsonOrUndefined(value) { + try { + return JSON.parse(value); + } catch { + return undefined; + } +} + // Post-result watchdog: once the agent writes a terminal safe-output the harness // arms a watchdog timer and kills the Codex process if it is still running after // POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS of inactivity. This prevents the step from @@ -203,6 +258,28 @@ function isInvalidRequestError(output) { }); } +/** + * Determines if Codex emitted a `turn.failed` provider event indicating the configured model + * does not support Codex's required `custom` tool-calling schema (the provider rejects the + * `tools` request parameter with code `unknown_parameter`). This is a model-capability mismatch + * — the model itself is valid but incompatible with Codex — so it is surfaced as a dedicated, + * non-retryable condition with actionable guidance rather than the generic invalid-request message. + * @param {string} output - Collected stdout+stderr from the process + * @returns {boolean} + */ +function isUnsupportedModelToolsError(output) { + return output.split(/\r?\n/).some(line => { + try { + const event = JSON.parse(line); + if (event?.type !== "turn.failed" || !event.error) return false; + const details = extractNestedProviderErrorDetails(event.error); + return !!details && details.param === "tools" && details.code === "unknown_parameter"; + } catch { + return false; + } + }); +} + /** * Determines if the collected output shows that Codex's internal stream-reconnect * retries are exhausted (i.e., the output contains "Reconnecting... N/N" where both @@ -778,6 +855,7 @@ async function main() { const isMissingApiKey = isMissingApiKeyError(result.output); const isServer = isServerError(result.output); const isInvalidModel = isInvalidModelError(result.output); + const isUnsupportedModelTools = isUnsupportedModelToolsError(result.output); const isInvalidRequest = isInvalidRequestError(result.output); const permissionDeniedCount = countPermissionDeniedIssues(result.output); const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output); @@ -792,6 +870,7 @@ async function main() { ` isMissingApiKeyError=${isMissingApiKey}` + ` isServerError=${isServer}` + ` isInvalidModelError=${isInvalidModel}` + + ` isUnsupportedModelToolsError=${isUnsupportedModelTools}` + ` isInvalidRequestError=${isInvalidRequest}` + ` permissionDeniedCount=${permissionDeniedCount}` + ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` + @@ -849,6 +928,15 @@ async function main() { return { action: "stop" }; } + if (isUnsupportedModelTools) { + log( + `attempt ${attempt + 1}: configured model does not support Codex's required tool-calling schema` + + ` ("tools" param rejected with code "unknown_parameter") — not retrying` + + ` (pick a model documented as compatible with Codex CLI, or remove the \`model:\` override in workflow frontmatter to use the engine default)` + ); + return { action: "stop" }; + } + if (isInvalidRequest) { log(`attempt ${attempt + 1}: invalid_request_error (HTTP 400) — not retrying (the provider rejected the request payload; an identical fresh run would fail the same way)`); return { action: "stop" }; @@ -913,6 +1001,7 @@ if (typeof module !== "undefined" && module.exports) { isMissingApiKeyError, isServerError, isInvalidModelError, + isUnsupportedModelToolsError, isInvalidRequestError, isReconnectExhaustedError, countPermissionDeniedIssues, diff --git a/setup/js/copilot_sdk_driver.cjs b/setup/js/copilot_sdk_driver.cjs index 00483011..4edaf57d 100644 --- a/setup/js/copilot_sdk_driver.cjs +++ b/setup/js/copilot_sdk_driver.cjs @@ -29,13 +29,14 @@ const fs = require("fs"); const { runWithCopilotSDK, extractPromptFromArgs } = require("./copilot_sdk_session.cjs"); const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); +const { parseCopilotSDKToolConfig } = require("./copilot_sdk_tool_config.cjs"); const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); const { applyModelFallback } = require("./model_fallback.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); // Re-export the session and permission helpers so that existing callers that // require("./copilot_sdk_driver.cjs") (e.g. copilot_harness.cjs) continue to work. -module.exports = { extractPromptFromArgs, runWithCopilotSDK, parsePermissionConfigFromServerArgs, parseMultiProviderJson }; +module.exports = { extractPromptFromArgs, runWithCopilotSDK, parsePermissionConfigFromServerArgs, parseCopilotSDKToolConfig, parseMultiProviderJson }; // --------------------------------------------------------------------------- // Standalone entry point @@ -111,22 +112,12 @@ async function main() { log(` provider: name=${p.name} type=${p.type} baseUrl=${p.baseUrl}${p.wireApi ? ` wireApi=${p.wireApi}` : ""}`); } - // --- Build permission config from sidecar server args ---------------- - // GH_AW_COPILOT_SDK_SERVER_ARGS holds the JSON-encoded --allow-tool flags - // that the Go engine passed to the sidecar. Mirror those same rules in the - // SDK session so the driver's onPermissionRequest handler aligns with the - // sidecar's pre-configured allow list (e.g. shell(safeoutputs:*) for - // workflows with safe-outputs enabled and a restricted bash allowlist). - const permissionConfig = parsePermissionConfigFromServerArgs(process.env.GH_AW_COPILOT_SDK_SERVER_ARGS); - if (permissionConfig) { - if (permissionConfig.allowAllTools) { - log("permission config: allow-all-tools (sidecar launched with --allow-all-tools)"); - } else { - log(`permission config: ${(permissionConfig.allowedTools ?? []).length} allow-tool entries from GH_AW_COPILOT_SDK_SERVER_ARGS`); - } - } else { - log("permission config: none (onPermissionRequest will use unrestricted behavior)"); - } + // --- Read compiler-owned tool and permission configuration ------------ + // One structured contract controls visibility, registration, and permission + // enforcement. Missing or invalid contracts fail before session creation. + const toolConfig = parseCopilotSDKToolConfig(process.env.GH_AW_COPILOT_SDK_TOOL_CONFIG); + const permissionConfig = toolConfig.permissions; + log(`permission config: ${permissionConfig.allowedTools.length} compiler-owned allow-tool entries`); // --- Run SDK session ------------------------------------------------- @@ -139,6 +130,7 @@ async function main() { providers, models: sdkModels, permissionConfig, + toolConfig, }); process.exit(result.exitCode); diff --git a/setup/js/copilot_sdk_session.cjs b/setup/js/copilot_sdk_session.cjs index 221814da..df75840b 100644 --- a/setup/js/copilot_sdk_session.cjs +++ b/setup/js/copilot_sdk_session.cjs @@ -36,6 +36,7 @@ const fs = require("fs"); const path = require("path"); const os = require("os"); const { buildCopilotSDKPermissionHandler, getEnvPositiveIntOrDefault, parseMaxToolDenialsLimit, MAX_TOOL_DENIALS_DEFAULT } = require("./copilot_sdk_permissions.cjs"); +const { buildCopilotSDKSessionToolConfig } = require("./copilot_sdk_tool_config.cjs"); const { resolveModelWithFallback } = require("./model_fallback.cjs"); const { extractShellCommandFromToolData } = require("./tool_call_details.cjs"); @@ -106,20 +107,42 @@ function extractPromptFromArgs(args) { * allowAllTools?: boolean, * allowedTools?: string[], * }, + * toolConfig?: import("./copilot_sdk_tool_config.cjs").CopilotSDKToolConfig, + * webFetchOptions?: {fetchImpl?: typeof fetch, timeoutMs?: number, maxRedirects?: number}, * coreLogger?: import("./copilot_sdk_permissions.cjs").CopilotSDKCoreLogger, * sdkModule?: { * CopilotClient: typeof import("@github/copilot-sdk").CopilotClient, * RuntimeConnection: typeof import("@github/copilot-sdk").RuntimeConnection, - * approveAll: typeof import("@github/copilot-sdk").approveAll + * approveAll: typeof import("@github/copilot-sdk").approveAll, + * ToolSet?: typeof import("@github/copilot-sdk").ToolSet, + * BuiltInTools?: typeof import("@github/copilot-sdk").BuiltInTools, + * defineTool?: typeof import("@github/copilot-sdk").defineTool, * }, * sessionStateBaseDir?: string, * }} options * @returns {Promise<{exitCode: number, output: string, hasOutput: boolean, durationMs: number}>} */ -async function runWithCopilotSDK({ sdkUri, prompt, logger, attempt = 0, model, connectionToken, providers, models: providerModels, maxToolDenials, permissionConfig, coreLogger, sdkModule, sessionStateBaseDir }) { +async function runWithCopilotSDK({ + sdkUri, + prompt, + logger, + attempt = 0, + model, + connectionToken, + providers, + models: providerModels, + maxToolDenials, + permissionConfig, + toolConfig, + webFetchOptions, + coreLogger, + sdkModule, + sessionStateBaseDir, +}) { // Lazy-require to avoid loading the SDK when it is not needed. // The SDK is large and has side-effects on import (worker threads, etc.). - const { CopilotClient, RuntimeConnection, approveAll } = sdkModule ?? require("@github/copilot-sdk"); + const sdk = sdkModule ?? require("@github/copilot-sdk"); + const { CopilotClient, RuntimeConnection, approveAll } = sdk; const startTime = Date.now(); let output = ""; @@ -260,6 +283,7 @@ async function runWithCopilotSDK({ sdkUri, prompt, logger, attempt = 0, model, c providers, models: providerModels, onPermissionRequest, + ...buildCopilotSDKSessionToolConfig(toolConfig, sdk, webFetchOptions), }; log(`creating session with model="${sessionConfig.model || "(none)"}" providers=${providers?.length ?? 0} models=${providerModels?.length ?? 0}`); session = await client.createSession(sessionConfig); diff --git a/setup/js/copilot_sdk_tool_config.cjs b/setup/js/copilot_sdk_tool_config.cjs new file mode 100644 index 00000000..abdc813a --- /dev/null +++ b/setup/js/copilot_sdk_tool_config.cjs @@ -0,0 +1,238 @@ +// @ts-check + +"use strict"; + +const { createCopilotSDKWebFetchTool } = require("./copilot_sdk_web_fetch.cjs"); + +const COPILOT_SDK_TOOL_CONFIG_VERSION = 1; +const COPILOT_SDK_NEUTRAL_BUILTIN_TOOLS = Object.freeze(["view", "rg", "glob", "sql"]); +const COPILOT_SDK_SHELL_BUILTIN_TOOLS = Object.freeze(["bash", "read_bash", "stop_bash", "list_bash"]); +const COPILOT_SDK_EDIT_BUILTIN_TOOLS = Object.freeze(["apply_patch", "edit", "create", "delete", "move", "write_bash"]); + +/** + * @typedef {{ + * bash: boolean, + * edit: boolean, + * webFetch: boolean, + * webSearch: boolean, + * mcp: boolean, + * cliProxy: boolean, + * }} CopilotSDKToolCapabilities + */ + +/** + * @typedef {{ + * version: number, + * capabilities: CopilotSDKToolCapabilities, + * permissions: {allowedTools: string[]}, + * explicitlyDisabledTools: string[], + * }} CopilotSDKToolConfig + */ + +/** + * @param {unknown} value + * @returns {value is Record} + */ +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * @param {unknown} value + * @param {string} field + * @returns {string[]} + */ +function parseStringArray(value, field) { + if (!Array.isArray(value) || value.some(entry => typeof entry !== "string" || entry.trim() === "")) { + throw new Error(`${field} must be an array of non-empty strings`); + } + const normalized = value.map(entry => entry.trim()); + if (new Set(normalized).size !== normalized.length) { + throw new Error(`${field} must not contain duplicate entries`); + } + return normalized; +} + +/** + * @param {Record} value + * @returns {CopilotSDKToolCapabilities} + */ +function parseCapabilities(value) { + const fields = ["bash", "edit", "webFetch", "webSearch", "mcp", "cliProxy"]; + for (const field of fields) { + if (typeof value[field] !== "boolean") { + throw new Error(`capabilities.${field} must be a boolean`); + } + } + return /** @type {CopilotSDKToolCapabilities} */ { + bash: Boolean(value.bash), + edit: Boolean(value.edit), + webFetch: Boolean(value.webFetch), + webSearch: Boolean(value.webSearch), + mcp: Boolean(value.mcp), + cliProxy: Boolean(value.cliProxy), + }; +} + +/** + * Return true for SDK permission entries that are not MCP server grants. + * web_fetch and web_search are intentionally only supported as unscoped + * built-in/custom tool permissions; scoped entries like web_fetch(...) are not + * part of the compiler contract and are deliberately treated as MCP grants so + * validateToolPermissionParity's webFetch check catches them with a clear + * "visibility and permissions differ" error instead of silently allowing them. + * + * @param {string} tool + * @returns {boolean} + */ +function isReservedSDKPermission(tool) { + return tool === "read" || tool === "write" || tool === "web_fetch" || tool === "web_search" || tool === "shell" || (tool.startsWith("read(") && tool.endsWith(")")) || (tool.startsWith("shell(") && tool.endsWith(")")); +} + +/** + * @param {CopilotSDKToolConfig} config + */ +function validateToolPermissionParity(config) { + const allowed = new Set(config.permissions.allowedTools); + const hasShellPermission = config.permissions.allowedTools.some(tool => tool === "shell" || (tool.startsWith("shell(") && tool.endsWith(")"))); + const hasMCPPermission = config.permissions.allowedTools.some(tool => !isReservedSDKPermission(tool)); + + if (config.capabilities.bash !== hasShellPermission) { + throw new Error("SDK tool contract mismatch: bash visibility and shell permissions differ"); + } + if (config.capabilities.edit !== allowed.has("write")) { + throw new Error("SDK tool contract mismatch: edit visibility and write permissions differ"); + } + if (config.capabilities.webFetch !== allowed.has("web_fetch")) { + throw new Error("SDK tool contract mismatch: web_fetch visibility and permissions differ"); + } + if (config.capabilities.webSearch !== allowed.has("web_search")) { + throw new Error("SDK tool contract mismatch: web_search visibility and permissions differ"); + } + if (!config.capabilities.mcp && hasMCPPermission) { + throw new Error("SDK tool contract mismatch: MCP permissions exist while MCP visibility is disabled"); + } + // cli-proxy mounts MCP servers as CLI wrapper scripts on PATH; those scripts are + // only reachable through the bash builtin, so a cliProxy capability without a + // bash capability is an unreachable, misleading contract. + if (config.capabilities.cliProxy && !config.capabilities.bash) { + throw new Error("SDK tool contract mismatch: cliProxy capability requires bash capability"); + } + const disabledCapability = new Map([ + ["bash", "bash"], + ["edit", "edit"], + ["web-fetch", "webFetch"], + ["web-search", "webSearch"], + ["cli-proxy", "cliProxy"], + ]); + for (const toolName of config.explicitlyDisabledTools) { + const capability = disabledCapability.get(toolName); + if (capability && config.capabilities[capability]) { + throw new Error(`SDK tool contract mismatch: explicitly disabled ${toolName} is visible`); + } + } +} + +/** + * Parse the compiler-owned GH_AW_COPILOT_SDK_TOOL_CONFIG value. Missing, + * malformed, inconsistent, or unsupported values fail closed. + * + * @param {string | undefined} value + * @returns {CopilotSDKToolConfig} + */ +function parseCopilotSDKToolConfig(value) { + if (!value) { + throw new Error("GH_AW_COPILOT_SDK_TOOL_CONFIG is required"); + } + let parsed; + try { + parsed = JSON.parse(value); + } catch (error) { + throw new Error(`GH_AW_COPILOT_SDK_TOOL_CONFIG must be valid JSON: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } + if (!isRecord(parsed)) { + throw new Error("GH_AW_COPILOT_SDK_TOOL_CONFIG must be a JSON object"); + } + if (parsed.version !== COPILOT_SDK_TOOL_CONFIG_VERSION) { + throw new Error(`unsupported GH_AW_COPILOT_SDK_TOOL_CONFIG version: ${String(parsed.version)}`); + } + if (!isRecord(parsed.capabilities)) { + throw new Error("GH_AW_COPILOT_SDK_TOOL_CONFIG capabilities must be an object"); + } + if (!isRecord(parsed.permissions)) { + throw new Error("GH_AW_COPILOT_SDK_TOOL_CONFIG permissions must be an object"); + } + + const allowedTools = parseStringArray(parsed.permissions.allowedTools, "permissions.allowedTools"); + if (allowedTools.length === 0) { + throw new Error("permissions.allowedTools must not be empty"); + } + const config = { + version: COPILOT_SDK_TOOL_CONFIG_VERSION, + capabilities: parseCapabilities(parsed.capabilities), + permissions: { + allowedTools, + }, + explicitlyDisabledTools: parsed.explicitlyDisabledTools == null ? [] : parseStringArray(parsed.explicitlyDisabledTools, "explicitlyDisabledTools"), + }; + validateToolPermissionParity(config); + return config; +} + +/** + * @param {CopilotSDKToolConfig | null | undefined} config + * @param {{ + * ToolSet?: typeof import("@github/copilot-sdk").ToolSet, + * BuiltInTools?: typeof import("@github/copilot-sdk").BuiltInTools, + * defineTool?: typeof import("@github/copilot-sdk").defineTool, + * }} sdk + * @param {{fetchImpl?: typeof fetch, timeoutMs?: number, maxRedirects?: number}} [options] + * @returns {Pick} + */ +function buildCopilotSDKSessionToolConfig(config, sdk, options = {}) { + if (!config) return {}; + if (typeof sdk.ToolSet !== "function" || !sdk.BuiltInTools || !Array.isArray(sdk.BuiltInTools.Isolated)) { + throw new Error("Copilot SDK ToolSet and BuiltInTools.Isolated are required for compiler-controlled tool filtering"); + } + + const availableTools = new sdk.ToolSet(); + availableTools.addBuiltIn(sdk.BuiltInTools.Isolated.filter(name => name !== "ask_user")); + availableTools.addBuiltIn(COPILOT_SDK_NEUTRAL_BUILTIN_TOOLS); + if (config.capabilities.bash) availableTools.addBuiltIn(COPILOT_SDK_SHELL_BUILTIN_TOOLS); + if (config.capabilities.edit) availableTools.addBuiltIn(COPILOT_SDK_EDIT_BUILTIN_TOOLS); + // The compiler currently always emits webSearch: false (the Copilot SDK runtime + // cannot authorize/execute web-search); this branch is kept ready for when a + // real implementation and permission are wired, guarded by the parity check + // above so a stray webSearch: true without a matching permission fails closed. + if (config.capabilities.webSearch) availableTools.addBuiltIn("web_search"); + if (config.capabilities.mcp) availableTools.addMcp("*"); + // cliProxy mounts MCP servers as CLI wrapper scripts on PATH; those scripts are + // invoked through the bash builtin (already added above when capabilities.bash + // is true), so cliProxy needs no separate SDK tool/availableTools entry here. + // validateToolPermissionParity enforces that cliProxy can only be true when + // bash is also true, so this capability is never silently unreachable. + + /** @type {import("@github/copilot-sdk").Tool[]} */ + const tools = []; + if (config.capabilities.webFetch) { + if (typeof sdk.defineTool !== "function") { + throw new Error("Copilot SDK defineTool is required when tools.web-fetch is enabled"); + } + tools.push(createCopilotSDKWebFetchTool(sdk.defineTool, options)); + availableTools.addCustom("web_fetch"); + } + return { availableTools, tools }; +} + +module.exports = { + COPILOT_SDK_TOOL_CONFIG_VERSION, + COPILOT_SDK_NEUTRAL_BUILTIN_TOOLS, + COPILOT_SDK_SHELL_BUILTIN_TOOLS, + COPILOT_SDK_EDIT_BUILTIN_TOOLS, + parseStringArray, + parseCapabilities, + isReservedSDKPermission, + validateToolPermissionParity, + parseCopilotSDKToolConfig, + buildCopilotSDKSessionToolConfig, +}; diff --git a/setup/js/copilot_sdk_web_fetch.cjs b/setup/js/copilot_sdk_web_fetch.cjs new file mode 100644 index 00000000..f8abf1e0 --- /dev/null +++ b/setup/js/copilot_sdk_web_fetch.cjs @@ -0,0 +1,297 @@ +// @ts-check + +"use strict"; + +const { fetch: undiciFetch, ProxyAgent } = require("undici"); + +const DEFAULT_MAX_LENGTH = 5_000; +const MAX_MAX_LENGTH = 20_000; +const DEFAULT_TIMEOUT_MS = 30_000; +const MAX_REDIRECTS = 5; +const MAX_RESPONSE_BYTES = 10 * 1024 * 1024; +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); + +/** + * @param {unknown} value + * @param {string} name + * @param {number} defaultValue + * @param {number} minimum + * @param {number} maximum + * @returns {number} + */ +function boundedInteger(value, name, defaultValue, minimum, maximum) { + if (value === undefined) return defaultValue; + if (!Number.isSafeInteger(value) || Number(value) < minimum || Number(value) > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + } + return Number(value); +} + +/** + * Validate URL syntax without making destination-policy decisions. AWF owns + * allow/deny policy for public, private, loopback, and enterprise destinations. + * + * @param {unknown} value + * @param {URL | undefined} [base] + * @returns {URL} + */ +function parseWebFetchURL(value, base) { + if (typeof value !== "string" || value.trim() === "") { + throw new Error("url must be a non-empty string"); + } + let url; + try { + url = base ? new URL(value, base) : new URL(value); + } catch { + throw new Error("url must be an absolute HTTP or HTTPS URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("url must use the http or https protocol"); + } + if (url.username || url.password) { + throw new Error("url must not contain credentials"); + } + return url; +} + +/** + * @param {ReadableStream | null} body + * @returns {Promise} + */ +async function readBoundedResponseBody(body) { + if (!body) return ""; + const reader = body.getReader(); + /** @type {Uint8Array[]} */ + const chunks = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > MAX_RESPONSE_BYTES) { + await reader.cancel(); + throw new Error(`response body exceeds ${MAX_RESPONSE_BYTES} bytes`); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks.map(chunk => Buffer.from(chunk))).toString("utf8"); +} + +/** + * @param {string} value + * @returns {string} + */ +function decodeBasicHTMLEntities(value) { + return value + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/"/gi, '"') + .replace(/&#(?:39|x27);/gi, "'"); +} + +/** + * @param {string} content + * @param {string} contentType + * @param {boolean} raw + * @returns {string} + */ +function simplifyWebContent(content, contentType, raw) { + if (raw || !contentType.toLowerCase().includes("text/html")) { + return content; + } + return decodeBasicHTMLEntities( + content + .replace(/<(script|style|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "") + .replace(//g, "") + .replace(/<\/?(?:p|div|section|article|main|header|footer|nav|aside|h[1-6]|li|ul|ol|table|tr|blockquote|pre)\b[^>]*>/gi, "\n") + .replace(//gi, "\n") + .replace(/<[^>]+>/g, "") + ) + .replace(/[ \t]+\n/g, "\n") + .replace(/\n[ \t]+/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +/** + * Use AWF's explicit proxy environment for every destination, including + * loopback/private names. Do not honor NO_PROXY here: AWF must observe and + * decide each model-requested destination. + * + * @param {{fetchImpl?: typeof fetch, env?: NodeJS.ProcessEnv}} options + */ +function createWebFetchTransport(options) { + if (options.fetchImpl) { + return { fetch: options.fetchImpl, close: async () => {} }; + } + const env = options.env ?? process.env; + /** @type {Map>} */ + const agents = new Map(); + return { + /** + * @param {URL} url + * @param {{method: "GET", redirect: "manual", signal: AbortSignal}} init + */ + fetch: async (url, init) => { + const proxyURL = url.protocol === "https:" ? env.HTTPS_PROXY || env.https_proxy || env.HTTP_PROXY || env.http_proxy : env.HTTP_PROXY || env.http_proxy || env.HTTPS_PROXY || env.https_proxy; + if (!proxyURL) { + // AWF must observe every web_fetch destination through its proxy. Fail closed + // instead of silently issuing an unobserved direct request when the proxy + // environment is missing or misconfigured. + throw new Error("web_fetch requires an AWF proxy (HTTPS_PROXY/HTTP_PROXY); refusing unobserved direct request"); + } + let dispatcher = agents.get(proxyURL); + if (!dispatcher) { + dispatcher = new ProxyAgent(proxyURL); + agents.set(proxyURL, dispatcher); + } + return undiciFetch(url, { ...init, dispatcher }); + }, + close: async () => { + await Promise.all([...agents.values()].map(agent => agent.close())); + }, + }; +} + +/** + * @typedef {{ + * url: string, + * raw?: boolean, + * max_length?: number, + * start_index?: number, + * }} CopilotSDKWebFetchInput + */ + +/** + * @param {CopilotSDKWebFetchInput} input + * @param {{ + * fetchImpl?: typeof fetch, + * env?: NodeJS.ProcessEnv, + * timeoutMs?: number, + * maxRedirects?: number, + * }} [options] + * @returns {Promise} + */ +async function executeCopilotSDKWebFetch(input, options = {}) { + if (!input || typeof input !== "object") { + throw new Error("web_fetch input must be an object"); + } + if (options.fetchImpl !== undefined && typeof options.fetchImpl !== "function") { + throw new Error("web_fetch requires a Fetch API implementation"); + } + const transport = createWebFetchTransport(options); + + const maxLength = boundedInteger(input.max_length, "max_length", DEFAULT_MAX_LENGTH, 1, MAX_MAX_LENGTH); + const startIndex = boundedInteger(input.start_index, "start_index", 0, 0, Number.MAX_SAFE_INTEGER); + const timeoutMs = boundedInteger(options.timeoutMs, "timeoutMs", DEFAULT_TIMEOUT_MS, 1, 5 * 60_000); + const maxRedirects = boundedInteger(options.maxRedirects, "maxRedirects", MAX_REDIRECTS, 0, 20); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + timeout.unref?.(); + + let currentURL = parseWebFetchURL(input.url); + let response; + try { + for (let redirectCount = 0; ; redirectCount++) { + response = await transport.fetch(currentURL, { + method: "GET", + redirect: "manual", + signal: controller.signal, + }); + if (!REDIRECT_STATUSES.has(response.status)) break; + const location = response.headers.get("location"); + if (!location) break; + if (redirectCount >= maxRedirects) { + throw new Error(`web_fetch exceeded ${maxRedirects} redirects`); + } + await response.body?.cancel(); + currentURL = parseWebFetchURL(location, currentURL); + } + + const contentType = response.headers.get("content-type") ?? ""; + const body = simplifyWebContent(await readBoundedResponseBody(response.body), contentType, input.raw === true); + const content = body.slice(startIndex, startIndex + maxLength); + return JSON.stringify({ + url: currentURL.toString(), + status: response.status, + content_type: contentType, + start_index: startIndex, + end_index: startIndex + content.length, + total_length: body.length, + truncated: startIndex + content.length < body.length, + content, + }); + } finally { + clearTimeout(timeout); + await transport.close(); + } +} + +/** + * Adapter for the SDK Tool handler boundary. + * + * @param {any} input + * @param {{fetchImpl?: typeof fetch, env?: NodeJS.ProcessEnv, timeoutMs?: number, maxRedirects?: number}} options + * @returns {Promise} + */ +function executeCopilotSDKWebFetchToolHandler(input, options) { + return executeCopilotSDKWebFetch(input, options); +} + +/** + * Create the compiler-controlled web_fetch tool. @github/copilot-sdk 1.0.11 + * explicitly supports replacing a same-name built-in through + * overridesBuiltInTool. Assert that defineTool preserves the override contract + * so future incompatible SDK changes fail during session initialization. + * + * @param {typeof import("@github/copilot-sdk").defineTool} defineTool + * @param {{fetchImpl?: typeof fetch, env?: NodeJS.ProcessEnv, timeoutMs?: number, maxRedirects?: number}} [options] + * @returns {import("@github/copilot-sdk").Tool} + */ +function createCopilotSDKWebFetchTool(defineTool, options = {}) { + if (typeof defineTool !== "function") { + throw new Error("Copilot SDK defineTool is required to register web_fetch"); + } + const tool = defineTool("web_fetch", { + description: "Fetch an HTTP or HTTPS URL from inside the AWF-protected agent boundary.", + parameters: { + type: "object", + additionalProperties: false, + required: ["url"], + properties: { + url: { type: "string", description: "Absolute HTTP or HTTPS URL to fetch." }, + raw: { type: "boolean", description: "Return raw HTML instead of simplified text." }, + max_length: { type: "integer", minimum: 1, maximum: MAX_MAX_LENGTH, description: "Maximum response characters to return." }, + start_index: { type: "integer", minimum: 0, description: "Character offset for pagination." }, + }, + }, + overridesBuiltInTool: true, + defer: "never", + handler: input => executeCopilotSDKWebFetchToolHandler(input, options), + }); + if (!tool || tool.name !== "web_fetch" || tool.overridesBuiltInTool !== true) { + throw new Error("Copilot SDK defineTool did not preserve the required web_fetch override contract"); + } + return tool; +} + +module.exports = { + DEFAULT_MAX_LENGTH, + MAX_MAX_LENGTH, + DEFAULT_TIMEOUT_MS, + MAX_REDIRECTS, + MAX_RESPONSE_BYTES, + boundedInteger, + parseWebFetchURL, + readBoundedResponseBody, + simplifyWebContent, + createWebFetchTransport, + executeCopilotSDKWebFetch, + createCopilotSDKWebFetchTool, +}; diff --git a/setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl b/setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl new file mode 100644 index 00000000..6c9b2fdc --- /dev/null +++ b/setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl @@ -0,0 +1,5 @@ +{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:54.002Z","event":"token_usage","request_id":"5c6141a6-7331-400f-94c8-d38defdcbc7e","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19288,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2242,"response_bytes":8642,"x_initiator":"user","ai_credits_this_response":0.29142,"ai_credits_total":0.29142,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false} +{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:56.563Z","event":"token_usage","request_id":"cc30e0e7-0491-484e-aaa9-6b020a2707f4","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19380,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2469,"response_bytes":8642,"x_initiator":"agent","ai_credits_this_response":0.2928,"ai_credits_total":0.58422,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false} +{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:58.490Z","event":"token_usage","request_id":"bf3f5fcd-173c-4da9-90d9-680dbc836352","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":272,"output_tokens":35,"cache_read_tokens":19200,"cache_write_tokens":0,"duration_ms":1799,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15018,"ai_credits_total":0.7344,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false} +{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:00.426Z","event":"token_usage","request_id":"776b23fd-a45c-46d4-9e74-95d5f0707402","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":236,"output_tokens":35,"cache_read_tokens":19328,"cache_write_tokens":0,"duration_ms":1861,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.1506,"ai_credits_total":0.885,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false} +{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:02.132Z","event":"token_usage","request_id":"59db8b91-cfc2-4cb9-9315-dbcb5e7bcfe7","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":200,"output_tokens":35,"cache_read_tokens":19456,"cache_write_tokens":0,"duration_ms":1552,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15102,"ai_credits_total":1.03602,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false} diff --git a/setup/js/model_costs.cjs b/setup/js/model_costs.cjs index b086fdb5..46730644 100644 --- a/setup/js/model_costs.cjs +++ b/setup/js/model_costs.cjs @@ -190,9 +190,10 @@ function usdToAIC(usd) { * @param {number} params.cacheReadTokens * @param {number} params.cacheWriteTokens * @param {number} [params.reasoningTokens] + * @param {boolean} [params.inputTokensIncludeCache] * @returns {number} */ -function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0 }) { +function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0, inputTokensIncludeCache }) { const pricing = findModelPricing(provider, model); if (!pricing) return 0; @@ -201,7 +202,12 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c const cacheRead = cacheReadTokens || 0; const cacheWrite = cacheWriteTokens || 0; const reasoning = reasoningTokens || 0; - const effectiveInput = providerIncludesCacheReadsInInput(provider) ? Math.max(input - cacheRead, 0) : input; + let effectiveInput = input; + if (inputTokensIncludeCache === true) { + effectiveInput = Math.max(input - cacheRead - cacheWrite, 0); + } else if (typeof inputTokensIncludeCache !== "boolean" && providerIncludesCacheReadsInInput(provider)) { + effectiveInput = Math.max(input - cacheRead, 0); + } const promptPrice = pricing.input || 0; const completionPrice = pricing.output || 0; @@ -221,6 +227,7 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c * @param {number} params.cacheReadTokens * @param {number} params.cacheWriteTokens * @param {number} [params.reasoningTokens] + * @param {boolean} [params.inputTokensIncludeCache] * @returns {number} */ function computeInferenceAIC(params) { diff --git a/setup/js/mount_mcp_as_cli.cjs b/setup/js/mount_mcp_as_cli.cjs index 3acaf78e..5460bd52 100644 --- a/setup/js/mount_mcp_as_cli.cjs +++ b/setup/js/mount_mcp_as_cli.cjs @@ -330,12 +330,12 @@ function parseMCPResponseBody(body) { * Follows the standard MCP handshake: initialize → notifications/initialized → tools/list. * * @param {string} serverUrl - HTTP URL of the MCP server endpoint - * @param {string} apiKey - Bearer token for gateway authentication + * @param {string} agentId - Bearer token for gateway authentication * @param {typeof import("@actions/core")} core - GitHub Actions core * @returns {Promise<{tools: Array<{name: string, description?: string, inputSchema?: unknown}>, emptyWasSuccessful: boolean}>} */ -async function fetchMCPToolsResult(serverUrl, apiKey, core) { - const authHeaders = { Authorization: apiKey }; +async function fetchMCPToolsResult(serverUrl, agentId, core) { + const authHeaders = { Authorization: agentId }; // Step 1: initialize – establish the session and capture Mcp-Session-Id if present /** @type {any} */ @@ -397,12 +397,12 @@ async function fetchMCPToolsResult(serverUrl, apiKey, core) { * Query the tools list from an MCP server via JSON-RPC. * * @param {string} serverUrl - HTTP URL of the MCP server endpoint - * @param {string} apiKey - ****** for gateway authentication + * @param {string} agentId - ****** for gateway authentication * @param {typeof import("@actions/core")} core - GitHub Actions core * @returns {Promise>} */ -async function fetchMCPTools(serverUrl, apiKey, core) { - const result = await fetchMCPToolsResult(serverUrl, apiKey, core); +async function fetchMCPTools(serverUrl, agentId, core) { + const result = await fetchMCPToolsResult(serverUrl, agentId, core); return result.tools; } @@ -419,7 +419,7 @@ async function fetchMCPTools(serverUrl, apiKey, core) { * stop immediately so unavailable backends still fail fast. * * @param {string} serverUrl - * @param {string} apiKey + * @param {string} agentId * @param {string} serverName - Server name, used only for log messages * @param {typeof import("@actions/core")} core * @param {object} [options] @@ -427,7 +427,7 @@ async function fetchMCPTools(serverUrl, apiKey, core) { * @param {(url: string, key: string, c: typeof import("@actions/core")) => Promise | {tools: Array<{name: string, description?: string, inputSchema?: unknown}>, emptyWasSuccessful: boolean}>} [options.fetchFn] - Fetch function (injectable for tests) * @returns {Promise>} */ -async function fetchMCPToolsWithRetry(serverUrl, apiKey, serverName, core, { sleep = undefined, fetchFn = undefined } = {}) { +async function fetchMCPToolsWithRetry(serverUrl, agentId, serverName, core, { sleep = undefined, fetchFn = undefined } = {}) { const doSleep = sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); const doFetchResult = async (url, key, c) => { if (!fetchFn) { @@ -439,11 +439,11 @@ async function fetchMCPToolsWithRetry(serverUrl, apiKey, serverName, core, { sle } return result; }; - let result = await doFetchResult(serverUrl, apiKey, core); + let result = await doFetchResult(serverUrl, agentId, core); for (let attempt = 1; attempt <= TOOLS_EMPTY_MAX_RETRIES && result.emptyWasSuccessful && result.tools.length === 0; attempt++) { core.warning(` tools/list returned 0 tools for '${serverName}', retrying in ${TOOLS_EMPTY_RETRY_DELAY_MS}ms (attempt ${attempt}/${TOOLS_EMPTY_MAX_RETRIES})...`); await doSleep(TOOLS_EMPTY_RETRY_DELAY_MS); - result = await doFetchResult(serverUrl, apiKey, core); + result = await doFetchResult(serverUrl, agentId, core); if (!result.emptyWasSuccessful) { core.warning(` stopping empty tools/list retries for '${serverName}' because tools/list did not complete successfully`); break; @@ -462,26 +462,26 @@ async function fetchMCPToolsWithRetry(serverUrl, apiKey, serverName, core, { sle * protocol (initialize → notifications/initialized → tools/call), help * display, argument parsing, console logging, and JSONL audit logging. * - * The gateway API key is baked directly into the generated script at - * generation time because MCP_GATEWAY_API_KEY is excluded from the AWF - * sandbox environment (--exclude-env MCP_GATEWAY_API_KEY) and would not + * The gateway agent ID is baked directly into the generated script at + * generation time because MCP_GATEWAY_AGENT_ID is excluded from the AWF + * sandbox environment (--exclude-env MCP_GATEWAY_AGENT_ID) and would not * be accessible to the agent at runtime. * * @param {string} serverName - Name of the MCP server * @param {string} serverUrl - HTTP URL of the MCP server endpoint * @param {string} toolsFile - Path to the cached tools JSON file - * @param {string} apiKey - Gateway API key, baked into the script at generation time + * @param {string} agentId - Gateway agent ID, baked into the script at generation time * @param {string} bridgeScript - Absolute path to mcp_cli_bridge.cjs * @returns {string} Content of the bash wrapper script */ -function generateCLIWrapperScript(serverName, serverUrl, toolsFile, apiKey, bridgeScript) { +function generateCLIWrapperScript(serverName, serverUrl, toolsFile, agentId, bridgeScript) { // Sanitize all values that are embedded in the shell script to prevent injection. // Server names are pre-validated by isValidServerName(), but we still escape all // values for defense-in-depth. const safeName = shellEscapeDoubleQuoted(serverName); const safeUrl = shellEscapeDoubleQuoted(serverUrl); const safeToolsFile = shellEscapeDoubleQuoted(toolsFile); - const safeApiKey = shellEscapeDoubleQuoted(apiKey); + const safeAgentId = shellEscapeDoubleQuoted(agentId); const safeBridge = shellEscapeDoubleQuoted(bridgeScript); return `#!/usr/bin/env bash @@ -502,7 +502,7 @@ exec node "${safeBridge}" \\ --server-name "${safeName}" \\ --server-url "${safeUrl}" \\ --tools-file "${safeToolsFile}" \\ - --api-key "${safeApiKey}" \\ + --api-key "${safeAgentId}" \\ "\$@" `; } @@ -556,9 +556,9 @@ async function main() { core.info(`Bridge script: ${bridgeScript}`); } - const apiKey = process.env.MCP_GATEWAY_API_KEY || ""; - if (!apiKey) { - core.warning("MCP_GATEWAY_API_KEY is not set; generated CLI wrappers will not be able to authenticate with the gateway"); + const agentId = process.env.MCP_GATEWAY_AGENT_ID || ""; + if (!agentId) { + core.warning("MCP_GATEWAY_AGENT_ID is not set; generated CLI wrappers will not be able to authenticate with the gateway"); } const gatewayDomain = process.env.MCP_GATEWAY_DOMAIN || ""; @@ -602,7 +602,7 @@ async function main() { // Query tools from the server using the host-accessible URL (mount step runs on host). // Retries on empty to handle the race between gateway health-reporting and // the backend finishing internal tool-schema construction (common with large configs). - let tools = await fetchMCPToolsWithRetry(url, apiKey, name, core); + let tools = await fetchMCPToolsWithRetry(url, agentId, name, core); const validate = SERVER_VALIDATORS[name]; if (validate) { tools = validate(tools, core); @@ -610,7 +610,7 @@ async function main() { core.info(` Found ${tools.length} tool(s)`); // Cache the tool list. This file only contains tool name/description/schema - // metadata returned by the gateway; it does not contain the API key or any + // metadata returned by the gateway; it does not contain the agent ID or any // other secret, so world-readable permissions (0o644) are acceptable here. try { fs.writeFileSync(toolsFile, JSON.stringify(tools, null, 2), { mode: 0o644 }); @@ -622,14 +622,14 @@ async function main() { const scriptPath = path.join(CLI_BIN_DIR, name); let scriptFd; try { - // Owner-only permissions: the wrapper script embeds the plaintext gateway API key, + // Owner-only permissions: the wrapper script embeds the plaintext gateway agent ID, // so it must not be world- or group-readable (matches chmod 600 used elsewhere for // this same credential, e.g. convert_gateway_config_copilot.sh). // Note: writeFileSync(mode) only applies when creating a new file; for existing files, // force mode with fchmodSync so prior permissive modes (e.g., 0o755) are corrected. scriptFd = fs.openSync(scriptPath, "w", 0o700); fs.fchmodSync(scriptFd, 0o700); - fs.writeFileSync(scriptFd, generateCLIWrapperScript(name, containerUrl, toolsFile, apiKey, bridgeScript), "utf8"); + fs.writeFileSync(scriptFd, generateCLIWrapperScript(name, containerUrl, toolsFile, agentId, bridgeScript), "utf8"); fs.closeSync(scriptFd); scriptFd = undefined; mountedServers.push(name); diff --git a/setup/js/parse_mcp_gateway_log.cjs b/setup/js/parse_mcp_gateway_log.cjs index 7927c11c..10b99103 100644 --- a/setup/js/parse_mcp_gateway_log.cjs +++ b/setup/js/parse_mcp_gateway_log.cjs @@ -67,14 +67,59 @@ function formatDurationMs(ms) { return `${minutes}m${secs}s`; } +/** + * AWF-reported AIC fields are numeric JSON fields. Invalid or missing values + * are identified so the caller can report that legacy pricing was used. + * + * @param {unknown} value + * @returns {number | null} + */ +function parseNonNegativeFiniteNumber(value) { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; +} + +/** + * Preserve AWF's reported six-decimal precision while retaining the + * historical three-decimal export for recomputed legacy records. + * + * @param {number} value + * @param {"awf_reported"|"recomputed"} source + * @returns {string} + */ +function formatAICForOutput(value, source) { + if (!Number.isFinite(value) || value < 0) return ""; + if (source !== "awf_reported") return value.toFixed(3); + const rounded = Number(value.toFixed(6)); + return String(rounded); +} + +/** + * Keep the human-readable table compact for large totals while preserving + * exact AWF precision for the normal per-run range. + * + * @param {number} value + * @param {"awf_reported"|"recomputed"} source + * @returns {string} + */ +function formatAICForTable(value, source) { + return source === "awf_reported" && value < 1000 ? formatAICForOutput(value, source) : formatAIC(value); +} + /** * Parses token-usage.jsonl content and returns an aggregated summary. + * + * token-usage.jsonl is agent-visible runtime telemetry. This parser uses its + * AWF-computed AIC fields only for diagnostics and public reporting; budget + * aborts, retries, authentication, and safe outputs use separate paths. + * * @param {string} jsonlContent - The token-usage.jsonl file content - * @returns {{totalInputTokens: number, totalOutputTokens: number, totalCacheReadTokens: number, totalCacheWriteTokens: number, totalRequests: number, totalDurationMs: number, totalAIC: number, ambientContextTokens: number|undefined, byModel: Object, entries: Array} | null} + * @returns {{totalInputTokens: number, totalOutputTokens: number, totalCacheReadTokens: number, totalCacheWriteTokens: number, totalRequests: number, totalDurationMs: number, totalAIC: number, aiCreditsSource: "awf_reported"|"recomputed", aiCreditsWarnings: string[], ambientContextTokens: number|undefined, byModel: Record, entries: Array} | null} * ambientContextTokens records first-request context size as: * input_tokens + ((cache_read_tokens + cache_write_tokens) / 10). */ function parseTokenUsageJsonl(jsonlContent) { + const seenRequestIds = new Set(); + let duplicateRecordCount = 0; const summary = { totalInputTokens: 0, totalOutputTokens: 0, @@ -83,9 +128,15 @@ function parseTokenUsageJsonl(jsonlContent) { totalRequests: 0, totalDurationMs: 0, totalAIC: 0, + /** @type {"awf_reported"|"recomputed"} */ + aiCreditsSource: "recomputed", + /** @type {string[]} */ + aiCreditsWarnings: [], + /** @type {number | undefined} */ ambientContextTokens: undefined, - byModel: {}, - /** @type {{ model: string, provider: string, inputTokens: number, outputTokens: number, cacheReadTokens: number, cacheWriteTokens: number, reasoningTokens: number, durationMs: number, deltaAIC: number }[]} */ + /** @type {Record} */ + byModel: Object.create(null), + /** @type {{ model: string, provider: string, inputTokens: number, outputTokens: number, cacheReadTokens: number, cacheWriteTokens: number, reasoningTokens: number, durationMs: number, timestampMs: number|null, originalIndex: number, inputTokensIncludeCache: boolean|undefined, hasInputTokensIncludeCacheField: boolean, reportedDeltaAIC: number|null, reportedTotalAIC: number|null, hasReportedDeltaField: boolean, hasReportedTotalField: boolean, deltaAIC: number, runningAIC: number }[]} */ entries: [], }; @@ -96,6 +147,14 @@ function parseTokenUsageJsonl(jsonlContent) { try { const entry = JSON.parse(trimmed); if (!entry || typeof entry !== "object") continue; + const requestId = typeof entry.request_id === "string" ? entry.request_id.trim() : ""; + const eventName = typeof entry.event === "string" && entry.event ? entry.event : "token_usage"; + const dedupeKey = requestId ? `${eventName}:${requestId}` : ""; + if (dedupeKey && seenRequestIds.has(dedupeKey)) { + duplicateRecordCount++; + continue; + } + if (dedupeKey) seenRequestIds.add(dedupeKey); const inputTokens = entry.input_tokens || 0; const outputTokens = entry.output_tokens || 0; @@ -103,6 +162,13 @@ function parseTokenUsageJsonl(jsonlContent) { const cacheWriteTokens = entry.cache_write_tokens || 0; const reasoningTokens = entry.reasoning_tokens || 0; const durationMs = entry.duration_ms || 0; + const parsedTimestamp = typeof entry.timestamp === "string" ? Date.parse(entry.timestamp) : Number.NaN; + const hasInputTokensIncludeCacheField = Object.prototype.hasOwnProperty.call(entry, "input_tokens_include_cache") && entry.input_tokens_include_cache !== null; + const inputTokensIncludeCache = typeof entry.input_tokens_include_cache === "boolean" ? entry.input_tokens_include_cache : undefined; + const hasReportedDelta = Object.prototype.hasOwnProperty.call(entry, "ai_credits_this_response"); + const hasReportedTotal = Object.prototype.hasOwnProperty.call(entry, "ai_credits_total"); + const reportedDeltaAIC = parseNonNegativeFiniteNumber(entry.ai_credits_this_response); + const reportedTotalAIC = parseNonNegativeFiniteNumber(entry.ai_credits_total); summary.totalInputTokens += inputTokens; summary.totalOutputTokens += outputTokens; @@ -136,41 +202,130 @@ function parseTokenUsageJsonl(jsonlContent) { m.requests++; m.durationMs += durationMs; - summary.entries.push({ model, provider: m.provider, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens, durationMs, deltaAIC: 0 }); + summary.entries.push({ + model, + provider: m.provider, + inputTokens, + outputTokens, + cacheReadTokens, + cacheWriteTokens, + reasoningTokens, + durationMs, + timestampMs: Number.isFinite(parsedTimestamp) ? parsedTimestamp : null, + originalIndex: summary.entries.length, + inputTokensIncludeCache, + hasInputTokensIncludeCacheField, + reportedDeltaAIC, + reportedTotalAIC, + hasReportedDeltaField: hasReportedDelta, + hasReportedTotalField: hasReportedTotal, + deltaAIC: 0, + runningAIC: 0, + }); } catch { // Malformed line — ignored. } } if (summary.totalRequests === 0) return null; + if (duplicateRecordCount > 0) { + summary.aiCreditsWarnings.push(`${duplicateRecordCount} duplicate token usage record(s) were ignored by event and request_id.`); + } + + const hasReportedAIC = summary.entries.some(entry => entry.reportedDeltaAIC !== null || entry.reportedTotalAIC !== null); + const hasAnyReportedAICFields = summary.entries.some(entry => entry.hasReportedDeltaField || entry.hasReportedTotalField); + const hasExplicitCacheSemantics = summary.entries.some(entry => typeof entry.inputTokensIncludeCache === "boolean"); + let invalidCacheSemanticsCount = 0; + + if (!hasAnyReportedAICFields && !hasExplicitCacheSemantics) { + invalidCacheSemanticsCount = summary.entries.filter(entry => entry.hasInputTokensIncludeCacheField && typeof entry.inputTokensIncludeCache !== "boolean").length; + // Preserve the legacy aggregation contract exactly for records emitted before + // AWF added reported AIC and explicit cache-semantics fields. + let totalAIC = 0; + for (const [model, usage] of Object.entries(summary.byModel)) { + const aic = computeInferenceAIC({ + provider: usage.provider || "", + model, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + reasoningTokens: usage.reasoningTokens || 0, + }); + usage.aic = aic; + totalAIC += aic; + } + summary.totalAIC = totalAIC; - let totalAIC = 0; - for (const [model, usage] of Object.entries(summary.byModel)) { - const aic = computeInferenceAIC({ - provider: usage.provider || "", - model, - inputTokens: usage.inputTokens, - outputTokens: usage.outputTokens, - cacheReadTokens: usage.cacheReadTokens, - cacheWriteTokens: usage.cacheWriteTokens, - reasoningTokens: usage.reasoningTokens || 0, + for (const entry of summary.entries) { + entry.deltaAIC = computeInferenceAIC({ + provider: entry.provider || "", + model: entry.model, + inputTokens: entry.inputTokens, + outputTokens: entry.outputTokens, + cacheReadTokens: entry.cacheReadTokens, + cacheWriteTokens: entry.cacheWriteTokens, + reasoningTokens: entry.reasoningTokens || 0, + }); + } + } else { + summary.entries.sort((left, right) => { + if (left.timestampMs !== null && right.timestampMs !== null) { + return left.timestampMs - right.timestampMs || left.originalIndex - right.originalIndex; + } + if (left.timestampMs !== null) return -1; + if (right.timestampMs !== null) return 1; + return left.originalIndex - right.originalIndex; }); - usage.aic = aic; - totalAIC += aic; + const firstEntry = summary.entries[0]; + if (firstEntry) { + summary.ambientContextTokens = firstEntry.inputTokens + (firstEntry.cacheReadTokens + firstEntry.cacheWriteTokens) / 10; + } + let runningAIC = 0; + let fallbackRecordCount = 0; + for (const usage of Object.values(summary.byModel)) { + usage.aic = 0; + } + + for (let index = 0; index < summary.entries.length; index++) { + const entry = summary.entries[index]; + const reportedFieldsMissingOrInvalid = hasAnyReportedAICFields && (!entry.hasReportedDeltaField || entry.reportedDeltaAIC === null || !entry.hasReportedTotalField || entry.reportedTotalAIC === null); + if (reportedFieldsMissingOrInvalid) fallbackRecordCount++; + + if (entry.reportedDeltaAIC !== null) { + entry.deltaAIC = entry.reportedDeltaAIC; + } else { + if (entry.hasInputTokensIncludeCacheField && typeof entry.inputTokensIncludeCache !== "boolean") { + invalidCacheSemanticsCount++; + } + entry.deltaAIC = computeInferenceAIC({ + provider: entry.provider || "", + model: entry.model, + inputTokens: entry.inputTokens, + outputTokens: entry.outputTokens, + cacheReadTokens: entry.cacheReadTokens, + cacheWriteTokens: entry.cacheWriteTokens, + reasoningTokens: entry.reasoningTokens || 0, + inputTokensIncludeCache: entry.inputTokensIncludeCache, + }); + } + summary.byModel[entry.model].aic += entry.deltaAIC; + runningAIC = entry.reportedTotalAIC ?? runningAIC + entry.deltaAIC; + entry.runningAIC = runningAIC; + } + + summary.totalAIC = runningAIC; + summary.aiCreditsSource = hasReportedAIC ? "awf_reported" : "recomputed"; + if (fallbackRecordCount > 0) { + summary.aiCreditsWarnings.push(`${fallbackRecordCount} token usage record(s) had missing or invalid AWF-reported AI Credits fields; fallback accounting was used for the missing values.`); + } + const summedDeltaAIC = Object.values(summary.byModel).reduce((total, usage) => total + (usage.aic || 0), 0); + if (summary.aiCreditsSource === "awf_reported" && Math.abs(summedDeltaAIC - summary.totalAIC) > 1e-6 * Math.max(1, Math.abs(summary.totalAIC))) { + summary.aiCreditsWarnings.push("The AWF-reported cumulative AI Credits total differs from the sum of per-request credits; the cumulative total was preserved for reporting."); + } } - summary.totalAIC = totalAIC; - - // Compute per-request AI credits. - for (const entry of summary.entries) { - entry.deltaAIC = computeInferenceAIC({ - provider: entry.provider || "", - model: entry.model, - inputTokens: entry.inputTokens, - outputTokens: entry.outputTokens, - cacheReadTokens: entry.cacheReadTokens, - cacheWriteTokens: entry.cacheWriteTokens, - reasoningTokens: entry.reasoningTokens || 0, - }); + if (invalidCacheSemanticsCount > 0) { + summary.aiCreditsWarnings.push(`${invalidCacheSemanticsCount} token usage record(s) had invalid input_tokens_include_cache values; legacy provider cache semantics were used.`); } return summary; @@ -180,7 +335,7 @@ function parseTokenUsageJsonl(jsonlContent) { * Generates a markdown summary section for token usage data. * Renders one row per request in chronological order with per-request AI credits, * a running AI credits total, followed by an aggregate totals row and legend. - * @param {{totalInputTokens: number, totalOutputTokens: number, totalCacheReadTokens: number, totalCacheWriteTokens: number, totalRequests: number, totalDurationMs: number, totalAIC: number, byModel: Object, entries: Array} | null} summary + * @param {ReturnType} summary * @returns {string} Markdown section, or empty string if no data */ function generateTokenUsageSummary(summary) { @@ -192,22 +347,33 @@ function generateTokenUsageSummary(summary) { const entries = summary.entries || []; let compoundedAIC = 0; + const formatSummaryAIC = value => formatAICForTable(value, summary.aiCreditsSource); for (let i = 0; i < entries.length; i++) { const entry = entries[i]; const deltaAIC = entry.deltaAIC || 0; compoundedAIC += deltaAIC; + const runningAIC = summary.aiCreditsSource === "awf_reported" ? entry.runningAIC : compoundedAIC; lines.push( - `| ${i + 1} | ${formatModelEmojiAlias(entry.model) || entry.model} | ${entry.inputTokens.toLocaleString()} | ${entry.outputTokens.toLocaleString()} | ${entry.cacheReadTokens.toLocaleString()} | ${entry.cacheWriteTokens.toLocaleString()} | ${formatAIC(deltaAIC)} | ${formatAIC(compoundedAIC)} | ${formatDurationMs(entry.durationMs)} |` + `| ${i + 1} | ${formatModelEmojiAlias(entry.model) || entry.model} | ${entry.inputTokens.toLocaleString()} | ${entry.outputTokens.toLocaleString()} | ${entry.cacheReadTokens.toLocaleString()} | ${entry.cacheWriteTokens.toLocaleString()} | ${formatSummaryAIC(deltaAIC)} | ${formatSummaryAIC(runningAIC)} | ${formatDurationMs(entry.durationMs)} |` ); } - const totalAIC = formatAIC(summary.totalAIC || 0); + const totalAIC = formatSummaryAIC(summary.totalAIC || 0); lines.push( `| **Total** | | **${summary.totalInputTokens.toLocaleString()}** | **${summary.totalOutputTokens.toLocaleString()}** | **${summary.totalCacheReadTokens.toLocaleString()}** | **${summary.totalCacheWriteTokens.toLocaleString()}** | | **${totalAIC}** | **${formatDurationMs(summary.totalDurationMs)}** |` ); lines.push(""); - lines.push("Legend: `Alias` shows the model shorthand used in the table. `ΔAI Credits` is the per-request cost, and `AI Credits` is the running total computed with the current AI credits pricing model."); + const accountingDescription = + summary.aiCreditsSource === "awf_reported" + ? summary.aiCreditsWarnings.length > 0 + ? "mirrored from AWF fields where available, with warned fallback accounting" + : "mirrored from AWF fields for reporting" + : "recomputed with the current AI credits pricing model for legacy records"; + lines.push(`Legend: \`Alias\` shows the model shorthand used in the table. \`ΔAI Credits\` is the per-request cost, and \`AI Credits\` is the running total ${accountingDescription}.`); + for (const warning of summary.aiCreditsWarnings) { + lines.push(`Warning: ${warning}`); + } lines.push(""); return lines.join("\n") + "\n"; @@ -233,11 +399,14 @@ async function writeStepSummaryWithTokenUsage(coreObj) { if (content?.trim()) { coreObj.info(`Found token-usage.jsonl (${content.length} bytes)`); const parsedSummary = parseTokenUsageJsonl(content); - if (parsedSummary && parsedSummary.totalAIC > 0) { - const roundedAIC = parsedSummary.totalAIC.toFixed(3); - coreObj.exportVariable("GH_AW_AIC", roundedAIC); - coreObj.setOutput("aic", roundedAIC); - coreObj.info(`AI Credits: ${roundedAIC}`); + for (const warning of parsedSummary?.aiCreditsWarnings || []) { + coreObj.warning?.(`[ai-credits] ${warning}`); + } + if (parsedSummary && (parsedSummary.aiCreditsSource === "awf_reported" || parsedSummary.totalAIC > 0)) { + const aic = formatAICForOutput(parsedSummary.totalAIC, parsedSummary.aiCreditsSource); + coreObj.exportVariable("GH_AW_AIC", aic); + coreObj.setOutput("aic", aic); + coreObj.info(`AI Credits: ${aic}`); } if (parsedSummary && typeof parsedSummary.ambientContextTokens === "number" && parsedSummary.ambientContextTokens > 0) { const roundedAmbientContext = String(Math.round(parsedSummary.ambientContextTokens)); @@ -1158,6 +1327,8 @@ if (typeof module !== "undefined" && module.exports) { parseTokenUsageJsonl, generateTokenUsageSummary, formatDurationMs, + formatAICForOutput, + writeStepSummaryWithTokenUsage, hasAICreditsRateLimitError, hasUnknownModelAICreditsError, setUnknownModelAICreditsOutput, diff --git a/setup/js/parse_token_usage.cjs b/setup/js/parse_token_usage.cjs index c571610a..e7daa3d3 100644 --- a/setup/js/parse_token_usage.cjs +++ b/setup/js/parse_token_usage.cjs @@ -4,7 +4,7 @@ const fs = require("fs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { ERR_PARSE } = require("./error_codes.cjs"); -const { parseTokenUsageJsonl, generateTokenUsageSummary } = require("./parse_mcp_gateway_log.cjs"); +const { parseTokenUsageJsonl, generateTokenUsageSummary, formatAICForOutput } = require("./parse_mcp_gateway_log.cjs"); const { calculateWorkingSetFromJSONL } = require("./working_set_metrics.cjs"); /** @@ -50,12 +50,24 @@ function getReadableTokenUsagePaths(paths) { * @returns {string} */ function extractRequestId(line) { - const match = line.match(/"request_id"\s*:\s*"((?:\\.|[^"\\])*)"/); - return match ? match[1] : ""; + const requestMatch = line.match(/"request_id"\s*:\s*"((?:\\.|[^"\\])*)"/); + return requestMatch ? requestMatch[1] : ""; } /** - * Reads token usage files and deduplicates overlapping lines by request_id. + * Extracts a cross-file dedupe key with lightweight matching (no full JSON parse). + * @param {string} line + * @returns {string} + */ +function extractTokenUsageDedupeKey(line) { + const requestId = extractRequestId(line); + if (!requestId) return ""; + const eventMatch = line.match(/"event"\s*:\s*"((?:\\.|[^"\\])*)"/); + return `${eventMatch ? eventMatch[1] : "token_usage"}:${requestId}`; +} + +/** + * Reads token usage files and deduplicates overlapping lines by event and request_id. * Falls back to raw line dedupe when request_id is absent. * @param {string[]} paths * @returns {string} @@ -76,8 +88,7 @@ function readDedupedTokenUsage(paths) { for (const line of fileContent.split("\n")) { const trimmed = line.trim(); if (!trimmed) continue; - const requestId = extractRequestId(trimmed); - const dedupeKey = requestId ? `request_id:${requestId}` : trimmed; + const dedupeKey = extractTokenUsageDedupeKey(trimmed) || trimmed; if (uniqueLineKeys.has(dedupeKey)) continue; uniqueLineKeys.add(dedupeKey); dedupedLines.push(trimmed); @@ -205,6 +216,9 @@ async function main() { core.info("Token usage file contained no valid entries"); return; } + for (const warning of summary.aiCreditsWarnings) { + core.warning(`[ai-credits] ${warning}`); + } const markdown = generateTokenUsageSummary(summary); const workingSet = calculateWorkingSetFromJSONL(content).workingSet; if (markdown.length > 0) { @@ -234,7 +248,7 @@ async function main() { cache_read_tokens: summary.totalCacheReadTokens, cache_write_tokens: summary.totalCacheWriteTokens, ambient_context: Math.round(summary.ambientContextTokens || 0), - ai_credits: Number((summary.totalAIC || 0).toFixed(3)), + ai_credits: summary.aiCreditsSource === "awf_reported" ? Number(summary.totalAIC.toFixed(6)) : Number((summary.totalAIC || 0).toFixed(3)), ...(primaryModel ? { primary_model: primaryModel } : {}), }; fs.writeFileSync(AGENT_USAGE_PATH, JSON.stringify(agentUsage) + "\n"); @@ -244,8 +258,8 @@ async function main() { core.setOutput("primary_model", primaryModel); core.info(`Primary model: ${primaryModel}`); } - if (summary.totalAIC > 0) { - const aic = summary.totalAIC.toFixed(3); + if (summary.aiCreditsSource === "awf_reported" || summary.totalAIC > 0) { + const aic = formatAICForOutput(summary.totalAIC, summary.aiCreditsSource); core.exportVariable("GH_AW_AIC", aic); core.setOutput("aic", aic); core.info(`AI Credits: ${aic}`); @@ -267,6 +281,7 @@ if (typeof module !== "undefined" && module.exports) { main, getReadableTokenUsagePaths, extractRequestId, + extractTokenUsageDedupeKey, readDedupedTokenUsage, getSummaryTitle, buildStepSummarySection, diff --git a/setup/js/pi_agent_core_driver.cjs b/setup/js/pi_agent_core_driver.cjs index e6972ffd..7489eefe 100644 --- a/setup/js/pi_agent_core_driver.cjs +++ b/setup/js/pi_agent_core_driver.cjs @@ -235,7 +235,18 @@ function buildModel(gatewayConfig, modelStr) { } // Determine the pi-ai api type for the provider. - const api = provider === "anthropic" ? "anthropic-messages" : "openai-completions"; + // Real OpenAI models are only published under "openai-responses" in Pi's upstream + // model catalog; OpenAI's Chat Completions endpoint rejects function tools whenever + // reasoning_effort is anything other than "none" (see + // https://developers.openai.com/api/docs/guides/responses-vs-chat-completions), so + // routing the "openai" provider through /responses keeps tool calling working. + // GitHub Copilot's gateway keeps its existing chat-completions-style protocol. + let api = "openai-completions"; + if (provider === "anthropic") { + api = "anthropic-messages"; + } else if (provider === "openai") { + api = "openai-responses"; + } return { id: modelId, diff --git a/setup/js/pi_models_json.cjs b/setup/js/pi_models_json.cjs index 44c4e9bb..1dd62533 100644 --- a/setup/js/pi_models_json.cjs +++ b/setup/js/pi_models_json.cjs @@ -79,16 +79,16 @@ function resolveGatewayBaseUrl(options) { * "COPILOT_GITHUB_TOKEN") causes Pi to automatically use the value that is * already present in the container environment. * - * @param {{ baseUrl: string, apiKeyEnvVar: string, modelId: string }} options + * @param {{ baseUrl: string, apiKeyEnvVar: string, modelId: string, api?: string }} options * @returns {string} */ function buildModelsJSON(options) { - const { baseUrl, apiKeyEnvVar, modelId } = options; + const { baseUrl, apiKeyEnvVar, modelId, api } = options; return JSON.stringify({ providers: { "aw-gateway": { baseUrl, - api: "openai-completions", + api: api || "openai-completions", apiKey: apiKeyEnvVar, models: [{ id: modelId }], }, @@ -96,6 +96,33 @@ function buildModelsJSON(options) { }); } +/** + * Resolve the Pi API family for a given normalized GH_AW_LLM_PROVIDER value. + * + * Real OpenAI models are only published under the "openai-responses" API in Pi's + * upstream model catalog (@earendil-works/pi-ai) — the "openai-completions" family + * is reserved for OpenAI-compatible-but-not-OpenAI providers (Groq, DeepSeek, etc.). + * Since OpenAI's Chat Completions endpoint rejects function tools whenever + * reasoning_effort is anything other than "none" (see + * https://developers.openai.com/api/docs/guides/responses-vs-chat-completions), + * routing the "openai" provider through /responses keeps tool calling working for + * all reasoning-capable models without requiring workflow authors to opt in. + * + * Other providers (github/copilot, anthropic) keep their existing chat-completions- + * style gateway protocol, which is unaffected by this OpenAI-specific restriction. + * The AWF api-proxy gateway (used here) exposes a normalized chat-completions-style + * surface for every backend it fronts, including anthropic — this is a distinct + * protocol layer from the native "anthropic-messages" api used in no-firewall mode + * (see pi_agent_core_driver.cjs's buildModel), so anthropic intentionally falls + * through to "openai-completions" here rather than "anthropic-messages". + * + * @param {string} provider - normalized GH_AW_LLM_PROVIDER value (e.g. "openai", "anthropic", "github") + * @returns {string} + */ +function resolvePiApiForProvider(provider) { + return provider === "openai" || provider === "codex" ? "openai-responses" : "openai-completions"; +} + async function main() { const logger = DEFAULT_LOGGER; const modelId = process.env.GH_AW_PI_MODEL_ID || ""; @@ -127,7 +154,10 @@ async function main() { const { baseUrl, source } = resolveGatewayBaseUrl({ provider, fallbackPort, reflectData, logger }); logger(`resolved gateway baseUrl=${baseUrl} (source=${source}, provider=${provider}, fallbackPort=${fallbackPort})`); - const modelsJSON = buildModelsJSON({ baseUrl, apiKeyEnvVar, modelId }); + const api = resolvePiApiForProvider(provider); + logger(`resolved gateway api=${api} (provider=${provider})`); + + const modelsJSON = buildModelsJSON({ baseUrl, apiKeyEnvVar, modelId, api }); fs.mkdirSync(path.dirname(outputPath), { recursive: true }); fs.writeFileSync(outputPath, modelsJSON, "utf8"); logger(`wrote ${outputPath}`); @@ -140,4 +170,4 @@ if (require.main === module) { }); } -module.exports = { main, resolveGatewayBaseUrl, buildModelsJSON, DEFAULT_PI_CODING_AGENT_DIR }; +module.exports = { main, resolveGatewayBaseUrl, buildModelsJSON, resolvePiApiForProvider, DEFAULT_PI_CODING_AGENT_DIR }; diff --git a/setup/js/pi_provider.cjs b/setup/js/pi_provider.cjs index cf302aa3..4501d531 100644 --- a/setup/js/pi_provider.cjs +++ b/setup/js/pi_provider.cjs @@ -277,7 +277,12 @@ function registerConfiguredProviders(pi, logger) { ["openai", "codex"], { apiKey: openAIKey, - api: "openai-completions", + // Real OpenAI models are only published under "openai-responses" in Pi's + // upstream model catalog. OpenAI's Chat Completions endpoint rejects function + // tools whenever reasoning_effort is anything other than "none", so routing + // through /responses keeps tool calling working for reasoning-capable models. + // See: https://developers.openai.com/api/docs/guides/responses-vs-chat-completions + api: "openai-responses", ...(process.env.OPENAI_BASE_URL ? { baseUrl: process.env.OPENAI_BASE_URL } : {}), }, logger diff --git a/setup/js/send_otlp_span.cjs b/setup/js/send_otlp_span.cjs index cb323d1a..d3b2abfb 100644 --- a/setup/js/send_otlp_span.cjs +++ b/setup/js/send_otlp_span.cjs @@ -708,6 +708,87 @@ function buildExperimentAttributes(assignments) { return attrs; } +/** + * Build summary attributes and per-result events from valid deterministic grader output. + * Free-form grader messages, details, and errors are intentionally excluded because + * custom graders may derive them from trace content containing sensitive values. + * + * @param {any} graderOutput + * @param {number} eventTimeMs + * @returns {{attributes: Array<{key: string, value: object}>, events: Array<{timeUnixNano: string, name: string, attributes: Array<{key: string, value: object}>}>}} + */ +function buildGraderTelemetry(graderOutput, eventTimeMs) { + if (!graderOutput || typeof graderOutput !== "object" || !Array.isArray(graderOutput.results) || graderOutput.results.length === 0) { + return { attributes: [], events: [] }; + } + + const results = graderOutput.results.filter(result => result && typeof result === "object" && typeof result.id === "string" && result.id); + if (results.length === 0) { + return { attributes: [], events: [] }; + } + + const countByStatus = status => results.filter(result => result.status === status).length; + const attributes = [ + buildAttr("gh-aw.graders.count", results.length), + buildAttr("gh-aw.graders.passed", countByStatus("pass")), + buildAttr("gh-aw.graders.failed", countByStatus("fail")), + buildAttr("gh-aw.graders.errors", countByStatus("error")), + buildAttr("gh-aw.graders.unavailable", countByStatus("unavailable")), + buildAttr("gh-aw.graders.other", results.length - countByStatus("pass") - countByStatus("fail") - countByStatus("error") - countByStatus("unavailable")), + ]; + const timeUnixNano = toNanoString(eventTimeMs); + const events = results.map(result => { + const resultAttributes = [buildAttr("gh-aw.grader.id", result.id)]; + if (typeof result.name === "string" && result.name) resultAttributes.push(buildAttr("gh-aw.grader.name", result.name)); + if (typeof result.status === "string" && result.status) resultAttributes.push(buildAttr("gh-aw.grader.status", result.status)); + if (typeof result.source === "string" && result.source) resultAttributes.push(buildAttr("gh-aw.grader.source", result.source)); + if (typeof result.unit === "string" && result.unit) resultAttributes.push(buildAttr("gh-aw.grader.unit", result.unit)); + if (typeof result.value === "number" && Number.isFinite(result.value)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.value", result.value)); + if (typeof result.passed === "boolean") resultAttributes.push(buildAttr("gh-aw.grader.passed", result.passed)); + if (typeof result.severity === "string" && result.severity) resultAttributes.push(buildAttr("gh-aw.grader.severity", result.severity)); + if (typeof result.baselineValue === "number" && Number.isFinite(result.baselineValue)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.baseline_value", result.baselineValue)); + if (typeof result.deltaFromBaseline === "number" && Number.isFinite(result.deltaFromBaseline)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.delta_from_baseline", result.deltaFromBaseline)); + return { timeUnixNano, name: "grader.result", attributes: resultAttributes }; + }); + + return { attributes, events }; +} + +/** + * Build summary attributes and per-result events from BinEval JSONL records. + * Only bounded, structured fields are included; free-form questions are excluded. + * + * @param {any[]} evalResults + * @param {number} eventTimeMs + * @returns {{attributes: Array<{key: string, value: object}>, events: Array<{timeUnixNano: string, name: string, attributes: Array<{key: string, value: object}>}>}} + */ +function buildEvalTelemetry(evalResults, eventTimeMs) { + if (!Array.isArray(evalResults)) { + return { attributes: [], events: [] }; + } + const results = evalResults.filter(result => result && typeof result === "object" && typeof result.id === "string" && result.id); + if (results.length === 0) { + return { attributes: [], events: [] }; + } + + const normalizedAnswers = results.map(result => { + const answer = typeof result.answer === "string" ? result.answer.trim().toUpperCase() : "UNKNOWN"; + return answer === "YES" || answer === "NO" ? answer : "UNKNOWN"; + }); + const countAnswer = answer => normalizedAnswers.filter(value => value === answer).length; + const attributes = [buildAttr("gh-aw.evals.count", results.length), buildAttr("gh-aw.evals.yes", countAnswer("YES")), buildAttr("gh-aw.evals.no", countAnswer("NO")), buildAttr("gh-aw.evals.unknown", countAnswer("UNKNOWN"))]; + const timeUnixNano = toNanoString(eventTimeMs); + const events = results.map((result, index) => { + const resultAttributes = [buildAttr("gh-aw.eval.id", result.id), buildAttr("gh-aw.eval.answer", normalizedAnswers[index])]; + if (typeof result.model === "string" && result.model) { + resultAttributes.push(buildAttr("gh-aw.eval.model", result.model)); + } + return { timeUnixNano, name: "eval.result", attributes: resultAttributes }; + }); + + return { attributes, events }; +} + // --------------------------------------------------------------------------- // Custom OTLP attributes (GH_AW_OTLP_ATTRIBUTES) // --------------------------------------------------------------------------- @@ -1450,6 +1531,20 @@ function readJSONIfExists(filePath) { } } +/** + * Safely read and parse a JSONL file. Returns an empty array on any error. + * + * @param {string} filePath - Absolute path to the JSONL file + * @returns {any[]} + */ +function readJSONLIfExists(filePath) { + try { + return parseJsonlContent(fs.readFileSync(filePath, "utf8")); + } catch { + return []; + } +} + /** * Path to the GitHub rate-limit JSONL log file. * Mirrors GITHUB_RATE_LIMITS_JSONL_PATH from constants.cjs without introducing @@ -1991,6 +2086,8 @@ function readAgentRuntimeMetrics() { * - `/tmp/gh-aw/agent_usage.json` – per-type token breakdown written by parse_token_usage.cjs; * provides `input_tokens`, `output_tokens`, * `cache_read_tokens`, and `cache_write_tokens` counters + * - `/tmp/gh-aw/agent/graders/grader_results.json` – deterministic grader + * summary attributes and per-result span events * * @param {string} spanName - OTLP span name (e.g. `"gh-aw.job.conclusion"`) * @param {{ startMs?: number }} [options] @@ -2327,6 +2424,9 @@ async function sendJobConclusionSpan(spanName, options = {}) { } } + const graderTelemetry = jobName === "agent" ? buildGraderTelemetry(readJSONIfExists("/tmp/gh-aw/agent/graders/grader_results.json"), endMs) : { attributes: [], events: [] }; + const evalTelemetry = jobName === "evals" ? buildEvalTelemetry(readJSONLIfExists("/tmp/gh-aw/evals.jsonl"), endMs) : { attributes: [], events: [] }; + const resourceAttributes = buildGitHubActionsResourceAttributes({ repository, runId, @@ -2385,7 +2485,7 @@ async function sendJobConclusionSpan(spanName, options = {}) { }); }; - const spanEvents = buildSpanEvents(endMs); + const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events, ...evalTelemetry.events]; // Prefer the timestamp written at the very beginning of the Execute Agent CLI step // (captures true step start on the host, before the AWF container launches) so the @@ -2480,6 +2580,12 @@ async function sendJobConclusionSpan(spanName, options = {}) { } } + // Grader results are run-level outcomes. They belong only on the agent job's + // conclusion span, rather than its dedicated child span or downstream jobs + // which may have downloaded the agent artifact. + attributes.push(...graderTelemetry.attributes); + attributes.push(...evalTelemetry.attributes); + // Only attach token-usage attributes to jobs that actually executed model usage. // Most downstream jobs (conclusion, safe_outputs) may have agent_usage.json on // disk via artifact download but must NOT emit token data — otherwise every @@ -2564,6 +2670,8 @@ module.exports = { OTEL_JSONL_PATH, appendToOTLPJSONL, buildExperimentAttributes, + buildGraderTelemetry, + buildEvalTelemetry, parseOTLPCustomAttributes, buildCustomOTLPAttributes, }; diff --git a/setup/js/start_mcp_gateway.cjs b/setup/js/start_mcp_gateway.cjs index a4e6220d..b76cb0af 100644 --- a/setup/js/start_mcp_gateway.cjs +++ b/setup/js/start_mcp_gateway.cjs @@ -18,7 +18,7 @@ require("./shim.cjs"); * * Required environment variables: * - MCP_GATEWAY_DOCKER_COMMAND: Container image to run (required) - * - MCP_GATEWAY_API_KEY: API key for gateway authentication (required for converter scripts) + * - MCP_GATEWAY_AGENT_ID: agent ID for gateway authentication (required for converter scripts) * - MCP_GATEWAY_PORT: Port for MCP gateway * - MCP_GATEWAY_DOMAIN: Domain for MCP server URLs (e.g., host.docker.internal) * - RUNNER_TEMP: GitHub Actions runner temp directory @@ -249,7 +249,7 @@ function writeGatewayStartupMarker(markerPath) { */ const gatewayCredentialRedactions = [ { pattern: /(Bearer\s+)\S+/gi, replacement: "$1[REDACTED]" }, - { pattern: /((?:api[_-]?key|token|secret|password|authorization)"?\s*[:=]\s*"?)[^\s,}"]+/gi, replacement: "$1[REDACTED]" }, + { pattern: /((?:agent[_-]?id|api[_-]?key|token|secret|password|authorization)"?\s*[:=]\s*"?)[^\s,}"]+/gi, replacement: "$1[REDACTED]" }, ]; /** @@ -660,7 +660,7 @@ async function main() { process.umask(0o077); const dockerCommand = process.env.MCP_GATEWAY_DOCKER_COMMAND; - const apiKey = process.env.MCP_GATEWAY_API_KEY; + const agentId = process.env.MCP_GATEWAY_AGENT_ID; const gatewayPort = process.env.MCP_GATEWAY_PORT; const gatewayDomain = process.env.MCP_GATEWAY_DOMAIN; const runnerTemp = process.env.RUNNER_TEMP; @@ -821,8 +821,8 @@ async function main() { core.setFailed("ERROR: Gateway configuration is missing required 'domain' field"); return; } - if (!("apiKey" in gw) || gw.apiKey == null) { - core.setFailed("ERROR: Gateway configuration is missing required 'apiKey' field"); + if (!("agentId" in gw) || gw.agentId == null) { + core.setFailed("ERROR: Gateway configuration is missing required 'agentId' field"); return; } @@ -1137,11 +1137,11 @@ async function main() { const configConvertStart = nowMs(); process.env.MCP_GATEWAY_OUTPUT = outputPath; - // Validate MCP_GATEWAY_API_KEY - if (!apiKey) { + // Validate MCP_GATEWAY_AGENT_ID + if (!agentId) { stopGatewayProcess(gatewayPid); core.error("This variable should be set in the workflow before calling start_mcp_gateway.cjs"); - core.setFailed("ERROR: MCP_GATEWAY_API_KEY environment variable must be set for converter scripts"); + core.setFailed("ERROR: MCP_GATEWAY_AGENT_ID environment variable must be set for converter scripts"); return; } @@ -1239,11 +1239,11 @@ async function main() { if (fs.existsSync(checkScript)) { core.info("Running MCP server checks..."); - // Pass apiKey via MCP_GATEWAY_API_KEY env var (already set) rather than + // Pass agentId via MCP_GATEWAY_AGENT_ID env var (already set) rather than // as a shell argument to avoid shell metacharacter injection risks. const safePort = String(gatewayPort).replace(/[^0-9]/g, ""); try { - execFileSync("bash", [checkScript, outputPath, `http://localhost:${safePort}`, process.env.MCP_GATEWAY_API_KEY || ""], { + execFileSync("bash", [checkScript, outputPath, `http://localhost:${safePort}`, process.env.MCP_GATEWAY_AGENT_ID || ""], { stdio: "inherit", env: { ...process.env, GH_AW_MCP_OPTIONAL_SERVERS: optionalServerNames.join(",") }, timeout: MCP_SERVER_CHECK_TIMEOUT_MS, @@ -1334,7 +1334,7 @@ async function main() { // Write GitHub Actions step outputs // ----------------------------------------------------------------------- if (githubOutput) { - const outputs = [`gateway-pid=${gatewayPid}`, `gateway-port=${gatewayPort}`, `gateway-api-key=${apiKey}`, `gateway-domain=${gatewayDomain}`].join("\n"); + const outputs = [`gateway-pid=${gatewayPid}`, `gateway-port=${gatewayPort}`, `gateway-agent-id=${agentId}`, `gateway-domain=${gatewayDomain}`].join("\n"); try { fs.appendFileSync(githubOutput, outputs + "\n"); } catch { diff --git a/setup/js/testdata/copilot_sdk_web_fetch_contract.json b/setup/js/testdata/copilot_sdk_web_fetch_contract.json new file mode 100644 index 00000000..cbc34f36 --- /dev/null +++ b/setup/js/testdata/copilot_sdk_web_fetch_contract.json @@ -0,0 +1,18 @@ +{ + "serverArgs": ["--disable-builtin-mcps", "--no-ask-user", "--allow-tool", "web_fetch"], + "toolConfig": { + "version": 1, + "capabilities": { + "bash": false, + "edit": false, + "webFetch": true, + "webSearch": false, + "mcp": false, + "cliProxy": false + }, + "permissions": { + "allowedTools": ["read", "web_fetch"] + }, + "explicitlyDisabledTools": ["bash", "cli-proxy", "edit", "github"] + } +} diff --git a/setup/js/trace_graders.cjs b/setup/js/trace_graders.cjs index 0a2b64f3..ab6418f7 100644 --- a/setup/js/trace_graders.cjs +++ b/setup/js/trace_graders.cjs @@ -524,8 +524,7 @@ function normalizeResult(id, rawResult, meta) { if (typeof rawResult === "object" && rawResult !== null && !Array.isArray(rawResult)) { // Object result from custom script value = rawResult.value; - if (rawResult.unit) base.unit = String(rawResult.unit); - if (rawResult.severity) base.severity = String(rawResult.severity); + if (typeof rawResult.severity === "string" && ["error", "warning", "info", "note"].includes(rawResult.severity)) base.severity = rawResult.severity; if (rawResult.details) base.details = String(rawResult.details); if (rawResult.message) base.message = String(rawResult.message); if (typeof rawResult.passed === "boolean") base.passed = rawResult.passed; diff --git a/setup/sh/check_mcp_servers.sh b/setup/sh/check_mcp_servers.sh index 408308b9..6aae01a7 100755 --- a/setup/sh/check_mcp_servers.sh +++ b/setup/sh/check_mcp_servers.sh @@ -25,25 +25,25 @@ print_timing() { echo "⏱️ TIMING: $label took ${duration}ms" } -# Usage: check_mcp_servers.sh GATEWAY_CONFIG_PATH GATEWAY_URL GATEWAY_API_KEY +# Usage: check_mcp_servers.sh GATEWAY_CONFIG_PATH GATEWAY_URL GATEWAY_AGENT_ID # # Arguments: # GATEWAY_CONFIG_PATH : Path to the gateway output configuration file (gateway-output.json) # GATEWAY_URL : The HTTP URL of the MCP gateway (e.g., http://localhost:8080) -# GATEWAY_API_KEY : API key for gateway authentication +# GATEWAY_AGENT_ID : Agent/session identifier for gateway authentication # # Exit codes: # 0 - At least one server connected and no required servers failed (optional server failures logged as warnings) # 1 - Invalid arguments, configuration file issues, no successful connections, or required server failures if [ "$#" -ne 3 ]; then - echo "Usage: $0 GATEWAY_CONFIG_PATH GATEWAY_URL GATEWAY_API_KEY" >&2 + echo "Usage: $0 GATEWAY_CONFIG_PATH GATEWAY_URL GATEWAY_AGENT_ID" >&2 exit 1 fi GATEWAY_CONFIG_PATH="$1" GATEWAY_URL="$2" -GATEWAY_API_KEY="$3" +GATEWAY_AGENT_ID="$3" # Optional comma-separated list of non-critical server names. The gateway output # does not echo the `required` flag from the input configuration, so the caller diff --git a/setup/sh/start_mcp_gateway.sh b/setup/sh/start_mcp_gateway.sh index e5a1fb09..35890eda 100755 --- a/setup/sh/start_mcp_gateway.sh +++ b/setup/sh/start_mcp_gateway.sh @@ -27,7 +27,7 @@ print_timing() { # Required environment variables: # - MCP_GATEWAY_DOCKER_COMMAND: Container image to run (required) -# - MCP_GATEWAY_API_KEY: API key for gateway authentication (required for converter scripts) +# - MCP_GATEWAY_AGENT_ID: agent ID for gateway authentication (required for converter scripts) # Validate that container is specified (command execution is not supported per spec) if [ -z "$MCP_GATEWAY_DOCKER_COMMAND" ]; then @@ -58,7 +58,7 @@ if [ -L /tmp/gh-aw ] || [ -L /tmp/gh-aw/mcp-config ]; then exit 1 fi # Restrict directory permissions so only the runner process owner can read config files -# (which contain bearer tokens and API keys) +# (which contain bearer tokens and agent IDs) chmod 700 /tmp/gh-aw/mcp-config GATEWAY_STDOUT=/tmp/gh-aw/mcp-config/gateway-output.json @@ -77,7 +77,7 @@ print_gateway_startup_diagnostics() { if [ -s "$GATEWAY_STDERR" ]; then sed -E \ -e 's/(Bearer[[:space:]]+)[^[:space:]]+/\1[REDACTED]/Ig' \ - -e 's/((api[_-]?key|token|secret|password|authorization)"?[[:space:]]*[:=][[:space:]]*"?)[^[:space:],}"]+/\1[REDACTED]/Ig' \ + -e 's/((agent[_-]?id|api[_-]?key|token|secret|password|authorization)"?[[:space:]]*[:=][[:space:]]*"?)[^[:space:],}"]+/\1[REDACTED]/Ig' \ "$GATEWAY_STDERR" | bash "$LOG_RENDERER" "Gateway stderr" else echo "Gateway stderr: (empty)" @@ -159,8 +159,8 @@ if ! echo "$MCP_CONFIG" | jq -e '.gateway.domain' >/dev/null 2>&1; then exit 1 fi -if ! echo "$MCP_CONFIG" | jq -e '.gateway.apiKey' >/dev/null 2>&1; then - echo "ERROR: Gateway configuration is missing required 'apiKey' field" +if ! echo "$MCP_CONFIG" | jq -e '.gateway.agentId' >/dev/null 2>&1; then + echo "ERROR: Gateway configuration is missing required 'agentId' field" exit 1 fi @@ -359,7 +359,7 @@ if [ ! -s "$GATEWAY_STDOUT" ]; then exit 1 fi -# Restrict gateway output file permissions - it contains the bearer token / API key +# Restrict gateway output file permissions - it contains the bearer token / agent ID chmod 600 /tmp/gh-aw/mcp-config/gateway-output.json # Check if output contains an error payload instead of valid configuration @@ -376,9 +376,9 @@ echo "Converting gateway configuration to agent format..." CONFIG_CONVERT_START=$(date +%s%3N) export MCP_GATEWAY_OUTPUT=/tmp/gh-aw/mcp-config/gateway-output.json -# Validate MCP_GATEWAY_API_KEY is set (required by converter scripts) -if [ -z "$MCP_GATEWAY_API_KEY" ]; then - echo "ERROR: MCP_GATEWAY_API_KEY environment variable must be set for converter scripts" +# Validate MCP_GATEWAY_AGENT_ID is set (required by converter scripts) +if [ -z "$MCP_GATEWAY_AGENT_ID" ]; then + echo "ERROR: MCP_GATEWAY_AGENT_ID environment variable must be set for converter scripts" echo "This variable should be set in the workflow before calling start_mcp_gateway.sh" exit 1 fi @@ -456,7 +456,7 @@ if [ -f ${RUNNER_TEMP}/gh-aw/actions/check_mcp_servers.sh ]; then if ! bash ${RUNNER_TEMP}/gh-aw/actions/check_mcp_servers.sh \ /tmp/gh-aw/mcp-config/gateway-output.json \ "http://localhost:${MCP_GATEWAY_PORT}" \ - "${MCP_GATEWAY_API_KEY}"; then + "${MCP_GATEWAY_AGENT_ID}"; then echo "ERROR: MCP server checks failed - no servers could be connected" echo "Gateway process will be terminated" kill $GATEWAY_PID 2>/dev/null || true @@ -504,10 +504,10 @@ print_timing $SCRIPT_START_TIME "Overall gateway startup" echo "" # Output PID as GitHub Actions step output for use in cleanup -# Output port and API key for use in stop script (per MCP Gateway Specification v1.1.0) +# Output port and agent ID for use in stop script (per MCP Gateway Specification v1.1.0) { echo "gateway-pid=$GATEWAY_PID" echo "gateway-port=${MCP_GATEWAY_PORT}" - echo "gateway-api-key=${MCP_GATEWAY_API_KEY}" + echo "gateway-agent-id=${MCP_GATEWAY_AGENT_ID}" echo "gateway-domain=${MCP_GATEWAY_DOMAIN}" } >> "$GITHUB_OUTPUT" diff --git a/setup/sh/stop_mcp_gateway.sh b/setup/sh/stop_mcp_gateway.sh index c6827425..18fdc16d 100755 --- a/setup/sh/stop_mcp_gateway.sh +++ b/setup/sh/stop_mcp_gateway.sh @@ -39,11 +39,11 @@ fi # Try graceful shutdown via /close endpoint if gateway variables are available # Per MCP Gateway Specification v1.1.0, the /close endpoint: -# - Requires authentication with API key +# - Requires authentication with agent ID # - Returns 200 OK on success # - Returns 410 Gone if already closed (idempotent) # - Gracefully terminates containers and cleans up resources -if [ -n "$MCP_GATEWAY_PORT" ] && [ -n "$MCP_GATEWAY_API_KEY" ]; then +if [ -n "$MCP_GATEWAY_PORT" ] && [ -n "$MCP_GATEWAY_AGENT_ID" ]; then echo "Attempting graceful shutdown via /close endpoint..." # Use localhost for health check since: @@ -52,8 +52,8 @@ if [ -n "$MCP_GATEWAY_PORT" ] && [ -n "$MCP_GATEWAY_API_KEY" ]; then CLOSE_URL="http://localhost:${MCP_GATEWAY_PORT}/close" # Try to invoke the /close endpoint (with timeout) - # Per spec, the endpoint requires Authorization header with the API key - CLOSE_RESPONSE=$(curl -f -s -m 10 -X POST -H "Authorization: ${MCP_GATEWAY_API_KEY}" "$CLOSE_URL" 2>&1) && { + # Per spec, the endpoint requires Authorization header with the agent ID + CLOSE_RESPONSE=$(curl -f -s -m 10 -X POST -H "Authorization: ${MCP_GATEWAY_AGENT_ID}" "$CLOSE_URL" 2>&1) && { echo "Gateway accepted close request" echo "Response: $CLOSE_RESPONSE" @@ -74,7 +74,7 @@ if [ -n "$MCP_GATEWAY_PORT" ] && [ -n "$MCP_GATEWAY_API_KEY" ]; then echo "Falling back to kill signal..." } else - echo "Gateway environment variables not available (MCP_GATEWAY_PORT or MCP_GATEWAY_API_KEY missing)" + echo "Gateway environment variables not available (MCP_GATEWAY_PORT or MCP_GATEWAY_AGENT_ID missing)" echo "Falling back to kill signal..." fi