From 5edad3da644c6d70ff8dea99cb8d2368175484c7 Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Fri, 7 Aug 2026 22:46:59 +0000 Subject: [PATCH 1/6] Stabilize live plugin validation --- .github/workflows/canary.yml | 10 +- .github/workflows/live-a2a.yml | 12 +- .github/workflows/live-channels.yml | 30 +- .github/workflows/live-external-events.yml | 23 +- .github/workflows/live-voice.yml | 33 +-- .github/workflows/tests.yml | 20 +- scripts/nato-marker.mjs | 65 ++-- tests/ci/npm_with_retry.sh | 19 ++ tests/contract/live-harness.test.ts | 25 +- tests/live/a2a_driver.py | 16 +- tests/live/a2a_preflight.py | 38 +++ tests/live/call-pairing.ts | 12 +- tests/live/email-reply.test.ts | 5 +- tests/live/external-event.test.ts | 6 +- tests/live/helpers.ts | 124 ++++---- tests/live/setup.ts | 23 +- tests/live/sms.test.ts | 17 +- tests/live/test_a2a_preflight.py | 82 ++++++ tests/live/voice.test.ts | 327 ++++++++++++--------- tests/unit/call-pairing.test.ts | 8 +- tests/unit/live-read-retry.test.ts | 23 ++ tests/unit/nato-marker.test.ts | 33 ++- tests/unit/voice-leg-assertions.test.ts | 37 +++ 23 files changed, 625 insertions(+), 363 deletions(-) create mode 100755 tests/ci/npm_with_retry.sh create mode 100644 tests/live/a2a_preflight.py create mode 100644 tests/live/test_a2a_preflight.py create mode 100644 tests/unit/live-read-retry.test.ts create mode 100644 tests/unit/voice-leg-assertions.test.ts diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index b702ee8..7048892 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -23,9 +23,9 @@ jobs: cache: npm - name: Install dependencies from the unpublished SDK source run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@latest @opencode-ai/plugin@latest - run: npm run lint @@ -48,12 +48,12 @@ jobs: cache: npm - name: Install dependencies from the unpublished SDK source run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - - run: npm install -g opencode-ai@latest + - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - run: bash scripts/smoke-loader.sh notify: diff --git a/.github/workflows/live-a2a.yml b/.github/workflows/live-a2a.yml index bd17bb9..aef9f59 100644 --- a/.github/workflows/live-a2a.yml +++ b/.github/workflows/live-a2a.yml @@ -62,11 +62,11 @@ jobs: - name: Install plugin and host run: | - npm ci - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ @inkbox/sdk@0.5.9 \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - npm install -g opencode-ai@latest + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - name: Boot the AUT gateway env: @@ -87,11 +87,11 @@ jobs: INKBOX_BASE_URL: ${{ vars.INKBOX_BASE_URL || 'https://inkbox.ai' }} run: python3 tests/live/a2a_driver.py - - name: Dump gateway logs on failure + - name: Report content-free failure state if: failure() run: | - tail -n 300 "$AUT_GATEWAY_LOG" 2>/dev/null || true - tail -n 100 "$AUT_SERVE_LOG" 2>/dev/null || true + test -s "$AUT_GATEWAY_LOG" && echo "gateway_log=present" || echo "gateway_log=empty" + test -s "$AUT_SERVE_LOG" && echo "serve_log=present" || echo "serve_log=empty" - name: Stop gateway if: always() diff --git a/.github/workflows/live-channels.yml b/.github/workflows/live-channels.yml index f23da1d..5f8f6a1 100644 --- a/.github/workflows/live-channels.yml +++ b/.github/workflows/live-channels.yml @@ -60,12 +60,12 @@ jobs: cache: npm - name: Install dependencies from the unpublished SDK source run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - - run: npm install -g opencode-ai@latest + - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - name: Start mock model if: matrix.mode == 'mock' @@ -77,7 +77,7 @@ jobs: http://127.0.0.1:8088/v1/models >/dev/null && { echo "mock model ready"; exit 0; } sleep 1 done - echo "::error::mock model did not start"; cat "$RUNNER_TEMP/mock.log"; exit 1 + echo "::error::mock model did not start"; exit 1 - name: Boot the AUT gateway (${{ matrix.mode }}) env: @@ -101,14 +101,12 @@ jobs: npx vitest run --config vitest.live.config.ts \ tests/live/email-reply.test.ts tests/live/sms.test.ts - # Failure-only: these logs carry live phone/email content and this - # repo's Action logs are public. - - name: Dump logs (on failure only) + - name: Report content-free failure state if: failure() || cancelled() run: | - echo "=== gateway.log ==="; cat "$AUT_GATEWAY_LOG" || true - echo "=== serve.log ==="; tail -n 100 "$AUT_SERVE_LOG" || true - echo "=== mock.log ==="; cat "$RUNNER_TEMP/mock.log" 2>/dev/null || true + test -s "$AUT_GATEWAY_LOG" && echo "gateway_log=present" || echo "gateway_log=empty" + test -s "$AUT_SERVE_LOG" && echo "serve_log=present" || echo "serve_log=empty" + test -s "$RUNNER_TEMP/mock.log" && echo "mock_log=present" || echo "mock_log=empty" - name: Tear down (always) if: always() @@ -116,15 +114,3 @@ jobs: kill "$AUT_GATEWAY_PID" 2>/dev/null || true kill "$AUT_SERVE_PID" 2>/dev/null || true kill "$(cat "$RUNNER_TEMP/mock.pid" 2>/dev/null)" 2>/dev/null || true - - - name: Upload artifacts (on failure only) - if: failure() || cancelled() - uses: actions/upload-artifact@v7 - with: - name: live-logs-${{ matrix.mode }} - retention-days: 5 - path: | - ${{ env.AUT_GATEWAY_LOG }} - ${{ env.AUT_SERVE_LOG }} - ${{ runner.temp }}/mock.log - if-no-files-found: ignore diff --git a/.github/workflows/live-external-events.yml b/.github/workflows/live-external-events.yml index c60301c..adfd2e8 100644 --- a/.github/workflows/live-external-events.yml +++ b/.github/workflows/live-external-events.yml @@ -49,12 +49,12 @@ jobs: cache: npm - name: Install dependencies from the unpublished SDK source run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - - run: npm install -g opencode-ai@latest + - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - name: Boot the AUT gateway (real model, external events on) env: @@ -79,25 +79,14 @@ jobs: run: | npx vitest run --config vitest.live.config.ts tests/live/external-event.test.ts - - name: Dump logs (on failure only) + - name: Report content-free failure state if: failure() run: | - echo "=== gateway.log ==="; cat "$AUT_GATEWAY_LOG" || true - echo "=== serve.log ==="; tail -n 100 "$AUT_SERVE_LOG" || true + test -s "$AUT_GATEWAY_LOG" && echo "gateway_log=present" || echo "gateway_log=empty" + test -s "$AUT_SERVE_LOG" && echo "serve_log=present" || echo "serve_log=empty" - name: Tear down (always) if: always() run: | kill "$AUT_GATEWAY_PID" 2>/dev/null || true kill "$AUT_SERVE_PID" 2>/dev/null || true - - - name: Upload artifacts (on failure only) - if: failure() - uses: actions/upload-artifact@v7 - with: - name: external-events-logs - retention-days: 5 - path: | - ${{ env.AUT_GATEWAY_LOG }} - ${{ env.AUT_SERVE_LOG }} - if-no-files-found: ignore diff --git a/.github/workflows/live-voice.yml b/.github/workflows/live-voice.yml index 23256f3..a5253e8 100644 --- a/.github/workflows/live-voice.yml +++ b/.github/workflows/live-voice.yml @@ -68,12 +68,12 @@ jobs: cache: npm - name: Install dependencies from the unpublished SDK source run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - - run: npm install -g opencode-ai@latest + - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - name: Boot the AUT gateway (voice; ${{ matrix.scenario }}) env: @@ -99,17 +99,17 @@ jobs: if [ "${{ matrix.scenario }}" = "outbound_hosted" ]; then HOSTED_MARKER="$(node scripts/nato-marker.mjs "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT")" echo "HOSTED_POST_CALL_MARKER=$HOSTED_MARKER" >> "$GITHUB_ENV" - export VOICE_DRIVER_LINE="After we hang up, send me one SMS. Create one post-call action now with the title Send SMS and put this exact five-word SMS body in the action details: $HOSTED_MARKER. Wait for the action tool to succeed, then read all five words back to me. Do not paraphrase, omit a word, or send the SMS during the call." + export VOICE_DRIVER_LINE="After we hang up, send me one SMS with this exact three-word body: $HOSTED_MARKER. Record one post-call SMS action now. Once the action tool succeeds, read the exact three words back to me. Do not send the SMS during the call." export VOICE_DRIVER_LISTEN=180 export VOICE_DRIVER_AUTO_STOP=false fi nohup node tests/live/voice-driver.mjs > "$RUNNER_TEMP/driver.log" 2>&1 & echo $! > "$RUNNER_TEMP/driver.pid" for _ in $(seq 1 30); do # up to ~90s - [ -s "$VOICE_DRIVER_STATE" ] && { echo "driver ready:"; cat "$VOICE_DRIVER_STATE"; exit 0; } + [ -s "$VOICE_DRIVER_STATE" ] && { echo "driver ready"; exit 0; } sleep 3 done - echo "::error::driver did not become ready"; cat "$RUNNER_TEMP/driver.log"; exit 1 + echo "::error::driver did not become ready"; exit 1 - name: Run voice test (${{ matrix.scenario }}) env: @@ -123,13 +123,12 @@ jobs: run: | npx vitest run --config vitest.live.config.ts tests/live/voice.test.ts - # Failure-only: these logs carry live call content and this repo is public. - - name: Dump logs (on failure only) + - name: Report content-free failure state if: failure() || cancelled() run: | - echo "=== gateway.log ==="; cat "$AUT_GATEWAY_LOG" || true - echo "=== serve.log ==="; tail -n 100 "$AUT_SERVE_LOG" || true - echo "=== driver.log ==="; cat "$RUNNER_TEMP/driver.log" || true + test -s "$AUT_GATEWAY_LOG" && echo "gateway_log=present" || echo "gateway_log=empty" + test -s "$AUT_SERVE_LOG" && echo "serve_log=present" || echo "serve_log=empty" + test -s "$RUNNER_TEMP/driver.log" && echo "driver_log=present" || echo "driver_log=empty" - name: Tear down (always) if: always() @@ -138,15 +137,3 @@ jobs: kill "$AUT_GATEWAY_PID" 2>/dev/null || true kill "$AUT_SERVE_PID" 2>/dev/null || true sleep 3 # let the driver revert its number on exit - - - name: Upload artifacts (on failure only) - if: failure() || cancelled() - uses: actions/upload-artifact@v7 - with: - name: voice-logs-${{ matrix.scenario }} - retention-days: 5 - path: | - ${{ env.AUT_GATEWAY_LOG }} - ${{ env.AUT_SERVE_LOG }} - ${{ runner.temp }}/driver.log - if-no-files-found: ignore diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2791c5d..df2e93d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,15 +19,19 @@ jobs: with: node-version: 22 cache: npm + - uses: actions/setup-python@v6 + with: + python-version: "3.12" - run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - run: npm run lint - run: npm run typecheck - run: npm test + - run: python3 -m unittest tests/live/test_a2a_preflight.py - run: npm run build # Same contract suite canary runs on a schedule, but here it gates PRs so a @@ -46,13 +50,13 @@ jobs: node-version: 22 cache: npm - run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - run: | - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@latest @opencode-ai/plugin@latest - run: npm run typecheck @@ -72,10 +76,10 @@ jobs: node-version: 22 cache: npm - run: | - npm ci --prefix .ci/inkbox/sdk/typescript + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" ci --prefix .ci/inkbox/sdk/typescript npm run build --prefix .ci/inkbox/sdk/typescript - npm install --no-save --package-lock=false \ + bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install --no-save --package-lock=false \ ./.ci/inkbox/sdk/typescript \ @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18 - - run: npm install -g opencode-ai@latest + - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest - run: INKBOX_SDK_PATH="$PWD/.ci/inkbox/sdk/typescript" bash scripts/smoke-loader.sh diff --git a/scripts/nato-marker.mjs b/scripts/nato-marker.mjs index f2a6fb0..395c0db 100644 --- a/scripts/nato-marker.mjs +++ b/scripts/nato-marker.mjs @@ -1,44 +1,39 @@ +import { createHash } from "node:crypto"; import { pathToFileURL } from "node:url"; -const RADIO_WORDS = [ - "alpha", - "bravo", - "charlie", - "delta", - "echo", - "foxtrot", - "golf", - "hotel", - "india", - "juliet", - "kilo", - "lima", - "mike", - "november", - "oscar", - "papa", - "quebec", - "romeo", - "sierra", - "tango", - "uniform", - "victor", - "whiskey", - "xray", - "yankee", - "zulu", +const SPEECH_WORDS = [ + "banana", + "elephant", + "pineapple", + "alligator", + "motorcycle", + "umbrella", + "dinosaur", + "potato", + "computer", + "volcano", + "airplane", + "butterfly", + "kangaroo", + "octopus", + "calendar", + "chocolate", + "hospital", + "library", + "sandwich", + "telescope", ]; export function natoMarker(runId, runAttempt) { - let value = BigInt(runId) * 10n + BigInt(runAttempt); - const used = new Set(); + const token = `${runId}-${runAttempt}`; + if (!/^\d+-\d+$/.test(token)) throw new Error("run id and attempt must be numeric"); + let value = BigInt(`0x${createHash("sha256").update(token).digest("hex")}`); + const available = [...SPEECH_WORDS]; const marker = []; - for (let count = 0; count < 5; count += 1) { - let index = Number(value % BigInt(RADIO_WORDS.length)); - value /= BigInt(RADIO_WORDS.length); - while (used.has(index)) index = (index + 1) % RADIO_WORDS.length; - used.add(index); - marker.push(RADIO_WORDS[index]); + for (let count = 0; count < 3; count += 1) { + const index = Number(value % BigInt(available.length)); + value /= BigInt(available.length); + marker.push(available.splice(index, 1)[0]); } return marker.join(" "); } diff --git a/tests/ci/npm_with_retry.sh b/tests/ci/npm_with_retry.sh new file mode 100755 index 0000000..264c720 --- /dev/null +++ b/tests/ci/npm_with_retry.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +attempts=4 +last_status=1 +for attempt in $(seq 1 "$attempts"); do + if npm "$@"; then + exit 0 + else + last_status=$? + fi + if [ "$attempt" -lt "$attempts" ]; then + echo "::warning::npm setup attempt $attempt failed; retrying" + sleep $((attempt * 3)) + fi +done + +echo "::error::npm setup failed after $attempts attempts" +exit "$last_status" diff --git a/tests/contract/live-harness.test.ts b/tests/contract/live-harness.test.ts index 250db81..90d103f 100644 --- a/tests/contract/live-harness.test.ts +++ b/tests/contract/live-harness.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; const liveAut = readFileSync("scripts/live-aut.sh", "utf8"); const liveChannels = readFileSync(".github/workflows/live-channels.yml", "utf8"); const liveVoice = readFileSync(".github/workflows/live-voice.yml", "utf8"); +const liveStack = readFileSync(".github/workflows/live-stack.yml", "utf8"); function shellCommands(source: string): string[] { return source.replace(/\\\n\s*/g, " ").split("\n"); @@ -58,15 +59,29 @@ describe("live harness readiness bounds", () => { it("requires the hosted caller to persist and read back the exact SMS body", () => { expect(liveVoice).toContain( - 'export VOICE_DRIVER_LINE="After we hang up, send me one SMS. Create one post-call action now with the title Send SMS and put this exact five-word SMS body in the action details: $HOSTED_MARKER. Wait for the action tool to succeed, then read all five words back to me. Do not paraphrase, omit a word, or send the SMS during the call."', + 'export VOICE_DRIVER_LINE="After we hang up, send me one SMS with this exact three-word body: $HOSTED_MARKER. Record one post-call SMS action now. Once the action tool succeeds, read the exact three words back to me. Do not send the SMS during the call."', ); }); - it("preserves diagnostics when the voice job is cancelled by its timeout", () => { - expect(liveVoice.match(/if: failure\(\) \|\| cancelled\(\)/g)).toHaveLength(2); + it("keeps failure diagnostics content-free and out of public artifacts", () => { + for (const workflow of [liveChannels, liveVoice]) { + expect(workflow).toContain("Report content-free failure state"); + expect(workflow).not.toContain("actions/upload-artifact"); + expect(workflow).not.toMatch(/cat .*\.log/); + expect(workflow).not.toMatch(/tail .*\.log/); + } }); - it("preserves diagnostics when the live-channel job is cancelled by its timeout", () => { - expect(liveChannels.match(/if: failure\(\) \|\| cancelled\(\)/g)).toHaveLength(2); + it("runs the complete live matrix on PRs without canceling active cycles", () => { + expect(liveStack).toContain("pull_request:"); + expect(liveStack).toContain("github.event_name == 'pull_request'"); + expect(liveStack).toContain("cancel-in-progress: false"); + }); + + it("uses bounded retries only for npm setup", () => { + for (const workflow of [liveChannels, liveVoice]) { + expect(workflow).toContain('tests/ci/npm_with_retry.sh" ci'); + expect(workflow).toContain('tests/ci/npm_with_retry.sh" install'); + } }); }); diff --git a/tests/live/a2a_driver.py b/tests/live/a2a_driver.py index 70ea64a..a7ab40b 100644 --- a/tests/live/a2a_driver.py +++ b/tests/live/a2a_driver.py @@ -9,6 +9,7 @@ from typing import Any from inkbox import Inkbox +from a2a_preflight import enable_and_verify_card STOPPED_WIRE_STATES = { "TASK_STATE_COMPLETED", @@ -315,11 +316,22 @@ def main() -> None: base_url = os.environ.get("INKBOX_BASE_URL", "https://inkbox.ai").rstrip("/") aut = Inkbox(api_key=_required_env("AUT_INKBOX_API_KEY"), base_url=base_url) remote = Inkbox(api_key=_required_env("REMOTE_INKBOX_API_KEY"), base_url=base_url) - _, aut_handle = _identity(aut) + aut_identity, aut_handle = _identity(aut) remote_identity, remote_handle = _identity(remote) a2a = remote_identity.a2a_client() - target = a2a.fetch_card(f"{base_url}/a2a/{aut_handle}/card") + target = enable_and_verify_card( + aut_identity, + a2a, + f"{base_url}/a2a/{aut_handle}/card", + aut_handle, + ) remote_card_url = f"{base_url}/a2a/{remote_handle}/card" + enable_and_verify_card( + remote_identity, + a2a, + remote_card_url, + remote_handle, + ) run = uuid.uuid4().hex[:12] try: if scenario == "inbound-single": diff --git a/tests/live/a2a_preflight.py b/tests/live/a2a_preflight.py new file mode 100644 index 0000000..4147303 --- /dev/null +++ b/tests/live/a2a_preflight.py @@ -0,0 +1,38 @@ +"""Repeatable A2A receiver setup for the live protocol driver.""" + +from __future__ import annotations + +import time +from typing import Any + + +def enable_and_verify_card( + identity: Any, + a2a: Any, + card_url: str, + expected_handle: str, + *, + attempts: int = 6, + delay: float = 2.0, +) -> Any: + """Enable one receiver, then verify its exact card through the protocol.""" + settings = identity.a2a_enable() + if not bool(getattr(settings, "enabled", False)): + raise AssertionError("A2A receiver enablement did not persist") + + for attempt in range(1, attempts + 1): + try: + target = a2a.fetch_card(card_url) + except Exception: + if attempt == attempts: + raise AssertionError( + "A2A card endpoint did not become available after enablement" + ) from None + time.sleep(delay) + continue + if target.card.name != f"@{expected_handle}": + raise AssertionError( + "A2A card identity did not match the configured test identity" + ) + return target + raise AssertionError("A2A card preflight exhausted its bounded attempts") diff --git a/tests/live/call-pairing.ts b/tests/live/call-pairing.ts index df2b215..7096f04 100644 --- a/tests/live/call-pairing.ts +++ b/tests/live/call-pairing.ts @@ -1,6 +1,7 @@ export interface PairableCall { id: string; direction?: string; + status?: string; remotePhoneNumber?: string | null; createdAt?: Date | string | null; created_at?: Date | string | null; @@ -14,11 +15,12 @@ function createdAt(call: PairableCall): number | undefined { } function describe(calls: PairableCall[]) { - return calls.map((call) => ({ - id: call.id, - direction: call.direction, - createdAt: createdAt(call), - })); + return { + count: calls.length, + directions: calls.map((call) => call.direction ?? "unknown"), + statuses: calls.map((call) => call.status ?? "unknown"), + timestampsPresent: calls.map((call) => createdAt(call) !== undefined), + }; } /** Select a call pair only when ownership is unambiguous after duplicate grace. */ diff --git a/tests/live/email-reply.test.ts b/tests/live/email-reply.test.ts index 93b35a3..a1f96cf 100644 --- a/tests/live/email-reply.test.ts +++ b/tests/live/email-reply.test.ts @@ -43,8 +43,7 @@ describe.skipIf(!LIVE)("live email reply", () => { ); const body = `${reply.subject ?? ""}\n${reply.snippet ?? ""}`; assertNotErrorReply(body, "email"); - expect(body).toContain("REPLY_OK"); - expect(body).toContain(tag); + expect(body.includes("REPLY_OK") && body.includes(tag)).toBe(true); }); it("real model: replies with actual content", { @@ -70,6 +69,6 @@ describe.skipIf(!LIVE)("live email reply", () => { ); const body = `${reply.subject ?? ""}\n${reply.snippet ?? ""}`; assertNotErrorReply(body, "email"); - expect(body.toLowerCase()).toContain("confirmed"); + expect(body.toLowerCase().includes("confirmed")).toBe(true); }); }); diff --git a/tests/live/external-event.test.ts b/tests/live/external-event.test.ts index 2d17387..8fc1fdd 100644 --- a/tests/live/external-event.test.ts +++ b/tests/live/external-event.test.ts @@ -58,7 +58,7 @@ describe.skipIf(!LIVE || !REAL_MODEL || !SECRET || !GATEWAY_LOG)("live external const forged = await post(forgedBody, "sha256=deadbeef", forgedRequestId); expect(forged.status).toBe(401); await new Promise((resolve) => setTimeout(resolve, 2_000)); - expect(await gatewayLog()).not.toContain(forgedMarker); + expect((await gatewayLog()).includes(forgedMarker)).toBe(false); const validRequestId = randomBytes(12).toString("hex"); const validBody = workflowRunBody(randomBytes(8).toString("hex")); @@ -67,12 +67,12 @@ describe.skipIf(!LIVE || !REAL_MODEL || !SECRET || !GATEWAY_LOG)("live external .digest("hex")}`; const valid = await post(validBody, signature, validRequestId); expect(valid.status).toBeLessThan(300); - expect(JSON.parse(await valid.text())).toMatchObject({ ok: true }); + expect(JSON.parse(await valid.text()).ok).toBe(true); const marker = `external.turn_completed:github:${validRequestId}`; await pollUntil("external event model turn", async () => (await gatewayLog()).includes(marker) ? true : undefined, ); - expect(await gatewayLog()).toContain(marker); + expect((await gatewayLog()).includes(marker)).toBe(true); }); }); diff --git a/tests/live/helpers.ts b/tests/live/helpers.ts index 1be99a1..fcda899 100644 --- a/tests/live/helpers.ts +++ b/tests/live/helpers.ts @@ -30,6 +30,27 @@ export function client(apiKey: string): Inkbox { return new Inkbox({ apiKey, baseUrl: BASE_URL }); } +export async function retrySafeRead( + read: () => Promise, + options: { attempts?: number; delayMs?: number } = {}, +): Promise { + const attempts = options.attempts ?? 4; + const delayMs = options.delayMs ?? 1_000; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await read(); + } catch { + if (attempt === attempts) break; + await new Promise((resolve) => setTimeout(resolve, attempt * delayMs)); + } + } + throw new Error(`idempotent live API read failed after ${attempts} attempts`); +} + +export async function listCalls(c: Inkbox, limit = 30) { + return retrySafeRead(() => c.calls.list({ limit })); +} + export function nonce(): string { return `smoke-${randomBytes(4).toString("hex")}`; } @@ -65,9 +86,7 @@ export function assertNotErrorReply(body: string, label: string): void { const lower = body.toLowerCase(); const bad = ERROR_MARKERS.filter((m) => lower.includes(m)); if (bad.length > 0) { - throw new Error( - `${label}: reply is an error, not an answer (${bad.join(", ")}): ${body.slice(0, 200)}`, - ); + throw new Error(`${label}: reply is an error, not an answer (${bad.join(", ")})`); } } @@ -123,24 +142,22 @@ export async function inboundTextsFrom( return out; } -// A call's transcript split by who spoke. Read from the DRIVER's client, so -// "remote" segments are the AGENT's speech and "local" are the driver's. +// A call's transcript split by ownership-relative party. export async function callSegments( c: Inkbox, callId: string, -): Promise<{ agent: string[]; driver: string[] }> { +): Promise<{ remote: string[]; local: string[] }> { const segs = (await c.calls.transcripts(callId)) as Array<{ party?: string; text?: string }>; const pick = (party: string) => segs .filter((s) => (s.party ?? "").toLowerCase() === party && (s.text ?? "").trim() !== "") .map((s) => (s.text ?? "").trim()); - return { agent: pick("remote"), driver: pick("local") }; + return { remote: pick("remote"), local: pick("local") }; } -// Block until the transcript shows BOTH parties spoke, then return the agent's -// speech — proof the agent reached the caller out loud on a two-way call. +// Read from the AUT owner: remote is the caller and local is the agent. export async function waitTwoWayCall( - driver: Inkbox, + aut: Inkbox, callId: string, timeoutMs = TIMEOUT_MS, ): Promise { @@ -155,13 +172,13 @@ export async function waitTwoWayCall( return pollUntil( "two-way call transcript", async () => { - const { agent, driver: drv } = await callSegments(driver, callId).catch(() => ({ - agent: [], - driver: [], + const { remote, local } = await callSegments(aut, callId).catch(() => ({ + remote: [], + local: [], })); - if (agent.length > 0 && drv.length > 0) return agent.join(" | "); + if (remote.length > 0 && local.length > 0) return local.join(" | "); - const call = await driver.calls.get(callId).catch(() => undefined); + const call = await aut.calls.get(callId).catch(() => undefined); const status = (call?.status ?? "").toLowerCase(); const detail = () => JSON.stringify({ @@ -185,50 +202,40 @@ export async function waitTwoWayCall( ); } -// (useInkboxTts, useInkboxStt) of the AUT's most recent ANSWERED call in -// `direction` with the driver: (true,true) is Inkbox STT/TTS, (false,false) is -// the realtime path — so each leg can prove the speech path it claims. -export async function autSpeechMode( - aut: Inkbox, - direction: "inbound" | "outbound", - driverNumber: string, - excludedCallIds: Set = new Set(), -): Promise< - | { - id: string; - tts: boolean | null; - stt: boolean | null; - voicemailDetection?: string | null; - } - | undefined -> { - const tail = driverNumber.replace(/\D/g, "").slice(-10); - const calls = (await aut.calls.list({ limit: 10 })) as Array<{ - id: string; - direction?: string; - remotePhoneNumber?: string; - useInkboxTts: boolean | null; - useInkboxStt: boolean | null; - voicemailDetection?: string | null; - }>; - const c = calls.find( - (x) => - (x.direction ?? "").toLowerCase() === direction && - !excludedCallIds.has(x.id) && - (x.remotePhoneNumber ?? "").replace(/\D/g, "").slice(-10) === tail && - x.useInkboxTts !== null, - ); - return c - ? { - id: c.id, - tts: c.useInkboxTts, - stt: c.useInkboxStt, - voicemailDetection: c.voicemailDetection, +// Read from the driver owner: local is the scripted driver speech. +export async function waitDriverLocalSpeech( + driver: Inkbox, + callId: string, + timeoutMs = TIMEOUT_MS, +): Promise { + let completedAt: number | undefined; + const completedGraceMs = 15_000; + return pollUntil( + "driver-local call transcript", + async () => { + const { local } = await callSegments(driver, callId).catch(() => ({ + remote: [], + local: [], + })); + if (local.length > 0) return local.join(" | "); + const call = await driver.calls.get(callId).catch(() => undefined); + const status = String(call?.status ?? "").toLowerCase(); + if (["canceled", "failed"].includes(status)) { + throw new Error("driver call ended before local speech was persisted"); + } + if (status === "completed") { + completedAt ??= Date.now(); + if (Date.now() - completedAt > completedGraceMs) { + throw new Error("driver call completed without persisted local speech"); + } } - : undefined; + return undefined; + }, + timeoutMs, + ); } -// Settle, send an SMS to the AUT, and return the first NEW inbound reply. +// Settle, send an SMS to the AUT, and return the first new matching reply. // Settling first folds any trailing reply to a previous question into // `before`, so it can't be mis-matched as this question's answer. export async function askOverSms( @@ -236,6 +243,7 @@ export async function askOverSms( remotePhoneId: string, autNumber: string, text: string, + accept: (body: string) => boolean, ): Promise { let before = new Set((await inboundTextsFrom(remote, remotePhoneId, autNumber)).map((m) => m.id)); const quietDeadline = Date.now() + 2 * POLL_MS; @@ -250,9 +258,9 @@ export async function askOverSms( await remote.texts.send(remotePhoneId, { to: autNumber, text }); - const reply = await pollUntil(`SMS reply to ${JSON.stringify(text)}`, async () => { + const reply = await pollUntil("current correlated SMS reply", async () => { const inbound = await inboundTextsFrom(remote, remotePhoneId, autNumber); - return inbound.find((m) => !before.has(m.id)); + return inbound.find((m) => !before.has(m.id) && accept(m.text)); }); assertNotErrorReply(reply.text, "sms"); return reply.text; diff --git a/tests/live/setup.ts b/tests/live/setup.ts index a75a865..84d05b7 100644 --- a/tests/live/setup.ts +++ b/tests/live/setup.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll } from "vitest"; -import { AUT_KEY, client, phoneOf } from "./helpers.js"; +import { AUT_KEY, client, listCalls, phoneOf } from "./helpers.js"; const ENDED_CALL_STATUSES = new Set(["completed", "failed", "canceled"]); const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -18,7 +18,7 @@ const statusOf = (call: Call) => (call.status ?? "").toLowerCase(); async function ownedCalls(): Promise> { if (!aut) return new Map(); - const calls = await aut.calls.list({ limit: 100 }); + const calls = await listCalls(aut, 100); return new Map( calls.filter((call) => call.localPhoneNumber === localPhone).map((call) => [call.id, call]), ); @@ -29,13 +29,13 @@ async function hangup(call: Call): Promise { try { await aut.calls.hangup(call.id); return undefined; - } catch (error) { + } catch { try { const current = await aut.calls.get(call.id); if (ENDED_CALL_STATUSES.has(statusOf(current))) return undefined; - return `hangup=${String(error)}; status=${JSON.stringify(statusOf(current))}`; - } catch (getError) { - return `hangup=${String(error)}; get=${String(getError)}`; + return `status=${JSON.stringify(statusOf(current))}`; + } catch { + return "status=unknown"; } } } @@ -59,19 +59,18 @@ async function watchOnce(): Promise { async function finishNewCalls(): Promise { const deadline = Date.now() + 12_000; - const errors = new Map(); + const errors: string[] = []; for (;;) { const live = await newLiveCalls(); if (live.size === 0) return; - for (const [callId, call] of live) { + for (const [, call] of live) { const error = await hangup(call); - if (error) errors.set(callId, error); + if (error) errors.push(error); } if (Date.now() >= deadline) { throw new Error( - `live-test calls remained active after API cleanup: states=${JSON.stringify( - Object.fromEntries([...live].map(([id, call]) => [id, statusOf(call)])), - )} errors=${JSON.stringify(Object.fromEntries(errors))}`, + `live-test calls remained active after cleanup: count=${live.size} ` + + `states=${JSON.stringify([...live.values()].map(statusOf))} errors=${errors.length}`, ); } await delay(500); diff --git a/tests/live/sms.test.ts b/tests/live/sms.test.ts index 59c7c09..067c2ac 100644 --- a/tests/live/sms.test.ts +++ b/tests/live/sms.test.ts @@ -25,9 +25,14 @@ describe.skipIf(!LIVE)("live sms reply", () => { const remotePhone = await phoneOf(remote); const tag = nonce(); - const body = await askOverSms(remote, remotePhone.id, autPhone.number, `ping ${tag}`); - expect(body).toContain("REPLY_OK"); - expect(body).toContain(tag); + const body = await askOverSms( + remote, + remotePhone.id, + autPhone.number, + `ping ${tag}`, + (reply) => reply.includes("REPLY_OK") && reply.includes(tag), + ); + expect(body.includes("REPLY_OK") && body.includes(tag)).toBe(true); }); it("real model: reports its own identity when asked", { @@ -39,13 +44,15 @@ describe.skipIf(!LIVE)("live sms reply", () => { const autPhone = await phoneOf(aut); const remotePhone = await phoneOf(remote); const autEmail = await mailboxOf(aut); + const tag = nonce(); const body = await askOverSms( remote, remotePhone.id, autPhone.number, - "Reply with just your Inkbox email address — short.", + `Reply with your Inkbox email address and this exact reference: ${tag}`, + (reply) => reply.toLowerCase().includes(autEmail.toLowerCase()) && reply.includes(tag), ); - expect(body.toLowerCase()).toContain(autEmail.toLowerCase()); + expect(body.toLowerCase().includes(autEmail.toLowerCase()) && body.includes(tag)).toBe(true); }); }); diff --git a/tests/live/test_a2a_preflight.py b/tests/live/test_a2a_preflight.py new file mode 100644 index 0000000..845c6e6 --- /dev/null +++ b/tests/live/test_a2a_preflight.py @@ -0,0 +1,82 @@ +"""Focused checks for repeatable A2A card preflight.""" + +from types import SimpleNamespace +from unittest import TestCase, mock + +from tests.live.a2a_preflight import enable_and_verify_card + + +class _Identity: + def __init__(self, enabled: bool = True): + self.enabled = enabled + self.enable_calls = 0 + + def a2a_enable(self): + self.enable_calls += 1 + return SimpleNamespace(enabled=self.enabled) + + +class _A2A: + def __init__(self, outcomes): + self.outcomes = list(outcomes) + self.fetch_calls = 0 + + def fetch_card(self, _url): + self.fetch_calls += 1 + outcome = self.outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + +def _target(name: str): + return SimpleNamespace(card=SimpleNamespace(name=name)) + + +class A2APreflightTests(TestCase): + @mock.patch("tests.live.a2a_preflight.time.sleep", return_value=None) + def test_enables_once_and_retries_only_card_reads(self, _sleep): + identity = _Identity() + a2a = _A2A([RuntimeError("not ready"), _target("@test-agent")]) + + target = enable_and_verify_card( + identity, + a2a, + "https://example.test/a2a/test-agent/card", + "test-agent", + attempts=2, + delay=0, + ) + + self.assertEqual(target.card.name, "@test-agent") + self.assertEqual(identity.enable_calls, 1) + self.assertEqual(a2a.fetch_calls, 2) + + def test_rejects_mismatched_identity_without_retry(self): + identity = _Identity() + a2a = _A2A([_target("@different-agent")]) + + with self.assertRaisesRegex(AssertionError, "did not match"): + enable_and_verify_card( + identity, + a2a, + "https://example.test/a2a/test-agent/card", + "test-agent", + ) + + self.assertEqual(identity.enable_calls, 1) + self.assertEqual(a2a.fetch_calls, 1) + + def test_requires_enablement_postcondition(self): + identity = _Identity(enabled=False) + a2a = _A2A([_target("@test-agent")]) + + with self.assertRaisesRegex(AssertionError, "did not persist"): + enable_and_verify_card( + identity, + a2a, + "https://example.test/a2a/test-agent/card", + "test-agent", + ) + + self.assertEqual(a2a.fetch_calls, 0) diff --git a/tests/live/voice.test.ts b/tests/live/voice.test.ts index 94e5d48..8d9170c 100644 --- a/tests/live/voice.test.ts +++ b/tests/live/voice.test.ts @@ -14,14 +14,14 @@ import { describe, expect, it } from "vitest"; import { requireExactCallPair } from "./call-pairing.js"; import { AUT_KEY, - autSpeechMode, callSegments, client, - inboundTextsFrom, LIVE, + listCalls, phoneOf, REAL_MODEL, REMOTE_KEY, + waitDriverLocalSpeech, waitTwoWayCall, } from "./helpers.js"; import { containsVoiceMarker, hasAfterCallSmsIntent, hasSmsIntent } from "./voice-proof.js"; @@ -39,23 +39,15 @@ interface DriverState { } const callSummary = (call: { - id: string; direction: string; status: string; - localPhoneNumber: string | null; - remotePhoneNumber: string; - clientWebsocketUrl: string | null; useInkboxTts: boolean | null; useInkboxStt: boolean | null; hangupReason: string | null; isBlocked: boolean; }) => ({ - id: call.id, direction: call.direction, status: call.status, - localPhoneNumber: call.localPhoneNumber, - remotePhoneNumber: call.remotePhoneNumber, - clientWebsocketUrl: call.clientWebsocketUrl, useInkboxTts: call.useInkboxTts, useInkboxStt: call.useInkboxStt, hangupReason: call.hangupReason, @@ -120,7 +112,7 @@ async function hangupCall( try { await inkbox.calls.hangup(callId); return; - } catch (hangupError) { + } catch { let status = "unknown"; for (let attempt = 0; attempt < 20; attempt += 1) { try { @@ -131,12 +123,91 @@ async function hangupCall( if (["completed", "canceled", "failed"].includes(status)) return; await new Promise((resolve) => setTimeout(resolve, 500)); } - throw new Error( - `failed to hang up live test call ${callId}; status=${JSON.stringify(status)}; error=${String(hangupError)}`, + throw new Error(`failed to hang up live test call; final status=${JSON.stringify(status)}`); + } +} + +const terminalStatuses = new Set(["completed", "canceled", "failed"]); + +async function sweepActiveCalls( + inkbox: ReturnType, + matching: () => Promise, +): Promise { + const active = (await matching()).filter( + (call) => !terminalStatuses.has(String(call.status ?? "").toLowerCase()), + ); + await Promise.allSettled(active.map((call) => hangupCall(inkbox, call.id))); + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const remaining = (await matching()).filter( + (call) => !terminalStatuses.has(String(call.status ?? "").toLowerCase()), ); + if (remaining.length === 0) return; + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error("matching active calls did not settle before the live scenario"); +} + +async function cleanupFreshCalls( + inkbox: ReturnType, + matching: () => Promise, + before: Set, +): Promise { + const fresh = (await matching()).filter((call) => !before.has(call.id)); + const results = await Promise.allSettled(fresh.map((call) => hangupCall(inkbox, call.id))); + if (results.some((result) => result.status === "rejected")) { + throw new Error("one or more fresh live-test calls could not be cleaned up"); } } +async function waitForStableCallPair( + driverLegs: () => Promise, + autLegs: () => Promise, + beforeDriver: Set, + beforeAut: Set, + scenarioStartedAt: number, + deadline: number, +): Promise<{ driver: any; aut: any }> { + let stableKey: string | undefined; + let stableSince = 0; + let lastCounts = { driver: 0, aut: 0 }; + while (Date.now() < deadline) { + const freshDriver = (await driverLegs()).filter( + (call) => !beforeDriver.has(call.id) && (recordCreatedAt(call) ?? -1) >= scenarioStartedAt, + ); + const freshAut = (await autLegs()).filter( + (call) => !beforeAut.has(call.id) && (recordCreatedAt(call) ?? -1) >= scenarioStartedAt, + ); + lastCounts = { driver: freshDriver.length, aut: freshAut.length }; + if (freshDriver.length > 1 || freshAut.length > 1) { + return requireExactCallPair(freshDriver, freshAut, { + scenarioStartedAt, + maxCreationSkewMs: 60_000, + }); + } + if (freshDriver.length === 1 && freshAut.length === 1) { + const pair = requireExactCallPair(freshDriver, freshAut, { + scenarioStartedAt, + maxCreationSkewMs: 60_000, + }); + const key = `${pair.driver.id}:${pair.aut.id}`; + if (key !== stableKey) { + stableKey = key; + stableSince = Date.now(); + } else if (Date.now() - stableSince >= 6_000) { + return pair; + } + } else { + stableKey = undefined; + stableSince = 0; + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + throw new Error( + `timed out waiting for one stable call record per owner; driver_records=${lastCounts.driver} aut_records=${lastCounts.aut}`, + ); +} + describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { it.skipIf(SCENARIO !== "inbound_inkbox")( "inbound: driver calls, agent answers via Inkbox STT/TTS and replies", @@ -146,14 +217,32 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const remote = client(REMOTE_KEY as string); const aut = client(AUT_KEY as string); const autPhone = await phoneOf(aut); - const beforeAutCalls = new Set((await aut.calls.list({ limit: 30 })).map((item) => item.id)); + const autTail = tail(autPhone.number); + const driverTail = tail(st.number); + const driverLegs = async () => + (await listCalls(remote)).filter( + (call) => + String(call.direction ?? "").toLowerCase() === "outbound" && + tail(call.remotePhoneNumber ?? "") === autTail, + ); + const autLegs = async () => + (await listCalls(aut)).filter( + (call) => + String(call.direction ?? "").toLowerCase() === "inbound" && + tail(call.remotePhoneNumber ?? "") === driverTail, + ); // Server-side contact rules run before the plugin or its local allow-all // setting. Whitelisted smoke identities therefore need the driver allowed // explicitly or the call is rejected before either media WS connects. await ensureDriverAllowed(aut, st.number); + await sweepActiveCalls(remote, driverLegs); + await sweepActiveCalls(aut, autLegs); + const beforeDriver = new Set((await driverLegs()).map((call) => call.id)); + const beforeAut = new Set((await autLegs()).map((call) => call.id)); // Place the call to the agent, handing Inkbox the driver's own media WS. + const scenarioStartedAt = Date.now() - 5_000; const call = await remote.calls.place({ toNumber: autPhone.number, fromNumber: st.number, @@ -162,43 +251,35 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { }); console.info(`inbound call placed: ${JSON.stringify(callSummary(call))}`); try { - let agentSaid: string; - try { - agentSaid = await waitTwoWayCall(remote, call.id, VOICE_TIMEOUT_MS); - } catch (error) { - const [driverCall, autCalls, incomingAction, rules] = await Promise.all([ - remote.calls.get(call.id).catch((cause) => ({ error: String(cause) })), - aut.calls.list({ limit: 10 }).catch((cause) => [{ error: String(cause) }]), - aut.incomingCallAction.get().catch((cause) => ({ error: String(cause) })), - aut.phoneIdentityContactRules - .list((await aut.mailboxes.list())[0]?.emailAddress.split("@", 1)[0] ?? "") - .catch((cause) => [{ error: String(cause) }]), - ]); - throw new Error( - `${String(error)}; inbound diagnostics=${JSON.stringify({ - placedCall: callSummary(call), - driverCall, - autCalls, - incomingAction, - rules, - })}`, - ); - } + const pair = await waitForStableCallPair( + driverLegs, + autLegs, + beforeDriver, + beforeAut, + scenarioStartedAt, + Date.now() + VOICE_TIMEOUT_MS, + ); + expect(pair.driver.id === call.id).toBe(true); + const [driverSaid, agentSaid] = await Promise.all([ + waitDriverLocalSpeech(remote, pair.driver.id, VOICE_TIMEOUT_MS), + waitTwoWayCall(aut, pair.aut.id, VOICE_TIMEOUT_MS), + ]); + expect(driverSaid.length).toBeGreaterThan(0); expect(agentSaid.length).toBeGreaterThan(0); - const persistedDriverCall = await remote.calls.get(call.id); + const persistedDriverCall = await remote.calls.get(pair.driver.id); expect(String(persistedDriverCall.voicemailDetection).toLowerCase()).toBe("disabled"); - const mode = await autSpeechMode(aut, "inbound", st.number, beforeAutCalls); - expect(mode, "no answered inbound AUT call with the driver").toBeDefined(); + const mode = await aut.calls.get(pair.aut.id); expect( - mode?.tts && mode?.stt, - `inbound should be Inkbox STT/TTS, got ${JSON.stringify(mode)}`, + mode.useInkboxTts && mode.useInkboxStt, + `inbound should use managed STT/TTS; ${JSON.stringify(callSummary(mode))}`, ).toBe(true); // Voicemail detection applies to the driver's outbound dial and is // proven on persistedDriverCall above. The mirrored AUT row is an // inbound carrier record and does not carry that outbound setting. } finally { - await hangupCall(remote, call.id); + await cleanupFreshCalls(remote, driverLegs, beforeDriver); + await cleanupFreshCalls(aut, autLegs, beforeAut); } }, ); @@ -215,25 +296,22 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const driverTail = tail(st.number); const inboundFromAut = async () => - (await remote.calls.list({ limit: 30 })).filter( + (await listCalls(remote)).filter( (c) => (c.direction ?? "").toLowerCase() === "inbound" && tail(c.remotePhoneNumber ?? "") === autTail, ); - const before = new Set((await inboundFromAut()).map((c) => c.id)); const outboundFromAut = async () => - (await aut.calls.list({ limit: 30 })).filter( + (await listCalls(aut)).filter( (c) => (c.direction ?? "").toLowerCase() === "outbound" && tail(c.remotePhoneNumber ?? "") === driverTail, ); + await sweepActiveCalls(remote, inboundFromAut); + await sweepActiveCalls(aut, outboundFromAut); + const before = new Set((await inboundFromAut()).map((c) => c.id)); const beforeAut = new Set((await outboundFromAut()).map((c) => c.id)); - const beforeTexts = new Set( - (await inboundTextsFrom(remote, st.number_id, autPhone.number)).map( - (message) => message.id, - ), - ); const scenarioStartedAt = Date.now() - 5_000; const deadline = Date.now() + VOICE_TIMEOUT_MS; await remote.texts.send(st.number_id, { @@ -241,53 +319,32 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { text: "Please call me right now by phone and set voicemailDetection to disabled.", }); - let call: Awaited>[number] | undefined; try { - try { - const duplicateGraceMs = 10_000; - let firstPairAt: number | undefined; - while (Date.now() < deadline) { - const driverCalls = (await inboundFromAut()).filter((c) => !before.has(c.id)); - const autCalls = (await outboundFromAut()).filter((c) => !beforeAut.has(c.id)); - if (driverCalls.length > 0 && autCalls.length > 0) { - firstPairAt ??= Date.now(); - if (Date.now() - firstPairAt >= duplicateGraceMs) { - const pair = requireExactCallPair(driverCalls, autCalls, { - scenarioStartedAt, - maxCreationSkewMs: 60_000, - }); - call = pair.driver; - break; - } - } - await new Promise((resolve) => setTimeout(resolve, 2_000)); - } - if (!call) throw new Error("Timed out waiting for one unambiguous Realtime call pair."); - } catch (error) { - const replies = (await inboundTextsFrom(remote, st.number_id, autPhone.number)).filter( - (message) => !beforeTexts.has(message.id), - ); - throw new Error(`${String(error)}; AUT SMS replies=${JSON.stringify(replies)}`); - } - const agentSaid = await waitTwoWayCall(remote, call.id, VOICE_TIMEOUT_MS); - expect(agentSaid.length).toBeGreaterThan(0); - const persistedDriverCall = await remote.calls.get(call.id); - - const freshAutCalls = (await outboundFromAut()).filter((c) => !beforeAut.has(c.id)); - const pair = requireExactCallPair([persistedDriverCall], freshAutCalls, { + const pair = await waitForStableCallPair( + inboundFromAut, + outboundFromAut, + before, + beforeAut, scenarioStartedAt, - maxCreationSkewMs: 60_000, - }); + deadline, + ); + const [driverSaid, agentSaid] = await Promise.all([ + waitDriverLocalSpeech(remote, pair.driver.id, VOICE_TIMEOUT_MS), + waitTwoWayCall(aut, pair.aut.id, VOICE_TIMEOUT_MS), + ]); + expect(driverSaid.length).toBeGreaterThan(0); + expect(agentSaid.length).toBeGreaterThan(0); const mode: any = await aut.calls.get(pair.aut.id); expect( mode.useInkboxTts === false && mode.useInkboxStt === false, - `outbound should be Realtime, got ${JSON.stringify(mode)}`, + `outbound should use Realtime speech; ${JSON.stringify(callSummary(mode))}`, ).toBe(true); // Voicemail detection belongs to the AUT's call-capable outbound request. // The driver's mirrored inbound leg can report its unrelated provider default. expect(String(mode.voicemailDetection).toLowerCase()).toBe("disabled"); } finally { - await hangupCall(remote, call?.id); + await cleanupFreshCalls(remote, inboundFromAut, before); + await cleanupFreshCalls(aut, outboundFromAut, beforeAut); } }, ); @@ -309,34 +366,31 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const autIdentity = await aut.getIdentity(autHandle); const savedAuthority = (await autIdentity.getHostedAgentConfig()).authorityMode; const expectedAuthority = String((savedAuthority as any)?.value ?? savedAuthority); - const deadline = Date.now() + VOICE_TIMEOUT_MS; const progress = { phase: "baseline", last: "" }; const driverLegs = async () => - (await remote.calls.list({ limit: 30 })).filter( + (await listCalls(remote)).filter( (call) => String(call.direction ?? "").toLowerCase() === "inbound" && tail(call.remotePhoneNumber ?? "") === autTail, ); const autLegs = async () => - (await aut.calls.list({ limit: 30 })).filter( + (await listCalls(aut)).filter( (call) => String(call.direction ?? "").toLowerCase() === "outbound" && tail(call.remotePhoneNumber ?? "") === driverTail, ); + await sweepActiveCalls(remote, driverLegs); + await sweepActiveCalls(aut, autLegs); const baselineDriverCalls = await driverLegs(); const baselineAutCalls = await autLegs(); const beforeDriverCalls = new Set(baselineDriverCalls.map((call) => call.id)); const beforeAutCalls = new Set(baselineAutCalls.map((call) => call.id)); const baseline = await outboundTextsTo(aut, autPhone.id, st.number); const beforeSmsIds = new Set(baseline.map((message: any) => message.id)); - const watermark = Math.max( - 0, - ...baseline.map(recordCreatedAt).filter((value): value is number => value !== undefined), - ); - const scenarioStartedAt = Date.now() - 5_000; + const deadline = Date.now() + VOICE_TIMEOUT_MS; await remote.texts.send(st.number_id, { to: autPhone.number, text: @@ -346,35 +400,21 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { `Do not text before calling. Request ref ${Date.now().toString(36)}.`, }); - let driverCallId: string | undefined; let autCallId: string | undefined; try { - const duplicateGraceMs = 10_000; - let firstPairAt: number | undefined; - while (Date.now() < deadline) { - progress.phase = "hosted call placement"; - const freshDriver = (await driverLegs()).filter( - (call) => !beforeDriverCalls.has(call.id), - ); - const freshAut = (await autLegs()).filter((call) => !beforeAutCalls.has(call.id)); - progress.last = `driver_records=${freshDriver.length} aut_records=${freshAut.length}`; - if (freshDriver.length > 0 && freshAut.length > 0) { - firstPairAt ??= Date.now(); - if (Date.now() - firstPairAt >= duplicateGraceMs) { - const pair = requireExactCallPair(freshDriver, freshAut, { - scenarioStartedAt, - maxCreationSkewMs: 60_000, - }); - driverCallId = pair.driver.id; - autCallId = pair.aut.id; - break; - } - } - await new Promise((resolve) => setTimeout(resolve, 5_000)); - } - expect(driverCallId && autCallId, JSON.stringify(progress)).toBeTruthy(); - if (!driverCallId || !autCallId) throw new Error(JSON.stringify(progress)); - const call: any = await aut.calls.get(autCallId); + progress.phase = "hosted call placement"; + const pair = await waitForStableCallPair( + driverLegs, + autLegs, + beforeDriverCalls, + beforeAutCalls, + scenarioStartedAt, + deadline, + ); + const currentDriverCallId = pair.driver.id; + const currentAutCallId = pair.aut.id; + autCallId = currentAutCallId; + const call: any = await aut.calls.get(currentAutCallId); expect(String(call.mode?.value ?? call.mode).toLowerCase()).toBe("hosted_agent"); expect( String(call.voicemailDetection?.value ?? call.voicemailDetection).toLowerCase(), @@ -383,14 +423,17 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { expect(String(call.hostedAgentAuthorityMode?.value ?? call.hostedAgentAuthorityMode)).toBe( expectedAuthority, ); + const agentSaid = await waitTwoWayCall(aut, currentAutCallId, VOICE_TIMEOUT_MS); + expect(agentSaid.length).toBeGreaterThan(0); while (Date.now() < deadline) { progress.phase = "pre-hangup caller and open-action readiness"; - const [segments, currentAutCall] = await Promise.all([ - callSegments(remote, driverCallId).catch(() => ({ agent: [], driver: [] })), - aut.calls.get(autCallId), + const [driverSegments, autSegments, currentAutCall] = await Promise.all([ + callSegments(remote, currentDriverCallId).catch(() => ({ remote: [], local: [] })), + callSegments(aut, currentAutCallId).catch(() => ({ remote: [], local: [] })), + aut.calls.get(currentAutCallId), ]); - const caller = segments.driver + const caller = driverSegments.local .join(" ") .toLowerCase() .replace(/[^a-z0-9]+/g, " "); @@ -401,10 +444,8 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { [item.action, item.details].filter(Boolean).join(" "), ); const callerReady = - segments.agent.length > 0 && - hasAfterCallSmsIntent(caller) && - containsVoiceMarker(caller, HOSTED_MARKER); - const actionReady = actionEvidence.some( + hasAfterCallSmsIntent(caller) && containsVoiceMarker(caller, HOSTED_MARKER); + const matchingActions = actionEvidence.filter( (value: string) => hasSmsIntent(value) && containsVoiceMarker(value, HOSTED_MARKER), ); const smsActionCount = actionEvidence.filter((value: string) => @@ -413,18 +454,27 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const markerActionCount = actionEvidence.filter((value: string) => containsVoiceMarker(value, HOSTED_MARKER), ).length; + const twoWayReady = autSegments.remote.length > 0 && autSegments.local.length > 0; + const actionReady = + openActions.length === 1 && + matchingActions.length === 1 && + smsActionCount === 1 && + markerActionCount === 1; progress.last = - `agent_segments=${segments.agent.length} caller_ready=${callerReady} ` + + `agent_segments=${autSegments.local.length} two_way_ready=${twoWayReady} ` + + `caller_ready=${callerReady} ` + `action_ready=${actionReady} open_actions=${openActions.length} ` + `sms_actions=${smsActionCount} marker_actions=${markerActionCount}`; - if (callerReady && actionReady) break; + if (twoWayReady && callerReady && actionReady) break; await new Promise((resolve) => setTimeout(resolve, 5_000)); } expect(progress.phase).toBe("pre-hangup caller and open-action readiness"); + expect(progress.last).toContain("two_way_ready=true"); expect(progress.last).toContain("caller_ready=true"); expect(progress.last).toContain("action_ready=true"); } finally { - await hangupCall(remote, driverCallId); + await cleanupFreshCalls(remote, driverLegs, beforeDriverCalls); + await cleanupFreshCalls(aut, autLegs, beforeAutCalls); } const duplicateGraceMs = 10_000; @@ -435,7 +485,9 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const fresh = (await outboundTextsTo(aut, autPhone.id, st.number)).filter( (message: any) => { const created = recordCreatedAt(message); - return !beforeSmsIds.has(message.id) && created !== undefined && created >= watermark; + return ( + !beforeSmsIds.has(message.id) && created !== undefined && created >= scenarioStartedAt + ); }, ); matched = fresh.filter((message: any) => @@ -458,14 +510,13 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const afterGrace = (await outboundTextsTo(aut, autPhone.id, st.number)).filter( (message: any) => !beforeSmsIds.has(message.id) && - (recordCreatedAt(message) ?? -1) >= watermark && + (recordCreatedAt(message) ?? -1) >= scenarioStartedAt && containsVoiceMarker(String(message.text ?? ""), HOSTED_MARKER), ); - expect(afterGrace).toHaveLength(1); + expect(afterGrace.length).toBe(1); return; } - if (registryEntry?.state === "failed") - throw new Error(`hosted settlement failed: ${JSON.stringify(registryEntry)}`); + if (registryEntry?.state === "failed") throw new Error("hosted settlement failed"); await new Promise((resolve) => setTimeout(resolve, 5_000)); } throw new Error(`hosted SMS settlement timed out: ${JSON.stringify(progress)}`); diff --git a/tests/unit/call-pairing.test.ts b/tests/unit/call-pairing.test.ts index 5a645a0..4a722fc 100644 --- a/tests/unit/call-pairing.test.ts +++ b/tests/unit/call-pairing.test.ts @@ -19,7 +19,7 @@ describe("live call ownership pairing", () => { ).toMatchObject({ driver: { id: "driver" }, aut: { id: "aut" } }); }); - it("rejects duplicate driver or AUT legs with identifying diagnostics", () => { + it("rejects duplicate driver or AUT legs with sanitized diagnostics", () => { let diagnostic = ""; try { requireExactCallPair([call("driver-1"), call("driver-2")], [call("aut")], { @@ -29,14 +29,16 @@ describe("live call ownership pairing", () => { } catch (error) { diagnostic = String(error); } - expect(diagnostic).toMatch(/driver-1.*driver-2.*aut/); + expect(diagnostic).toContain('"count":2'); + expect(diagnostic).not.toContain("driver-1"); + expect(diagnostic).not.toContain("driver-2"); expect(diagnostic).not.toContain("14155550123"); expect(() => requireExactCallPair([call("driver")], [call("aut-1"), call("aut-2")], { scenarioStartedAt: started, maxCreationSkewMs: 5_000, }), - ).toThrow(/driver.*aut-1.*aut-2/); + ).toThrow(/expected exactly one driver leg and one AUT leg/); }); it("rejects stale and creation-skewed pairs", () => { diff --git a/tests/unit/live-read-retry.test.ts b/tests/unit/live-read-retry.test.ts new file mode 100644 index 0000000..6525ead --- /dev/null +++ b/tests/unit/live-read-retry.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it, vi } from "vitest"; +import { retrySafeRead } from "../live/helpers.js"; + +describe("live idempotent read retry", () => { + it("retries bounded read failures and returns the successful result", async () => { + const read = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("temporary")) + .mockResolvedValue("ready"); + + await expect(retrySafeRead(read, { attempts: 3, delayMs: 0 })).resolves.toBe("ready"); + expect(read).toHaveBeenCalledTimes(2); + }); + + it("does not continue after the bounded attempts", async () => { + const read = vi.fn<() => Promise>().mockRejectedValue(new Error("temporary")); + + await expect(retrySafeRead(read, { attempts: 2, delayMs: 0 })).rejects.toThrow( + "idempotent live API read failed after 2 attempts", + ); + expect(read).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/unit/nato-marker.test.ts b/tests/unit/nato-marker.test.ts index 1bc5db3..0600ad0 100644 --- a/tests/unit/nato-marker.test.ts +++ b/tests/unit/nato-marker.test.ts @@ -1,30 +1,37 @@ -import { execFileSync } from "node:child_process"; import { describe, expect, it } from "vitest"; +// The production marker stays an executable .mjs helper for workflow use. +// @ts-expect-error JavaScript CLI modules do not ship TypeScript declarations. +import { natoMarker } from "../../scripts/nato-marker.mjs"; -const NATO = new Set( - "alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa quebec romeo sierra tango uniform victor whiskey xray yankee zulu".split( +const SPEECH_WORDS = new Set( + "banana elephant pineapple alligator motorcycle umbrella dinosaur potato computer volcano airplane butterfly kangaroo octopus calendar chocolate hospital library sandwich telescope".split( " ", ), ); function marker(runId: string, attempt: string): string[] { - return execFileSync(process.execPath, ["scripts/nato-marker.mjs", runId, attempt], { - encoding: "utf8", - }) - .trim() - .split(" "); + return natoMarker(runId, attempt).split(" "); } -describe("hosted voice NATO marker", () => { +describe("hosted voice marker", () => { it.each([ ["0", "0"], ["1", "1"], ["676", "2"], ["999999999999", "9"], - ])("produces five distinct speech-safe words for run %s attempt %s", (runId, attempt) => { + ])("produces three distinct speech-safe words for run %s attempt %s", (runId, attempt) => { const words = marker(runId, attempt); - expect(words).toHaveLength(5); - expect(new Set(words).size).toBe(5); - expect(words.every((word) => NATO.has(word))).toBe(true); + expect(words).toHaveLength(3); + expect(new Set(words).size).toBe(3); + expect(words.every((word) => SPEECH_WORDS.has(word))).toBe(true); + }); + + it("uses substantially more than trailing run-id digits", () => { + const markers = new Set( + Array.from({ length: 1_000 }, (_, index) => + marker(`3119602${index.toString().padStart(4, "0")}`, "1").join(" "), + ), + ); + expect(markers.size).toBeGreaterThanOrEqual(850); }); }); diff --git a/tests/unit/voice-leg-assertions.test.ts b/tests/unit/voice-leg-assertions.test.ts new file mode 100644 index 0000000..8066ec5 --- /dev/null +++ b/tests/unit/voice-leg-assertions.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it, vi } from "vitest"; +import { waitDriverLocalSpeech, waitTwoWayCall } from "../live/helpers.js"; + +function ownerWithTranscript(segments: Array<{ party: string; text: string }>) { + return { + calls: { + transcripts: vi.fn().mockResolvedValue(segments), + get: vi.fn(), + }, + }; +} + +describe("live voice leg ownership", () => { + it("requires two parties on the AUT owner and returns AUT-local agent speech", async () => { + const aut = ownerWithTranscript([ + { party: "remote", text: "caller request" }, + { party: "local", text: "agent answer" }, + ]); + + await expect(waitTwoWayCall(aut as never, "current-aut-call", 100)).resolves.toBe( + "agent answer", + ); + expect(aut.calls.get).not.toHaveBeenCalled(); + }); + + it("uses only local speech as proof on the driver-owned leg", async () => { + const driver = ownerWithTranscript([ + { party: "local", text: "scripted caller line" }, + { party: "remote", text: "mirrored agent audio" }, + ]); + + await expect(waitDriverLocalSpeech(driver as never, "current-driver-call", 100)).resolves.toBe( + "scripted caller line", + ); + expect(driver.calls.get).not.toHaveBeenCalled(); + }); +}); From 82f5f880ba9683941aaa63908b26cb8d84e9e936 Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Fri, 7 Aug 2026 23:14:10 +0000 Subject: [PATCH 2/6] Restrict hosted post-call tools --- src/gateway/sessions.ts | 16 +++++++++---- tests/gateway/sessions.test.ts | 42 +++++++++++++++++++++++++++++++--- tests/live/voice.test.ts | 6 ++--- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/gateway/sessions.ts b/src/gateway/sessions.ts index bc46928..8277770 100644 --- a/src/gateway/sessions.ts +++ b/src/gateway/sessions.ts @@ -162,14 +162,20 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const agent = turn.agent ?? g.agent; const system = await identitySystem(); let tools: Record | undefined; - if (turn.hostedCapture?.phase === "initial") { - tools = { task: false, inkbox_a2a_call: false }; - } else if (turn.hostedCapture?.phase === "correction") { + if (turn.hostedCapture) { + // Hosted post-call turns expose only the tools required to complete + // communication commitments from the finished call. const listed = await deps.opencode.tool.ids({ query: { directory: deps.directory } }); const ids = (listed as any)?.data ?? listed; - if (!Array.isArray(ids)) throw new Error("Could not restrict the hosted correction turn."); + if (!Array.isArray(ids)) throw new Error("Could not restrict the hosted post-call turn."); tools = Object.fromEntries(ids.map((id) => [String(id), false])); - tools.inkbox_send_sms = true; + if (turn.hostedCapture.phase === "correction") { + tools.inkbox_send_sms = true; + } else { + for (const id of ids.map(String)) { + if (id.startsWith("inkbox_") && !id.includes("_a2a_")) tools[id] = true; + } + } } return { messageID: turn.messageID, diff --git a/tests/gateway/sessions.test.ts b/tests/gateway/sessions.test.ts index b6935f0..e9a7709 100644 --- a/tests/gateway/sessions.test.ts +++ b/tests/gateway/sessions.test.ts @@ -42,7 +42,16 @@ function makeManager(existingDir?: string) { const opencode = { tool: { ids: vi.fn(async () => ({ - data: ["bash", "edit", "task", "inkbox_send_sms", "inkbox_send_email"], + data: [ + "bash", + "edit", + "task", + "inkbox_send_sms", + "inkbox_send_email", + "inkbox_a2a_call", + "inkbox_list_a2a_tasks", + "inkbox_list_a2a_messages", + ], })), }, session: { @@ -386,7 +395,7 @@ describe("capture turns", () => { expect(d.identity.sendText).not.toHaveBeenCalled(); }); - it("keeps hosted initial delegation disabled", async () => { + it("limits hosted initial work to non-A2A Inkbox tools", async () => { const d = makeManager(); prepareHostedCall(d.dir); await d.mgr.runHostedCapture?.("ck", "call", { @@ -395,9 +404,36 @@ describe("capture turns", () => { phase: "initial", expectedTarget: "+14155550123", }); - expect(d.opencode.session.promptAsync.mock.calls[0][0].body.tools).toMatchObject({ + expect(d.opencode.session.promptAsync.mock.calls[0][0].body.tools).toEqual({ + bash: false, + edit: false, + task: false, + inkbox_send_sms: true, + inkbox_send_email: true, + inkbox_a2a_call: false, + inkbox_list_a2a_tasks: false, + inkbox_list_a2a_messages: false, + }); + }); + + it("limits a hosted correction to the SMS tool", async () => { + const d = makeManager(); + prepareHostedCall(d.dir); + await d.mgr.runHostedCapture?.("ck", "call", { + identityId: "ident-1", + callId: "call-1", + phase: "correction", + expectedTarget: "+14155550123", + }); + expect(d.opencode.session.promptAsync.mock.calls[0][0].body.tools).toEqual({ + bash: false, + edit: false, task: false, + inkbox_send_sms: true, + inkbox_send_email: false, inkbox_a2a_call: false, + inkbox_list_a2a_tasks: false, + inkbox_list_a2a_messages: false, }); }); diff --git a/tests/live/voice.test.ts b/tests/live/voice.test.ts index 8d9170c..c879b3d 100644 --- a/tests/live/voice.test.ts +++ b/tests/live/voice.test.ts @@ -242,7 +242,7 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const beforeAut = new Set((await autLegs()).map((call) => call.id)); // Place the call to the agent, handing Inkbox the driver's own media WS. - const scenarioStartedAt = Date.now() - 5_000; + const scenarioStartedAt = Date.now() - 10_000; const call = await remote.calls.place({ toNumber: autPhone.number, fromNumber: st.number, @@ -312,7 +312,7 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { await sweepActiveCalls(aut, outboundFromAut); const before = new Set((await inboundFromAut()).map((c) => c.id)); const beforeAut = new Set((await outboundFromAut()).map((c) => c.id)); - const scenarioStartedAt = Date.now() - 5_000; + const scenarioStartedAt = Date.now() - 10_000; const deadline = Date.now() + VOICE_TIMEOUT_MS; await remote.texts.send(st.number_id, { to: autPhone.number, @@ -389,7 +389,7 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { const beforeAutCalls = new Set(baselineAutCalls.map((call) => call.id)); const baseline = await outboundTextsTo(aut, autPhone.id, st.number); const beforeSmsIds = new Set(baseline.map((message: any) => message.id)); - const scenarioStartedAt = Date.now() - 5_000; + const scenarioStartedAt = Date.now() - 10_000; const deadline = Date.now() + VOICE_TIMEOUT_MS; await remote.texts.send(st.number_id, { to: autPhone.number, From 60b7e1811d9bf064ee89fc883d70592315c044ca Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Fri, 7 Aug 2026 23:39:55 +0000 Subject: [PATCH 3/6] Acknowledge external events after handoff --- src/gateway/dispatch.ts | 6 +++++- tests/gateway/dispatch.test.ts | 32 +++++++++++++++++++++++++++++-- tests/live/external-event.test.ts | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/gateway/dispatch.ts b/src/gateway/dispatch.ts index 111b93a..c81d2a5 100644 --- a/src/gateway/dispatch.ts +++ b/src/gateway/dispatch.ts @@ -50,7 +50,11 @@ export interface DispatchDeps { // may retry); filtered/ignored events return true (ack, no retry). export async function dispatchEvent(deps: DispatchDeps, event: VerifiedEvent): Promise { if (event.provider !== "inkbox") { - if (deps.onExternal) await deps.onExternal(event); + if (deps.onExternal) { + void deps + .onExternal(event) + .catch((error) => deps.logger.error("external.dispatch_failed", { error: String(error) })); + } return true; } const type = event.eventType ?? inferType(event.body); diff --git a/tests/gateway/dispatch.test.ts b/tests/gateway/dispatch.test.ts index 7d528b4..df6c453 100644 --- a/tests/gateway/dispatch.test.ts +++ b/tests/gateway/dispatch.test.ts @@ -435,8 +435,14 @@ describe("dispatchEvent sender agent identity", () => { }); describe("dispatchEvent external providers", () => { - it("hands a non-inkbox event to onExternal and acks", async () => { - const onExternal = vi.fn(async () => {}); + it("hands a non-inkbox event to onExternal and acks without waiting for the turn", async () => { + let finish: () => void = () => {}; + const onExternal = vi.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); const deps = makeDeps({ onExternal }); const external: VerifiedEvent = { provider: "github", @@ -451,6 +457,28 @@ describe("dispatchEvent external providers", () => { expect(ok).toBe(true); expect(onExternal).toHaveBeenCalledWith(external); expect(deps.sessions.handleInbound).not.toHaveBeenCalled(); + finish(); + }); + + it("logs an external turn rejected after acknowledgement", async () => { + const onExternal = vi.fn(async () => { + throw new Error("capture failed"); + }); + const deps = makeDeps({ onExternal }); + const external: VerifiedEvent = { + provider: "github", + verified: true, + eventType: "push", + body: { ref: "refs/heads/main" }, + headers: {}, + }; + + await expect(dispatchEvent(deps, external)).resolves.toBe(true); + await vi.waitFor(() => + expect(deps.logger.error).toHaveBeenCalledWith("external.dispatch_failed", { + error: "Error: capture failed", + }), + ); }); }); diff --git a/tests/live/external-event.test.ts b/tests/live/external-event.test.ts index 8fc1fdd..da3571d 100644 --- a/tests/live/external-event.test.ts +++ b/tests/live/external-event.test.ts @@ -25,7 +25,7 @@ function workflowRunBody(runId: string): string { name: "CI", event: "pull_request", status: "completed", - conclusion: "failure", + conclusion: "success", head_branch: "main", html_url: `https://github.com/${repository}/actions/runs/${runId}`, }, From d4cb1e92aa582283d66dcefe893cda1c8949476e Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Sat, 8 Aug 2026 00:47:31 +0000 Subject: [PATCH 4/6] Give mock completions unique response IDs --- tests/contract/mock-openai.test.ts | 89 ++++++++++++++++++++++++++++++ tests/live/mock-openai.mjs | 11 ++-- 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 tests/contract/mock-openai.test.ts diff --git a/tests/contract/mock-openai.test.ts b/tests/contract/mock-openai.test.ts new file mode 100644 index 0000000..74331c5 --- /dev/null +++ b/tests/contract/mock-openai.test.ts @@ -0,0 +1,89 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { createServer } from "node:net"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +let mock: ChildProcess | undefined; +let baseUrl = ""; + +async function unusedPort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("could not allocate a mock port"); + await new Promise((resolve) => server.close(() => resolve())); + return address.port; +} + +async function waitUntilReady(): Promise { + for (let attempt = 0; attempt < 50; attempt += 1) { + try { + const response = await fetch(`${baseUrl}/v1/models`); + if (response.ok) return; + } catch { + // The child has not bound its socket yet. + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("mock model did not start"); +} + +async function complete(nonce: string, stream: boolean): Promise { + return fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "mock-model", + stream, + messages: [{ role: "user", content: `ping ${nonce}` }], + }), + }); +} + +beforeAll(async () => { + const port = await unusedPort(); + baseUrl = `http://127.0.0.1:${port}`; + mock = spawn(process.execPath, ["tests/live/mock-openai.mjs", String(port)], { + stdio: "ignore", + }); + await waitUntilReady(); +}); + +afterAll(() => { + mock?.kill(); +}); + +describe("live OpenAI mock contract", () => { + it("uses a unique completion id for consecutive responses", async () => { + const first = await complete("smoke-11111111", false); + const second = await complete("smoke-22222222", false); + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); + const firstBody = (await first.json()) as any; + const secondBody = (await second.json()) as any; + expect(firstBody.id).toMatch(/^chatcmpl-/); + expect(secondBody.id).toMatch(/^chatcmpl-/); + expect(firstBody.id).not.toBe(secondBody.id); + expect(firstBody.choices[0].message.content).toContain("smoke-11111111"); + expect(secondBody.choices[0].message.content).toContain("smoke-22222222"); + }); + + it("keeps one unique id across every chunk of a streamed response", async () => { + const first = await complete("smoke-33333333", true); + const second = await complete("smoke-44444444", true); + const chunks = async (response: Response) => + (await response.text()) + .split("\n") + .filter((line) => line.startsWith("data: {") && !line.endsWith("[DONE]")) + .map((line) => JSON.parse(line.slice("data: ".length))); + const firstChunks = await chunks(first); + const secondChunks = await chunks(second); + expect(new Set(firstChunks.map((chunk) => chunk.id)).size).toBe(1); + expect(new Set(secondChunks.map((chunk) => chunk.id)).size).toBe(1); + expect(firstChunks[0].id).not.toBe(secondChunks[0].id); + expect(JSON.stringify(firstChunks)).toContain("smoke-33333333"); + expect(JSON.stringify(secondChunks)).toContain("smoke-44444444"); + }); +}); diff --git a/tests/live/mock-openai.mjs b/tests/live/mock-openai.mjs index 5ba3c74..5c21d41 100644 --- a/tests/live/mock-openai.mjs +++ b/tests/live/mock-openai.mjs @@ -7,6 +7,8 @@ // so a live test can assert the canned content travelled end to end. // // Run: node mock-openai.mjs [port] (default 8088; stdlib only) + +import { randomUUID } from "node:crypto"; import { createServer } from "node:http"; const PORT = Number(process.argv[2] ?? 8088); @@ -23,8 +25,8 @@ function sendJson(res, code, obj) { res.end(body); } -const completion = (model, text) => ({ - id: "chatcmpl-mock", +const completion = (id, model, text) => ({ + id, object: "chat.completion", created: Math.floor(Date.now() / 1000), model, @@ -56,8 +58,9 @@ createServer((req, res) => { } const model = body.model ?? "mock-model"; const text = replyText(body); + const id = `chatcmpl-${randomUUID()}`; - if (!body.stream) return sendJson(res, 200, completion(model, text)); + if (!body.stream) return sendJson(res, 200, completion(id, model, text)); // SSE streaming: one content delta, then the stop chunk, then [DONE]. res.writeHead(200, { @@ -67,7 +70,7 @@ createServer((req, res) => { }); const chunk = (delta, finish = null) => `data: ${JSON.stringify({ - id: "chatcmpl-mock", + id, object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model, From 40303a17c7683777e42d4f2b77e8d48511f2db7c Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Sat, 8 Aug 2026 01:44:17 +0000 Subject: [PATCH 5/6] Reinforce hosted action marker capture --- .github/workflows/live-voice.yml | 2 +- tests/contract/live-harness.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/live-voice.yml b/.github/workflows/live-voice.yml index a5253e8..09a0a41 100644 --- a/.github/workflows/live-voice.yml +++ b/.github/workflows/live-voice.yml @@ -99,7 +99,7 @@ jobs: if [ "${{ matrix.scenario }}" = "outbound_hosted" ]; then HOSTED_MARKER="$(node scripts/nato-marker.mjs "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT")" echo "HOSTED_POST_CALL_MARKER=$HOSTED_MARKER" >> "$GITHUB_ENV" - export VOICE_DRIVER_LINE="After we hang up, send me one SMS with this exact three-word body: $HOSTED_MARKER. Record one post-call SMS action now. Once the action tool succeeds, read the exact three words back to me. Do not send the SMS during the call." + export VOICE_DRIVER_LINE="After we hang up, send me one SMS containing exactly these three words: $HOSTED_MARKER. Create one post-call action now. Set both the action title and the action details to this exact five-word phrase: Send SMS $HOSTED_MARKER. Wait for the action tool to succeed, then read the exact three-word SMS body back to me. Do not paraphrase, omit a word, or send the SMS during the call." export VOICE_DRIVER_LISTEN=180 export VOICE_DRIVER_AUTO_STOP=false fi diff --git a/tests/contract/live-harness.test.ts b/tests/contract/live-harness.test.ts index 90d103f..df78175 100644 --- a/tests/contract/live-harness.test.ts +++ b/tests/contract/live-harness.test.ts @@ -59,7 +59,7 @@ describe("live harness readiness bounds", () => { it("requires the hosted caller to persist and read back the exact SMS body", () => { expect(liveVoice).toContain( - 'export VOICE_DRIVER_LINE="After we hang up, send me one SMS with this exact three-word body: $HOSTED_MARKER. Record one post-call SMS action now. Once the action tool succeeds, read the exact three words back to me. Do not send the SMS during the call."', + 'export VOICE_DRIVER_LINE="After we hang up, send me one SMS containing exactly these three words: $HOSTED_MARKER. Create one post-call action now. Set both the action title and the action details to this exact five-word phrase: Send SMS $HOSTED_MARKER. Wait for the action tool to succeed, then read the exact three-word SMS body back to me. Do not paraphrase, omit a word, or send the SMS during the call."', ); }); From 02de90d496df4b175d290790e9f8264c05f3873e Mon Sep 17 00:00:00 2001 From: dimavrem22 Date: Sat, 8 Aug 2026 05:09:16 +0000 Subject: [PATCH 6/6] Correlate hosted SMS delivery failures --- src/gateway/dispatch.ts | 5 ++ src/gateway/hosted-call-registry.ts | 13 +++++ src/tools/send-sms.ts | 8 ++- tests/gateway/dispatch.test.ts | 76 +++++++++++++++++++++++++++++ tests/live/voice-proof.ts | 8 +++ tests/live/voice.test.ts | 16 ++++-- tests/unit/hosted-send-sms.test.ts | 8 ++- tests/unit/voice-proof.test.ts | 11 +++++ 8 files changed, 140 insertions(+), 5 deletions(-) diff --git a/src/gateway/dispatch.ts b/src/gateway/dispatch.ts index c81d2a5..129a259 100644 --- a/src/gateway/dispatch.ts +++ b/src/gateway/dispatch.ts @@ -7,6 +7,7 @@ import type { ContactResolver } from "./contacts.js"; import { normalizeAddress } from "./contacts.js"; import type { NotifyOnce } from "./dedup.js"; import { deliveryFailureKey, deliveryFailureRecovery } from "./delivery-policy.js"; +import { isSuccessfulHostedSmsMessage } from "./hosted-call-registry.js"; import { downloadMedia, mediaDir } from "./media.js"; import { SILENT } from "./prompts.js"; import type { @@ -425,6 +426,10 @@ async function handleDeliveryFailure( const r = resourceOf(event.body, isText ? "text_message" : "message"); if (str(r?.direction)?.toLowerCase() === "inbound") return true; const messageId = str(r?.id); + if (isText && messageId && isSuccessfulHostedSmsMessage(messageId)) { + deps.logger.info("dispatch.hosted_sms_delivery_failed"); + return true; + } const recipientRows = Array.isArray(r?.recipients) ? r.recipients : []; const failedRecipient = recipientRows .map((item) => record(item)) diff --git a/src/gateway/hosted-call-registry.ts b/src/gateway/hosted-call-registry.ts index ab50e98..ba149f2 100644 --- a/src/gateway/hosted-call-registry.ts +++ b/src/gateway/hosted-call-registry.ts @@ -14,6 +14,7 @@ export interface HostedSmsAttempt { phase: "initial" | "correction"; id: string; messageId?: string; + providerMessageId?: string; target?: string; targetMatches: boolean; state: "pending" | "success" | "failed"; @@ -386,6 +387,7 @@ export function settleHostedSmsAttempt( guard: HostedSmsGuard, state: "success" | "failed", errorKind?: HostedSmsErrorKind, + providerMessageId?: string, ): void { withRegistryMutation((registry) => { const entry = registry[hostedCallKey(guard.identityId, guard.callId)]; @@ -395,10 +397,21 @@ export function settleHostedSmsAttempt( } attempt.state = state; attempt.errorKind = errorKind; + if (state === "success") attempt.providerMessageId = bounded(providerMessageId, 256); entry.updatedAt = Date.now(); }); } +export function isSuccessfulHostedSmsMessage(providerMessageId: string): boolean { + const id = providerMessageId.trim(); + if (!id) return false; + return Object.values(read()).some((entry) => + entry.smsAttempts.some( + (attempt) => attempt.state === "success" && attempt.providerMessageId === id, + ), + ); +} + export function classifyHostedSmsError(error: unknown): HostedSmsErrorKind { const message = (error instanceof Error ? error.message : String(error)).toLowerCase(); if ( diff --git a/src/tools/send-sms.ts b/src/tools/send-sms.ts index 29db112..ac17824 100644 --- a/src/tools/send-sms.ts +++ b/src/tools/send-sms.ts @@ -166,7 +166,13 @@ export function sendSmsTools(deps: ToolDeps): RegisteredTool[] { }; const msg = await identity.sendText(payload); providerAccepted = true; - if (hostedGuard) settleHostedSmsAttempt(hostedGuard, "success"); + if (hostedGuard) { + const providerMessageId = String(msg.id ?? "").trim(); + if (!providerMessageId) { + throw new Error("SMS provider accepted the message without a message id."); + } + settleHostedSmsAttempt(hostedGuard, "success", undefined, providerMessageId); + } const target = formatTargetSummary(msg, args); const status = msg.deliveryStatus ?? "unknown"; return { diff --git a/tests/gateway/dispatch.test.ts b/tests/gateway/dispatch.test.ts index df6c453..f038988 100644 --- a/tests/gateway/dispatch.test.ts +++ b/tests/gateway/dispatch.test.ts @@ -1,12 +1,21 @@ // Event routing: channel selection, sender filtering (self/control/allowlist), // reactions, deduped delivery-failure captures, sender agent identities, // external events, and media. +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { ResolvedConfig } from "../../src/config.js"; import { defaultGatewayConfig } from "../../src/config.js"; import { createNotifyOnce } from "../../src/gateway/dedup.js"; import type { DispatchDeps } from "../../src/gateway/dispatch.js"; import { dispatchEvent } from "../../src/gateway/dispatch.js"; +import { + activateHostedSmsCapture, + beginHostedSmsAttempt, + saveHostedCall, + settleHostedSmsAttempt, +} from "../../src/gateway/hosted-call-registry.js"; import { downloadMedia, mediaDir } from "../../src/gateway/media.js"; import { frameInbound } from "../../src/gateway/prompts.js"; import type { VerifiedEvent } from "../../src/gateway/types.js"; @@ -270,6 +279,73 @@ describe("dispatchEvent reactions", () => { }); describe("dispatchEvent delivery failures", () => { + it("does not start generic recovery for a settled hosted SMS", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "opencode-hosted-delivery-")); + process.env.INKBOX_OPENCODE_HOME = dir; + try { + saveHostedCall({ + identityId: "ident-1", + callId: "call-1", + eventId: "event-1", + state: "running", + event: { + id: "event-1", + event_type: "call.ended", + timestamp: "2026-08-01T00:00:00Z", + data: { call: { id: "call-1", mode: "hosted_agent" } }, + } as never, + }); + activateHostedSmsCapture({ + identityId: "ident-1", + callId: "call-1", + sessionID: "session-1", + phase: "initial", + expectedTarget: "+15551112222", + }); + const guard = beginHostedSmsAttempt({ + sessionID: "session-1", + target: "+15551112222", + hasConversationId: false, + }); + if (!guard) throw new Error("expected hosted SMS guard"); + settleHostedSmsAttempt(guard, "success", undefined, "hosted-message-1"); + saveHostedCall({ + identityId: "ident-1", + callId: "call-1", + eventId: "event-1", + state: "completed", + outcome: "success", + retryable: false, + event: { + id: "event-1", + event_type: "call.ended", + timestamp: "2026-08-01T00:00:00Z", + data: { call: { id: "call-1", mode: "hosted_agent" } }, + } as never, + }); + + const deps = makeDeps(); + const ok = await dispatchEvent( + deps, + event("text.delivery_failed", { + text_message: { + id: "hosted-message-1", + remote_phone_number: "+15551112222", + error_detail: "handset unreachable", + error_code: "undelivered", + }, + }), + ); + + expect(ok).toBe(true); + expect(deps.sessions.runCapture).not.toHaveBeenCalled(); + expect(deps.contacts.resolve).not.toHaveBeenCalled(); + } finally { + delete process.env.INKBOX_OPENCODE_HOME; + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("runs a capture on the first failure and dedupes a repeat with the same id", async () => { const deps = makeDeps(); const failure = event("text.delivery_failed", { diff --git a/tests/live/voice-proof.ts b/tests/live/voice-proof.ts index f3e53fd..42c784e 100644 --- a/tests/live/voice-proof.ts +++ b/tests/live/voice-proof.ts @@ -26,3 +26,11 @@ export function hasSmsIntent(value: string): boolean { /\b(?:text|sms) (?:me|the caller|the user|them|him|her)\b/.test(normalized) ); } + +export function wasAcceptedForDelivery(message: { + deliveryStatus?: unknown; + delivery_status?: unknown; +}): boolean { + const status = String(message.deliveryStatus ?? message.delivery_status ?? "").toLowerCase(); + return status !== "blocked_spam_filter"; +} diff --git a/tests/live/voice.test.ts b/tests/live/voice.test.ts index c879b3d..aba1403 100644 --- a/tests/live/voice.test.ts +++ b/tests/live/voice.test.ts @@ -24,7 +24,12 @@ import { waitDriverLocalSpeech, waitTwoWayCall, } from "./helpers.js"; -import { containsVoiceMarker, hasAfterCallSmsIntent, hasSmsIntent } from "./voice-proof.js"; +import { + containsVoiceMarker, + hasAfterCallSmsIntent, + hasSmsIntent, + wasAcceptedForDelivery, +} from "./voice-proof.js"; const SCENARIO = process.env.VOICE_SCENARIO ?? ""; const STATE_FILE = process.env.VOICE_DRIVER_STATE || "/tmp/voice_driver_state.json"; @@ -490,9 +495,10 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { ); }, ); - matched = fresh.filter((message: any) => + const markerRows = fresh.filter((message: any) => containsVoiceMarker(String(message.text ?? ""), HOSTED_MARKER), ); + matched = markerRows.filter(wasAcceptedForDelivery); try { const registry = JSON.parse( readFileSync( @@ -504,13 +510,17 @@ describe.skipIf(!LIVE || !REAL_MODEL)("live voice", () => { } catch { registryEntry = undefined; } - progress.last = `marker_rows=${matched.length} registry_state=${registryEntry?.state ?? "missing"}`; + progress.last = + `accepted_marker_rows=${matched.length} ` + + `blocked_marker_rows=${markerRows.length - matched.length} ` + + `registry_state=${registryEntry?.state ?? "missing"}`; if (matched.length === 1 && registryEntry?.state === "completed") { await new Promise((resolve) => setTimeout(resolve, duplicateGraceMs)); const afterGrace = (await outboundTextsTo(aut, autPhone.id, st.number)).filter( (message: any) => !beforeSmsIds.has(message.id) && (recordCreatedAt(message) ?? -1) >= scenarioStartedAt && + wasAcceptedForDelivery(message) && containsVoiceMarker(String(message.text ?? ""), HOSTED_MARKER), ); expect(afterGrace.length).toBe(1); diff --git a/tests/unit/hosted-send-sms.test.ts b/tests/unit/hosted-send-sms.test.ts index 6765c0c..8f507ea 100644 --- a/tests/unit/hosted-send-sms.test.ts +++ b/tests/unit/hosted-send-sms.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { activateHostedSmsCapture, getHostedCall, + isSuccessfulHostedSmsMessage, saveHostedCall, } from "../../src/gateway/hosted-call-registry.js"; import { sendSmsTools } from "../../src/tools/send-sms.js"; @@ -88,7 +89,12 @@ describe("hosted send SMS boundary", () => { } as any; await tool.definition.execute({ to: "+14155550123", text: "bravo maple" }, ctx); expect(sendText).toHaveBeenCalledOnce(); - expect(getHostedCall("ident-1", "call-1")?.smsAttempts[0].state).toBe("success"); + expect(getHostedCall("ident-1", "call-1")?.smsAttempts[0]).toMatchObject({ + state: "success", + providerMessageId: "sms-1", + }); + expect(isSuccessfulHostedSmsMessage("sms-1")).toBe(true); + expect(isSuccessfulHostedSmsMessage("sms-other")).toBe(false); }); it("does not rewrite a provider-accepted SMS as failed when success journaling fails", async () => { diff --git a/tests/unit/voice-proof.test.ts b/tests/unit/voice-proof.test.ts index fa43239..f831460 100644 --- a/tests/unit/voice-proof.test.ts +++ b/tests/unit/voice-proof.test.ts @@ -4,6 +4,7 @@ import { hasAfterCallSmsIntent, hasSmsIntent, normalizedVoiceTokens, + wasAcceptedForDelivery, } from "../live/voice-proof.js"; describe("hosted live voice proof normalization", () => { @@ -28,4 +29,14 @@ describe("hosted live voice proof normalization", () => { expect(hasSmsIntent("Send a text message containing the marker after the call.")).toBe(true); expect(hasSmsIntent("Review the text-message history.")).toBe(false); }); + + it("does not count a pre-delivery policy block as an accepted SMS", () => { + expect(wasAcceptedForDelivery({ deliveryStatus: "blocked_spam_filter" })).toBe(false); + expect(wasAcceptedForDelivery({ delivery_status: "blocked_spam_filter" })).toBe(false); + expect(wasAcceptedForDelivery({ deliveryStatus: "queued" })).toBe(true); + expect(wasAcceptedForDelivery({ deliveryStatus: "delivered" })).toBe(true); + expect(wasAcceptedForDelivery({ deliveryStatus: "delivery_failed" })).toBe(true); + expect(wasAcceptedForDelivery({ deliveryStatus: "sending_failed" })).toBe(true); + expect(wasAcceptedForDelivery({})).toBe(true); + }); });