From f54e795b5f7f60ca2823bd8a80e0b3d91a185b62 Mon Sep 17 00:00:00 2001 From: Jay/Fienna Liang Date: Mon, 7 Sep 2026 17:12:14 +0800 Subject: [PATCH] feat(runtime): add hosted external-context composition --- .../integration/core/agent-loop.test.ts | 88 ++++- .../core/heartbeat-runner-credentials.test.ts | 3 +- src/__tests__/integration/tools/tools.test.ts | 164 +++++++++ .../tools/external-context-contracts.test.ts | 81 +++++ src/advanced.ts | 36 +- .../llm/adapters/openai/openai-adapter.ts | 2 + src/core/runtime/loop/README.md | 13 + src/core/runtime/loop/service.ts | 81 +++-- src/core/runtime/loop/types.ts | 5 + src/core/runtime/tools/README.md | 3 + src/core/runtime/tools/service.ts | 13 +- src/core/runtime/tools/types.ts | 1 + .../tools/toolkits/external-context/README.md | 49 +++ .../toolkits/external-context/schemas.ts | 75 ++++ .../toolkits/external-context/view-image.ts | 327 ++++++++++++++---- .../toolkits/external-context/web-search.ts | 98 +++--- src/core/types.ts | 24 +- src/index.ts | 3 + 18 files changed, 911 insertions(+), 155 deletions(-) create mode 100644 src/__tests__/unit/tools/external-context-contracts.test.ts create mode 100644 src/core/tools/toolkits/external-context/README.md create mode 100644 src/core/tools/toolkits/external-context/schemas.ts diff --git a/src/__tests__/integration/core/agent-loop.test.ts b/src/__tests__/integration/core/agent-loop.test.ts index 4547ccd9..7c4e6839 100644 --- a/src/__tests__/integration/core/agent-loop.test.ts +++ b/src/__tests__/integration/core/agent-loop.test.ts @@ -1,11 +1,13 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { AgentLoopCheckpointService, AgentLoopRuntimeService } from '@/core/runtime/loop/index.js'; import { RuntimeToolService } from '@/core/runtime/tools/index.js'; import { ToolBundleComposer, type ToolToolkit } from '@/core/tools/index.js'; import { AgentSkillService, FileAgentSkillActivationRepository } from '@/core/skills/index.js'; +import { ProviderCredentialRepository } from '@/core/auth/index.js'; +import { LlmAdapterService } from '@/core/llm/index.js'; import type { ChatMessage, LlmAdapter, LlmResponse } from '../../../core/llm/types.js'; import type { AgentHeartbeatEvent, AgentLoopEvent, ToolDefinition } from '../../../advanced.js'; import { createLogger } from '../../../core/utils/logger.js'; @@ -19,6 +21,90 @@ import { const silentLogger = createLogger({ level: 'silent', console: false }); describe('AgentLoopRuntimeService.run', () => { + it('constructs an exact host toolkit with the run-scoped OAuth credential', async () => { + const root = await mkdtemp(join(tmpdir(), 'heddle-run-toolkit-credential-')); + const credentialStorePath = join(root, 'auth.json'); + new ProviderCredentialRepository({ storePath: credentialStorePath }).set({ + type: 'oauth', + provider: 'openai', + accessToken: 'stored-access-token', + refreshToken: 'stored-refresh-token', + expiresAt: Date.now() + 120_000, + accountId: 'account-123', + createdAt: '2026-09-07T00:00:00.000Z', + updatedAt: '2026-09-07T00:00:00.000Z', + }); + + let adapterCredential: unknown; + let toolkitCredential: unknown; + let toolkitCredentialSource: unknown; + let modelVisibleTools: string[] = []; + const fakeLlm: LlmAdapter = { + info: { + provider: 'openai', + model: 'gpt-5.4', + capabilities: { + toolCalls: true, + systemMessages: true, + reasoningSummaries: false, + parallelToolCalls: true, + }, + }, + async chat(_messages, tools): Promise { + modelVisibleTools = tools.map((tool) => tool.name); + return { content: 'Done.' }; + }, + }; + const createLlm = vi.spyOn(LlmAdapterService, 'create').mockImplementation((input) => { + adapterCredential = input.credentials?.credential; + return fakeLlm; + }); + const toolkit: ToolToolkit = { + id: 'host-project-context', + createTools(context) { + toolkitCredential = context.credential; + toolkitCredentialSource = context.providerCredentialSource; + return [{ + name: 'host_context_read', + description: 'Read bounded host context.', + parameters: { type: 'object', properties: {} }, + execute: async () => ({ ok: true, output: 'context' }), + }]; + }, + }; + + try { + const result = await AgentLoopRuntimeService.run({ + goal: 'Answer from bounded context.', + model: 'gpt-5.4', + credentialStorePath, + includeDefaultTools: false, + toolkits: [toolkit], + maxSteps: 1, + logger: silentLogger, + workspaceRoot: root, + }); + + expect(result.outcome).toBe('done'); + } finally { + createLlm.mockRestore(); + } + + expect(adapterCredential).toMatchObject({ + type: 'oauth-access-token', + provider: 'openai', + accessToken: 'stored-access-token', + accountId: 'account-123', + }); + expect(toolkitCredential).toBe(adapterCredential); + expect(toolkitCredentialSource).toMatchObject({ + type: 'oauth-access-token', + provider: 'openai', + accountId: 'account-123', + }); + expect(modelVisibleTools).toEqual(['host_context_read']); + }); + it('runs through the public execution loop and emits loop events around trace events', async () => { const workspaceRoot = resolve('/tmp/heddle-loop-test'); const seenMessages: ChatMessage[][] = []; diff --git a/src/__tests__/integration/core/heartbeat-runner-credentials.test.ts b/src/__tests__/integration/core/heartbeat-runner-credentials.test.ts index 9b3e26bf..455f01b1 100644 --- a/src/__tests__/integration/core/heartbeat-runner-credentials.test.ts +++ b/src/__tests__/integration/core/heartbeat-runner-credentials.test.ts @@ -54,13 +54,14 @@ describe('custom heartbeat runner credentials', () => { credentials: { apiKey: undefined, credential: expect.objectContaining({ - type: 'oauth', + type: 'oauth-access-token', provider: 'openai', accessToken: 'stored-access-token', }), credentialStorePath, }, })); + expect(createAdapter.mock.calls[0]?.[0].credentials?.credential).not.toHaveProperty('refreshToken'); const externallyVisibleState = JSON.stringify({ tasks: execution.savedTasks, diff --git a/src/__tests__/integration/tools/tools.test.ts b/src/__tests__/integration/tools/tools.test.ts index eda42842..fbe54d27 100644 --- a/src/__tests__/integration/tools/tools.test.ts +++ b/src/__tests__/integration/tools/tools.test.ts @@ -1,4 +1,5 @@ import { mkdtemp, mkdir, readFile, realpath, symlink, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -777,6 +778,169 @@ describe('viewImageTool', () => { }); }); + it('fails closed when a host does not authorize an opaque image reference', async () => { + const resourceResolver = vi.fn().mockResolvedValue(null); + const tool = createViewImageTool({ resourceResolver }); + + const result = await tool.execute({ reference: 'image-123' }); + + expect(result).toEqual({ + ok: false, + error: 'Image view failed: The host did not authorize or resolve the requested image reference.', + }); + expect(resourceResolver).toHaveBeenCalledWith('image-123', {}); + }); + + it('rejects unsupported host-resolved media before any provider call', async () => { + const tool = createViewImageTool({ + resourceResolver: async () => ({ + bytes: Buffer.from('not-an-image'), + mediaType: 'application/octet-stream', + }), + }); + + const result = await tool.execute({ reference: 'image-123' }); + + expect(result).toEqual({ + ok: false, + error: 'Image view failed: Host-resolved images must use image/png, image/jpeg, image/gif, or image/webp.', + }); + }); + + it('bounds host-resolved image streams before provider invocation', async () => { + const tool = createViewImageTool({ + maxImageBytes: 4, + resourceResolver: async () => ({ + bytes: (async function* () { + yield Buffer.from('123'); + yield Buffer.from('45'); + })(), + mediaType: 'image/png', + }), + }); + + const result = await tool.execute({ reference: 'image-123' }); + + expect(result).toEqual({ + ok: false, + error: 'Image view failed: Image exceeds the configured 4-byte inspection limit.', + }); + }); + + it.each([ + { + label: 'byte count', + metadata: { byteSize: 100 }, + error: 'Image view failed: Host-resolved image byte count does not match its metadata.', + }, + { + label: 'SHA-256', + metadata: { checksumSha256: '0'.repeat(64) }, + error: 'Image view failed: Host-resolved image failed its SHA-256 integrity check.', + }, + ])('rejects mismatched host-resolved $label metadata', async ({ metadata, error }) => { + const tool = createViewImageTool({ + resourceResolver: async () => ({ + bytes: Buffer.from('image-bytes'), + mediaType: 'image/png', + ...metadata, + }), + }); + + await expect(tool.execute({ reference: 'image-123' })).resolves.toEqual({ + ok: false, + error, + }); + }); + + it('honors cancellation before resolving host image content', async () => { + const controller = new AbortController(); + controller.abort(new Error('cancelled by host')); + const resourceResolver = vi.fn(); + const tool = createViewImageTool({ resourceResolver }); + + const result = await tool.execute({ reference: 'image-123' }, { signal: controller.signal }); + + expect(result).toEqual({ + ok: false, + error: 'Image view failed: cancelled by host', + }); + expect(resourceResolver).not.toHaveBeenCalled(); + }); + + it('inspects host-resolved bytes directly with request-scoped credentials', async () => { + const bytes = Buffer.from('host-owned-image-bytes'); + const credential = { + type: 'oauth-access-token', + provider: 'openai', + accessToken: 'request-access-token', + expiresAt: Date.now() + 120_000, + accountId: 'account-123', + } as const; + const resourceResolver = vi.fn(async () => ({ + bytes: (async function* () { + yield bytes.subarray(0, 8); + yield bytes.subarray(8); + })(), + mediaType: 'image/png', + byteSize: bytes.byteLength, + checksumSha256: createHash('sha256').update(bytes).digest('hex'), + })); + const requests: Array<{ headers: Headers; body: string }> = []; + vi.stubGlobal('fetch', vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + requests.push({ + headers: new Headers(init?.headers), + body: String(init?.body ?? ''), + }); + return new Response([ + 'event: response.output_text.done', + 'data: {"type":"response.output_text.done","text":"Stored project image.","content_index":0,"item_id":"msg_1","output_index":0,"sequence_number":1}', + '', + 'event: response.completed', + 'data: {"type":"response.completed","response":{"id":"resp_1","status":"completed","model":"gpt-5.4","output_text":"Stored project image.","output":[]}}', + '', + ].join('\n'), { + status: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + })); + const tool = createViewImageTool({ + model: 'gpt-5.4', + credential, + providerCredentialSource: { + type: 'oauth-access-token', + provider: 'openai', + expiresAt: credential.expiresAt, + accountId: credential.accountId, + }, + resourceResolver, + }); + + const result = await tool.execute({ + reference: 'image-123', + prompt: 'Describe the stored image.', + }); + + expect(result).toEqual({ + ok: true, + output: { + provider: 'openai', + model: 'gpt-5.4', + reference: 'image-123', + summary: 'Stored project image.', + }, + }); + expect(resourceResolver).toHaveBeenCalledWith('image-123', {}); + expect(requests).toHaveLength(1); + expect(requests[0]?.headers.get('authorization')).toBe('Bearer request-access-token'); + const body = JSON.parse(requests[0]?.body ?? '{}') as { + input?: Array<{ content?: Array<{ image_url?: string }> }>; + }; + expect(body.input?.[0]?.content?.[1]?.image_url).toBe( + `data:image/png;base64,${bytes.toString('base64')}`, + ); + }); + it('fails clearly when no OpenAI key is available for the default provider', async () => { const root = await mkdtemp(join(tmpdir(), 'heddle-view-image-')); const imagePath = join(root, 'screen.png'); diff --git a/src/__tests__/unit/tools/external-context-contracts.test.ts b/src/__tests__/unit/tools/external-context-contracts.test.ts new file mode 100644 index 00000000..4814d834 --- /dev/null +++ b/src/__tests__/unit/tools/external-context-contracts.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, expectTypeOf, it } from 'vitest'; +import type { ToolInput, ToolOutput } from '@/core/types.js'; +import { + createWebSearchTool, + type WebSearchToolDefinition, +} from '@/core/tools/toolkits/external-context/web-search.js'; +import { + createViewImageTool, + type ViewImageToolDefinition, +} from '@/core/tools/toolkits/external-context/view-image.js'; +import { + MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH, + MAX_VIEW_IMAGE_INPUTS, + MAX_WEB_SEARCH_CITATIONS, + ViewImageInputSchema, + ViewImageOutputSchema, + WebSearchInputSchema, + WebSearchOutputSchema, + type ViewImageInput, + type ViewImageOutput, + type WebSearchInput, + type WebSearchOutput, +} from '@/core/tools/toolkits/external-context/schemas.js'; + +describe('external-context public contracts', () => { + it('preserves built-in input and output types through ToolDefinition', () => { + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + }); + + it('publishes the exact schemas used by each tool definition', () => { + const search = createWebSearchTool(); + const image = createViewImageTool(); + + expect(search.inputSchema).toBe(WebSearchInputSchema); + expect(search.outputSchema).toBe(WebSearchOutputSchema); + expect(image.inputSchema).toBe(ViewImageInputSchema); + expect(image.outputSchema).toBe(ViewImageOutputSchema); + }); + + it('validates and normalizes bounded web search inputs', () => { + expect(WebSearchInputSchema.parse({ + query: ' current runtime docs ', + contextSize: 'high', + })).toEqual({ + query: 'current runtime docs', + contextSize: 'high', + }); + expect(WebSearchInputSchema.safeParse({ + query: 'x'.repeat(MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH + 1), + }).success).toBe(false); + expect(WebSearchInputSchema.safeParse({ query: 'docs', unexpected: true }).success).toBe(false); + expect(WebSearchOutputSchema.safeParse({ + provider: 'openai', + model: 'gpt-test', + summary: 'summary', + citations: [{ title: 'unsafe', url: 'javascript:alert(1)' }], + }).success).toBe(false); + }); + + it('bounds web search citations and image inputs in the canonical schemas', () => { + const citation = { title: 'Heddle', url: 'https://heddleagent.com' }; + expect(WebSearchOutputSchema.safeParse({ + provider: 'openai', + model: 'gpt-test', + summary: 'summary', + citations: Array.from({ length: MAX_WEB_SEARCH_CITATIONS + 1 }, () => citation), + }).success).toBe(false); + expect(ViewImageInputSchema.safeParse({ + references: Array.from({ length: MAX_VIEW_IMAGE_INPUTS + 1 }, (_, index) => `image-${index}`), + }).success).toBe(false); + expect(ViewImageInputSchema.safeParse({ prompt: 'no image' }).success).toBe(false); + expect(ViewImageOutputSchema.safeParse({ + provider: 'openai', + model: 'gpt-test', + summary: 'summary', + }).success).toBe(false); + }); +}); diff --git a/src/advanced.ts b/src/advanced.ts index fc4ec559..0e63775c 100644 --- a/src/advanced.ts +++ b/src/advanced.ts @@ -111,9 +111,39 @@ export type { MoveFileToolOptions } from './core/tools/toolkits/coding-files/mov export { searchFilesTool, createSearchFilesTool, DEFAULT_SEARCH_EXCLUDED_DIRS } from './core/tools/toolkits/coding-files/search-files.js'; export type { SearchFilesOptions } from './core/tools/toolkits/coding-files/search-files.js'; export { webSearchTool, createWebSearchTool } from './core/tools/toolkits/external-context/web-search.js'; -export type { WebSearchToolOptions } from './core/tools/toolkits/external-context/web-search.js'; -export { viewImageTool, createViewImageTool } from './core/tools/toolkits/external-context/view-image.js'; -export type { ViewImageToolOptions } from './core/tools/toolkits/external-context/view-image.js'; +export type { + WebSearchToolDefinition, + WebSearchToolOptions, +} from './core/tools/toolkits/external-context/web-search.js'; +export { + DEFAULT_MAX_IMAGE_BYTES, + viewImageTool, + createViewImageTool, +} from './core/tools/toolkits/external-context/view-image.js'; +export type { + ViewImageResource, + ViewImageResourceResolver, + ViewImageToolDefinition, + ViewImageToolOptions, +} from './core/tools/toolkits/external-context/view-image.js'; +export { + MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH, + MAX_VIEW_IMAGE_INPUTS, + MAX_WEB_SEARCH_CITATION_TITLE_LENGTH, + MAX_WEB_SEARCH_CITATIONS, + ViewImageInputSchema, + ViewImageOutputSchema, + WebSearchCitationSchema, + WebSearchInputSchema, + WebSearchOutputSchema, +} from './core/tools/toolkits/external-context/schemas.js'; +export type { + ViewImageInput, + ViewImageOutput, + WebSearchCitation, + WebSearchInput, + WebSearchOutput, +} from './core/tools/toolkits/external-context/schemas.js'; export { updatePlanTool } from './core/tools/toolkits/internal/update-plan.js'; export type { PlanItem, PlanItemStatus } from './core/tools/toolkits/internal/update-plan.js'; export { createRunShellInspectTool, createRunShellMutateTool } from './core/tools/toolkits/shell-process/run-shell.js'; diff --git a/src/core/llm/adapters/openai/openai-adapter.ts b/src/core/llm/adapters/openai/openai-adapter.ts index 3f2470fe..8ce6f74c 100644 --- a/src/core/llm/adapters/openai/openai-adapter.ts +++ b/src/core/llm/adapters/openai/openai-adapter.ts @@ -350,6 +350,7 @@ export class OpenAiCodexSseService { oauthFetch: ReturnType | undefined; body: unknown; endpoint?: string; + signal?: AbortSignal; }): Promise { if (!args.oauthFetch) { throw new Error('Missing OAuth fetch implementation for OpenAI Codex request.'); @@ -359,6 +360,7 @@ export class OpenAiCodexSseService { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(args.body), + signal: args.signal, }); if (!response.ok) { diff --git a/src/core/runtime/loop/README.md b/src/core/runtime/loop/README.md index 62da616b..0861d7ae 100644 --- a/src/core/runtime/loop/README.md +++ b/src/core/runtime/loop/README.md @@ -12,6 +12,19 @@ Use `AgentLoopCheckpointService` for state/checkpoint conversion and resume history extraction. Do not put chat sessions, heartbeat scheduling, or host UI logic in this folder. +## Request-scoped toolkits and credentials + +Hosts that need an exact tool surface can pass `toolkits` together with +`includeDefaultTools: false`. The runtime resolves or acquires the provider +credential once, then builds every toolkit with the existing +`ToolToolkitContext`; hosts should not pre-resolve credentials merely to create +provider-backed tools. `credentialStorePath` can be set independently when a +host's runtime state root and credential store are intentionally different. + +Static `tools` remain supported. Toolkit and tool names are duplicate-checked +through the same runtime tool assembly path whether default tools are enabled +or disabled. + ## Tool Concurrency `maxToolConcurrency` bounds parallel-safe tool execution for one run. The diff --git a/src/core/runtime/loop/service.ts b/src/core/runtime/loop/service.ts index 4d4b8c30..94bd926d 100644 --- a/src/core/runtime/loop/service.ts +++ b/src/core/runtime/loop/service.ts @@ -14,6 +14,7 @@ import type { ProviderCredentialSource, ResolvedProviderCredential, } from '../credentials/index.js'; +import { RuntimeCredentialService } from '../credentials/index.js'; import { LlmProviderRuntimeService } from '../provider-runtime/index.js'; import { RuntimeToolService } from '../tools/index.js'; import { AgentLoopCheckpointService } from './checkpoint.js'; @@ -27,8 +28,12 @@ export class AgentLoopRuntimeService { const runId = AgentLoopCheckpointService.resolveRunId(options.runId); const model = options.model ?? options.llm?.info?.model ?? process.env.OPENAI_MODEL ?? process.env.ANTHROPIC_MODEL ?? DEFAULT_OPENAI_MODEL; const workspaceRoot = resolve(options.workspaceRoot ?? process.cwd()); - const credentialStorePath = this.resolveCredentialStorePath({ workspaceRoot, stateDir: options.stateDir }); - const providerRuntime = LlmProviderRuntimeService.resolve({ + const credentialStorePath = this.resolveCredentialStorePath({ + workspaceRoot, + stateDir: options.stateDir, + credentialStorePath: options.credentialStorePath, + }); + const providerRuntime = await this.resolveProviderRuntime({ model, apiKey: options.apiKey, apiKeyProvider: options.apiKeyProvider, @@ -36,6 +41,8 @@ export class AgentLoopRuntimeService { credentialStorePath, preferApiKey: options.preferApiKey, reasoningEffort: options.reasoningEffort, + signal: options.abortSignal, + acquireRequestCredential: !options.llm || Boolean(options.toolkits?.length), }); if (!options.llm) { LlmProviderRuntimeService.assertRunnable(providerRuntime); @@ -177,12 +184,45 @@ export class AgentLoopRuntimeService { private static resolveCredentialStorePath(args: { workspaceRoot: string; stateDir?: string; + credentialStorePath?: string; }): string | undefined { + if (args.credentialStorePath) { + return resolve(args.workspaceRoot, args.credentialStorePath); + } return args.stateDir ? ProviderCredentialRepository.resolveStorePath(resolve(args.workspaceRoot, args.stateDir)) : undefined; } + private static async resolveProviderRuntime( + args: Parameters[0] & { + signal?: AbortSignal; + acquireRequestCredential: boolean; + }, + ): Promise> { + const { signal, acquireRequestCredential, ...input } = args; + const initial = LlmProviderRuntimeService.resolve(input); + if (!acquireRequestCredential || initial.credentialSource.type !== 'oauth') { + return initial; + } + + const credential = await RuntimeCredentialService.acquireRequestScopedCredentialForModel( + input.model, + { + storePath: input.credentialStorePath, + ...(signal ? { signal } : {}), + }, + ); + if (!credential) { + throw new Error(`Heddle could not acquire the stored ${initial.provider} credential for ${input.model}.`); + } + + return LlmProviderRuntimeService.resolve({ + ...input, + credential, + }); + } + private static async createLoopLlmAdapter(options: { model: string; apiKey?: string; @@ -218,27 +258,22 @@ export class AgentLoopRuntimeService { ): ToolDefinition[] { const providedTools = options.tools ?? []; const extraTools = options.extraTools ?? []; - if (options.includeDefaultTools === false) { - return [...providedTools, ...extraTools]; - } - - return [ - ...RuntimeToolService.createDefaultAgentTools({ - model: runtime.model, - apiKey: runtime.apiKey, - credential: runtime.credential, - providerCredentialSource: runtime.providerCredentialSource, - credentialStorePath: runtime.credentialStorePath, - workspaceRoot: runtime.workspaceRoot, - stateDir: options.stateDir, - stateRoot: this.resolveStateRoot(runtime.workspaceRoot, options.stateDir), - memoryDir: options.memoryDir, - searchIgnoreDirs: options.searchIgnoreDirs, - includePlanTool: options.includePlanTool, - }), - ...providedTools, - ...extraTools, - ]; + return RuntimeToolService.createDefaultAgentTools({ + model: runtime.model, + apiKey: runtime.apiKey, + credential: runtime.credential, + providerCredentialSource: runtime.providerCredentialSource, + credentialStorePath: runtime.credentialStorePath, + workspaceRoot: runtime.workspaceRoot, + stateDir: options.stateDir, + stateRoot: this.resolveStateRoot(runtime.workspaceRoot, options.stateDir), + memoryDir: options.memoryDir, + searchIgnoreDirs: options.searchIgnoreDirs, + includePlanTool: options.includePlanTool, + includeDefaultTools: options.includeDefaultTools, + toolkits: options.toolkits, + tools: [...providedTools, ...extraTools], + }); } private static async resolveSystemContext(args: { diff --git a/src/core/runtime/loop/types.ts b/src/core/runtime/loop/types.ts index 3b285d81..04f03fc6 100644 --- a/src/core/runtime/loop/types.ts +++ b/src/core/runtime/loop/types.ts @@ -19,6 +19,7 @@ import type { ChatMessage, LlmAdapter, LlmProvider, LlmUsage, ReasoningEffort } import type { RunFailure, RunResult, StopReason, ToolCall, ToolDefinition, TraceEvent } from '@/core/types.js'; import type { RuntimeProviderCredential } from '@/core/runtime/credentials/index.js'; import type { AgentModelContextRecovery } from '@/core/agent/index.js'; +import type { ToolToolkit } from '@/core/tools/index.js'; export type AgentLoopStatus = 'finished'; @@ -93,6 +94,8 @@ export type RunAgentLoopOptions = { apiKey?: string; apiKeyProvider?: LlmProvider | 'explicit'; credential?: RuntimeProviderCredential; + /** Optional credential store independent from runtime state/checkpoint roots. */ + credentialStorePath?: string; preferApiKey?: boolean; maxSteps?: number; maxToolConcurrency?: number; @@ -105,6 +108,8 @@ export type RunAgentLoopOptions = { resumeFrom?: AgentLoopState | AgentLoopCheckpoint; llm?: LlmAdapter; tools?: ToolDefinition[]; + /** Construct request-scoped tools after Heddle resolves the run credential. */ + toolkits?: ToolToolkit[]; extraTools?: ToolDefinition[]; includeDefaultTools?: boolean; includePlanTool?: boolean; diff --git a/src/core/runtime/tools/README.md b/src/core/runtime/tools/README.md index c58b2585..7e01ef1a 100644 --- a/src/core/runtime/tools/README.md +++ b/src/core/runtime/tools/README.md @@ -15,6 +15,9 @@ the normal agent loop path. Programmatic hosts can also pass additional `ToolToolkit` values. Runtime tools compose host toolkits after the default toolkit list, using the same duplicate-id and duplicate-tool-name checks as built-in toolkits. +`AgentLoopRuntimeService.run(...)` can compose only those request-scoped +toolkits by setting `includeDefaultTools: false`; the toolkit context then +contains the provider credential already resolved for that run. Runtime tools also enforce resolved default-tool visibility policy. For MCP, `hiddenMcpServerIds` hides host-owned servers from the generic `mcp_*` toolkit diff --git a/src/core/runtime/tools/service.ts b/src/core/runtime/tools/service.ts index c1512d71..576f5a81 100644 --- a/src/core/runtime/tools/service.ts +++ b/src/core/runtime/tools/service.ts @@ -28,14 +28,15 @@ export class RuntimeToolService { join(stateRoot, 'memory'); const memoryMode = options.memoryMode ?? 'read-and-record'; + const defaultToolkits = options.includeDefaultTools === false ? [] : this.createDefaultToolkits({ + artifactsEnabled: options.artifactsEnabled ?? true, + includePlanTool: options.includePlanTool, + browserAutomationEnabled: BrowserAutomationCapabilityService.isEnabled({ stateRoot }), + stateRoot, + }); const tools = RuntimeToolService.withHostTools({ defaultTools: ToolBundleComposer.compose({ - toolkits: this.createDefaultToolkits({ - artifactsEnabled: options.artifactsEnabled ?? true, - includePlanTool: options.includePlanTool, - browserAutomationEnabled: BrowserAutomationCapabilityService.isEnabled({ stateRoot }), - stateRoot, - }).concat(options.toolkits ?? []), + toolkits: defaultToolkits.concat(options.toolkits ?? []), context: { workspaceRoot, stateRoot, diff --git a/src/core/runtime/tools/types.ts b/src/core/runtime/tools/types.ts index dd87722a..f66b2055 100644 --- a/src/core/runtime/tools/types.ts +++ b/src/core/runtime/tools/types.ts @@ -28,4 +28,5 @@ export type DefaultAgentToolsOptions = { toolProfile?: RuntimeToolSelectionProfile; searchIgnoreDirs?: string[]; includePlanTool?: boolean; + includeDefaultTools?: boolean; }; diff --git a/src/core/tools/toolkits/external-context/README.md b/src/core/tools/toolkits/external-context/README.md new file mode 100644 index 00000000..48040efd --- /dev/null +++ b/src/core/tools/toolkits/external-context/README.md @@ -0,0 +1,49 @@ +# External-context tools + +This toolkit owns provider-backed reads of information that is outside the +runtime workspace. `web_search` returns a bounded cited summary; +`view_image` describes supported local images or host-authorized image content. +It does not own product authorization, object storage, provenance persistence, +or decisions about which references a user may access. + +## Public contracts + +The advanced entrypoint exports canonical Zod schemas and inferred types for +both tools. The tool definitions expose those same schemas through +`inputSchema` / `outputSchema`, and every execution validates with them. A host +that wraps a tool to persist citations or project results should import these +contracts rather than recreate their shapes. + +`view_image` keeps its local `path` / `paths` input. A hosted product can also +configure `resourceResolver` and expose opaque `reference` / `references` +values to the model: + +```ts +const imageTool = createViewImageTool({ + ...runtimeContext, + resourceResolver: async (reference, { signal }) => { + const authorized = await productStore.readAuthorized(reference, signal); + return authorized && { + bytes: authorized.body, + mediaType: authorized.mediaType, + byteSize: authorized.byteSize, + checksumSha256: authorized.checksumSha256, + }; + }, +}); +``` + +The resolver is the authorization boundary. A missing result is treated as +unavailable or unauthorized without widening access. Heddle validates the +media type, reads no more than `maxImageBytes` (20 MiB by default), honors the +run cancellation signal, checks optional byte-count/SHA-256 metadata, and sends +the in-memory content to the selected provider without a temporary file. + +## Request-scoped credentials + +Do not resolve provider credentials in an adopter toolkit. Pass that toolkit to +`AgentLoopRuntimeService.run({ toolkits: [...] })`; Heddle acquires one +request-scoped credential and invokes `createTools(context)` with the same +resolved context used for the run. This works with `includeDefaultTools: false` +for exact, bounded tool sets. + diff --git a/src/core/tools/toolkits/external-context/schemas.ts b/src/core/tools/toolkits/external-context/schemas.ts new file mode 100644 index 00000000..c440f90c --- /dev/null +++ b/src/core/tools/toolkits/external-context/schemas.ts @@ -0,0 +1,75 @@ +import { z } from 'zod'; + +export const MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH = 2_000; +export const MAX_WEB_SEARCH_CITATIONS = 20; +export const MAX_WEB_SEARCH_CITATION_TITLE_LENGTH = 1_000; +export const MAX_VIEW_IMAGE_INPUTS = 10; + +const ExternalContextLocatorSchema = z.string().trim().min(1); + +export const WebSearchInputSchema = z.object({ + query: z.string().trim().min(1).max(MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH), + contextSize: z.enum(['low', 'medium', 'high']).optional(), +}).strict(); + +export const WebSearchCitationSchema = z.object({ + title: z.string().trim().min(1).max(MAX_WEB_SEARCH_CITATION_TITLE_LENGTH), + url: z.httpUrl(), +}).strict(); + +export const WebSearchOutputSchema = z.object({ + provider: z.enum(['openai', 'anthropic']), + model: z.string().trim().min(1), + summary: z.string(), + citations: z.array(WebSearchCitationSchema).max(MAX_WEB_SEARCH_CITATIONS), +}).strict(); + +export const ViewImageInputSchema = z.object({ + path: ExternalContextLocatorSchema.optional(), + paths: z.array(ExternalContextLocatorSchema).min(1).max(MAX_VIEW_IMAGE_INPUTS).optional(), + reference: ExternalContextLocatorSchema.optional(), + references: z.array(ExternalContextLocatorSchema).min(1).max(MAX_VIEW_IMAGE_INPUTS).optional(), + prompt: z.string().trim().max(MAX_EXTERNAL_CONTEXT_PROMPT_LENGTH).optional(), +}).strict().superRefine((input, context) => { + const inputCount = [ + ...(input.path ? [input.path] : []), + ...(input.paths ?? []), + ...(input.reference ? [input.reference] : []), + ...(input.references ?? []), + ].length; + if (inputCount === 0) { + context.addIssue({ + code: 'custom', + message: 'Provide at least one image path or reference.', + }); + } + if (inputCount > MAX_VIEW_IMAGE_INPUTS) { + context.addIssue({ + code: 'custom', + message: `Provide at most ${MAX_VIEW_IMAGE_INPUTS} images.`, + }); + } +}); + +export const ViewImageOutputSchema = z.object({ + provider: z.enum(['openai', 'anthropic']), + model: z.string().trim().min(1), + path: ExternalContextLocatorSchema.optional(), + paths: z.array(ExternalContextLocatorSchema).min(1).max(MAX_VIEW_IMAGE_INPUTS).optional(), + reference: ExternalContextLocatorSchema.optional(), + references: z.array(ExternalContextLocatorSchema).min(1).max(MAX_VIEW_IMAGE_INPUTS).optional(), + summary: z.string(), +}).strict().superRefine((output, context) => { + if (!output.path && !output.paths && !output.reference && !output.references) { + context.addIssue({ + code: 'custom', + message: 'Image inspection output must identify at least one path or reference.', + }); + } +}); + +export type WebSearchInput = z.infer; +export type WebSearchCitation = z.infer; +export type WebSearchOutput = z.infer; +export type ViewImageInput = z.infer; +export type ViewImageOutput = z.infer; diff --git a/src/core/tools/toolkits/external-context/view-image.ts b/src/core/tools/toolkits/external-context/view-image.ts index 71152f58..9432b29d 100644 --- a/src/core/tools/toolkits/external-context/view-image.ts +++ b/src/core/tools/toolkits/external-context/view-image.ts @@ -3,13 +3,18 @@ // Host-side image viewing MVP backed by the active model provider. // --------------------------------------------------------------------------- -import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; import { extname, resolve } from 'node:path'; import Anthropic from '@anthropic-ai/sdk'; import type { ImageBlockParam } from '@anthropic-ai/sdk/resources/messages/messages'; import OpenAI from 'openai'; import type { ResponseInputImage, ResponseInputText } from 'openai/resources/responses/responses.js'; -import type { ToolDefinition, ToolResult } from '../../../types.js'; +import type { + ToolDefinition, + ToolExecutionContext, + ToolResult, +} from '../../../types.js'; import { LlmAdapterService } from '../../../llm/index.js'; import { OpenAiCodexSseService, @@ -23,19 +28,33 @@ import { type ProviderCredentialSource, type ResolvedProviderCredential, } from '../../../runtime/credentials/index.js'; - -type ViewImageInput = { - path?: string; - paths?: string[]; - prompt?: string; -}; +import { + ViewImageInputSchema, + ViewImageOutputSchema, + type ViewImageInput, + type ViewImageOutput, +} from './schemas.js'; type ImageViewFile = { - path: string; + source: + | { type: 'path'; value: string } + | { type: 'reference'; value: string }; mediaType: string; data: Buffer; }; +export type ViewImageResource = { + bytes: Uint8Array | AsyncIterable; + mediaType: string; + byteSize?: number; + checksumSha256?: string; +}; + +export type ViewImageResourceResolver = ( + reference: string, + context: ToolExecutionContext, +) => Promise; + export type ViewImageToolOptions = { model?: string; provider?: LlmProvider; @@ -44,19 +63,27 @@ export type ViewImageToolOptions = { providerCredentialSource?: ProviderCredentialSource; credentialStorePath?: string; workspaceRoot?: string; + resourceResolver?: ViewImageResourceResolver; + maxImageBytes?: number; }; const DEFAULT_IMAGE_PROMPT = 'Describe the image for a coding assistant. Focus on UI text, error messages, filenames, commands, code, diagrams, and any details relevant to software work.'; -const MAX_IMAGE_VIEW_FILES = 10; +export const DEFAULT_MAX_IMAGE_BYTES = 20 * 1024 * 1024; + +export type ViewImageToolDefinition = ToolDefinition; -export const viewImageTool: ToolDefinition = createViewImageTool(); +export const viewImageTool: ViewImageToolDefinition = createViewImageTool(); -export function createViewImageTool(options: ViewImageToolOptions = {}): ToolDefinition { +export function createViewImageTool(options: ViewImageToolOptions = {}): ViewImageToolDefinition { + const maxImageBytes = resolveMaxImageBytes(options.maxImageBytes); + const supportsReferences = Boolean(options.resourceResolver); return { name: 'view_image', description: - 'Inspect one or more local image files when the user references screenshots, diagrams, or other visual file paths and the image contents are actually needed. Use this only after the user has provided or implied concrete image paths. Input examples: { "path": "/absolute/path/to/screenshot.png" } or { "paths": ["/absolute/path/to/a.png", "/absolute/path/to/b.png"] }. Optional field: prompt for a more specific visual question. Returns a concise text description of the image contents.', + supportsReferences ? + 'Inspect one or more local image paths or host-authorized opaque image references when visual contents are needed. Input examples: { "path": "/absolute/path/to/screenshot.png" } or { "reference": "host-image-reference" }. Optional field: prompt for a more specific visual question. Returns a concise text description.' + : 'Inspect one or more local image files when the user references screenshots, diagrams, or other visual file paths and the image contents are actually needed. Use this only after the user has provided or implied concrete image paths. Input examples: { "path": "/absolute/path/to/screenshot.png" } or { "paths": ["/absolute/path/to/a.png", "/absolute/path/to/b.png"] }. Optional field: prompt for a more specific visual question. Returns a concise text description of the image contents.', parameters: { type: 'object', additionalProperties: false, @@ -70,62 +97,78 @@ export function createViewImageTool(options: ViewImageToolOptions = {}): ToolDef items: { type: 'string' }, description: 'Paths to local image files.', }, + ...(supportsReferences ? { + reference: { + type: 'string', + description: 'Opaque host-authorized image reference.', + }, + references: { + type: 'array', + items: { type: 'string' }, + description: 'Opaque host-authorized image references.', + }, + } : {}), prompt: { type: 'string', description: 'Optional focused instruction for what to extract from the image.', }, }, + anyOf: [ + { required: ['path'] }, + { required: ['paths'] }, + ...(supportsReferences ? [ + { required: ['reference'] }, + { required: ['references'] }, + ] : []), + ], }, - async execute(raw: unknown): Promise { - if (!isViewImageInput(raw)) { + inputSchema: ViewImageInputSchema, + outputSchema: ViewImageOutputSchema, + async execute(raw: unknown, context?: ToolExecutionContext): Promise> { + const parsed = ViewImageInputSchema.safeParse(raw); + if (!parsed.success) { return { ok: false, - error: 'Invalid input for view_image. Required field: path or paths. Optional field: prompt.', + error: supportsReferences ? + 'Invalid input for view_image. Required field: path, paths, reference, or references. Optional field: prompt.' + : 'Invalid input for view_image. Required field: path or paths. Optional field: prompt.', }; } - const input = raw as ViewImageInput; + const input = parsed.data; const workspaceRoot = options.workspaceRoot ?? process.cwd(); const paths = normalizeImagePaths(input); - if (paths.length > MAX_IMAGE_VIEW_FILES) { + if (paths.some((path) => !detectMediaType(resolve(workspaceRoot, path)))) { return { ok: false, - error: `view_image supports at most ${MAX_IMAGE_VIEW_FILES} images per call.`, + error: 'view_image supports .png, .jpg, .jpeg, .gif, and .webp files.', }; } - - const provider = options.provider ?? LlmAdapterService.inferProvider(options.model ?? DEFAULT_OPENAI_MODEL); - const prompt = input.prompt?.trim() || DEFAULT_IMAGE_PROMPT; - const fileInputs = paths.flatMap((path) => { - const filePath = resolve(workspaceRoot, path); - const mediaType = detectMediaType(filePath); - if (!mediaType) { - return []; - } - - return [{ filePath, mediaType }]; - }); - if (fileInputs.length !== paths.length) { + const references = normalizeImageReferences(input); + if (references.length > 0 && !options.resourceResolver) { return { ok: false, - error: 'view_image supports .png, .jpg, .jpeg, .gif, and .webp files.', + error: 'view_image cannot resolve opaque references because no host resource resolver is configured.', }; } + const provider = options.provider ?? LlmAdapterService.inferProvider(options.model ?? DEFAULT_OPENAI_MODEL); + const prompt = input.prompt || DEFAULT_IMAGE_PROMPT; + try { - const files = await Promise.all(fileInputs.map(async (file) => { - return { - path: file.filePath, - mediaType: file.mediaType, - data: await readFile(file.filePath), - } satisfies ImageViewFile; - })); + const files = await resolveImageViewFiles({ + input, + workspaceRoot, + resourceResolver: options.resourceResolver, + maxImageBytes, + context: context ?? {}, + }); switch (provider) { case 'openai': - return await executeOpenAiImageView({ files, prompt, options }); + return await executeOpenAiImageView({ files, prompt, options, signal: context?.signal }); case 'anthropic': - return await executeAnthropicImageView({ files, prompt, options }); + return await executeAnthropicImageView({ files, prompt, options, signal: context?.signal }); case 'google': return { ok: false, @@ -159,7 +202,8 @@ async function executeOpenAiImageView(args: { files: ImageViewFile[]; prompt: string; options: ViewImageToolOptions; -}): Promise { + signal?: AbortSignal; +}): Promise> { const model = args.options.model ?? DEFAULT_OPENAI_MODEL; const oauthCredential = OpenAiOAuthFetchService.isAccountCredential(args.options.credential) ? args.options.credential @@ -222,6 +266,7 @@ async function executeOpenAiImageView(args: { model: candidateModel, prompt: args.prompt, imageUrls: inputImages.map((image) => image.imageUrl), + signal: args.signal, }) : await client.responses.create({ model: candidateModel, @@ -236,16 +281,16 @@ async function executeOpenAiImageView(args: { } satisfies ResponseInputImage)), ], }], - }); + }, { signal: args.signal }); return { ok: true, - output: { + output: ViewImageOutputSchema.parse({ provider: 'openai', model: response.model, - ...formatImageOutputPaths(args.files), + ...formatImageOutputSources(args.files), summary: response.output_text?.trim() || 'No image description returned.', - }, + }), }; } catch (error) { lastError = error; @@ -270,6 +315,7 @@ async function executeOpenAiOAuthImageStream(args: { model: string; prompt: string; imageUrls: string[]; + signal?: AbortSignal; }): Promise<{ model: string; output_text?: string }> { if (!args.oauthFetch) { throw new Error('Missing OAuth fetch implementation for OpenAI image inspection.'); @@ -296,6 +342,7 @@ async function executeOpenAiOAuthImageStream(args: { ], }], }, + signal: args.signal, }); const outputText = OpenAiCodexSseService.extractOutputText(text); @@ -309,7 +356,8 @@ async function executeAnthropicImageView(args: { files: ImageViewFile[]; prompt: string; options: ViewImageToolOptions; -}): Promise { + signal?: AbortSignal; +}): Promise> { const apiKey = firstDefinedNonEmpty(args.options.apiKey, process.env.ANTHROPIC_API_KEY, process.env.PERSONAL_ANTHROPIC_API_KEY); if (!apiKey) { return { @@ -355,55 +403,182 @@ async function executeAnthropicImageView(args: { }, ], }], - }); + }, { signal: args.signal }); return { ok: true, - output: { + output: ViewImageOutputSchema.parse({ provider: 'anthropic', model: response.model, - ...formatImageOutputPaths(args.files), + ...formatImageOutputSources(args.files), summary: response.content .flatMap((block) => (block.type === 'text' ? [block.text] : [])) .join('\n') .trim() || 'No image description returned.', - }, + }), }; } -function isViewImageInput(raw: unknown): raw is ViewImageInput { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { - return false; +function normalizeImagePaths(input: ViewImageInput): string[] { + return [ + ...(typeof input.path === 'string' ? [input.path] : []), + ...(input.paths ?? []), + ].map((path) => path.trim()); +} + +function normalizeImageReferences(input: ViewImageInput): string[] { + return [ + ...(typeof input.reference === 'string' ? [input.reference] : []), + ...(input.references ?? []), + ].map((reference) => reference.trim()); +} + +async function resolveImageViewFiles(args: { + input: ViewImageInput; + workspaceRoot: string; + resourceResolver?: ViewImageResourceResolver; + maxImageBytes: number; + context: ToolExecutionContext; +}): Promise { + const pathFiles = normalizeImagePaths(args.input).map(async (path): Promise => { + const filePath = resolve(args.workspaceRoot, path); + const mediaType = detectMediaType(filePath); + if (!mediaType) { + throw new Error('Unsupported local image media type.'); + } + + args.context.signal?.throwIfAborted(); + const metadata = await stat(filePath); + if (!metadata.isFile()) { + throw new Error('The requested image path is not a regular file.'); + } + assertImageByteSize(metadata.size, args.maxImageBytes); + const data = await readFile( + filePath, + args.context.signal ? { signal: args.context.signal } : undefined, + ); + assertImageByteSize(data.byteLength, args.maxImageBytes); + return { + source: { type: 'path', value: filePath }, + mediaType, + data, + }; + }); + const referenceFiles = normalizeImageReferences(args.input).map(async (reference): Promise => { + if (!args.resourceResolver) { + throw new Error('No host image resource resolver is configured.'); + } + + args.context.signal?.throwIfAborted(); + const resource = await args.resourceResolver(reference, args.context); + args.context.signal?.throwIfAborted(); + if (!resource) { + throw new Error('The host did not authorize or resolve the requested image reference.'); + } + + const mediaType = normalizeImageMediaType(resource.mediaType); + if (!mediaType) { + throw new Error('Host-resolved images must use image/png, image/jpeg, image/gif, or image/webp.'); + } + if (resource.byteSize !== undefined) { + assertExpectedImageByteSize(resource.byteSize); + assertImageByteSize(resource.byteSize, args.maxImageBytes); + } + if (resource.checksumSha256 !== undefined && !/^[a-f\d]{64}$/i.test(resource.checksumSha256)) { + throw new Error('Host-resolved image SHA-256 metadata must contain 64 hexadecimal characters.'); + } + + const data = await readImageResourceBytes({ + bytes: resource.bytes, + maxImageBytes: args.maxImageBytes, + signal: args.context.signal, + }); + if (resource.byteSize !== undefined && data.byteLength !== resource.byteSize) { + throw new Error('Host-resolved image byte count does not match its metadata.'); + } + if ( + resource.checksumSha256 !== undefined + && createHash('sha256').update(data).digest('hex') !== resource.checksumSha256.toLowerCase() + ) { + throw new Error('Host-resolved image failed its SHA-256 integrity check.'); + } + + return { + source: { type: 'reference', value: reference }, + mediaType, + data, + }; + }); + + return await Promise.all([...pathFiles, ...referenceFiles]); +} + +async function readImageResourceBytes(args: { + bytes: Uint8Array | AsyncIterable; + maxImageBytes: number; + signal?: AbortSignal; +}): Promise { + if (args.bytes instanceof Uint8Array) { + args.signal?.throwIfAborted(); + assertImageByteSize(args.bytes.byteLength, args.maxImageBytes); + return Buffer.from(args.bytes); + } + if (!isAsyncByteIterable(args.bytes)) { + throw new Error('Host-resolved image bytes must be a Uint8Array or async iterable of Uint8Array chunks.'); } - const input = raw as Record; - const keys = Object.keys(input); - if (keys.some((key) => key !== 'path' && key !== 'paths' && key !== 'prompt')) { - return false; + const chunks: Buffer[] = []; + let byteSize = 0; + for await (const chunk of args.bytes) { + args.signal?.throwIfAborted(); + if (!(chunk instanceof Uint8Array)) { + throw new Error('Host-resolved image streams must yield Uint8Array chunks.'); + } + byteSize += chunk.byteLength; + assertImageByteSize(byteSize, args.maxImageBytes); + chunks.push(Buffer.from(chunk)); } + args.signal?.throwIfAborted(); + return Buffer.concat(chunks, byteSize); +} - const hasPath = typeof input.path === 'string' && input.path.trim().length > 0; - const hasPaths = Array.isArray(input.paths) - && input.paths.length > 0 - && input.paths.every((path) => typeof path === 'string' && path.trim().length > 0); - if (!hasPath && !hasPaths) { - return false; +function isAsyncByteIterable(value: unknown): value is AsyncIterable { + return Boolean( + value + && typeof value === 'object' + && Symbol.asyncIterator in value + && typeof (value as AsyncIterable)[Symbol.asyncIterator] === 'function', + ); +} + +function assertExpectedImageByteSize(byteSize: number): void { + if (!Number.isSafeInteger(byteSize) || byteSize < 0) { + throw new Error('Host-resolved image byte-size metadata must be a non-negative safe integer.'); } +} - return input.prompt === undefined || typeof input.prompt === 'string'; +function assertImageByteSize(byteSize: number, maxImageBytes: number): void { + if (byteSize > maxImageBytes) { + throw new Error(`Image exceeds the configured ${maxImageBytes}-byte inspection limit.`); + } } -function normalizeImagePaths(input: ViewImageInput): string[] { - return [ - ...(typeof input.path === 'string' ? [input.path] : []), - ...(input.paths ?? []), - ].map((path) => path.trim()); +function resolveMaxImageBytes(value: number | undefined): number { + const maxImageBytes = value ?? DEFAULT_MAX_IMAGE_BYTES; + if (!Number.isSafeInteger(maxImageBytes) || maxImageBytes <= 0) { + throw new RangeError('view_image maxImageBytes must be a positive safe integer.'); + } + return maxImageBytes; } -function formatImageOutputPaths(files: ImageViewFile[]) { - const paths = files.map((file) => file.path); - return paths.length === 1 ? { path: paths[0] } : { paths }; +function formatImageOutputSources(files: ImageViewFile[]) { + const paths = files.flatMap((file) => file.source.type === 'path' ? [file.source.value] : []); + const references = files.flatMap((file) => file.source.type === 'reference' ? [file.source.value] : []); + return { + ...(paths.length === 1 ? { path: paths[0] } : paths.length > 1 ? { paths } : {}), + ...(references.length === 1 ? { reference: references[0] } : references.length > 1 ? { references } : {}), + }; } function detectMediaType(filePath: string): string | undefined { @@ -422,6 +597,10 @@ function detectMediaType(filePath: string): string | undefined { } } +function normalizeImageMediaType(mediaType: string): 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | undefined { + return toAnthropicMediaType(mediaType.trim().toLowerCase()); +} + function toAnthropicMediaType(mediaType: string): 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | undefined { if ( mediaType === 'image/jpeg' diff --git a/src/core/tools/toolkits/external-context/web-search.ts b/src/core/tools/toolkits/external-context/web-search.ts index 9d1cbeb9..ab8ff8cd 100644 --- a/src/core/tools/toolkits/external-context/web-search.ts +++ b/src/core/tools/toolkits/external-context/web-search.ts @@ -12,7 +12,7 @@ import type { } from '@anthropic-ai/sdk/resources/messages/messages'; import OpenAI from 'openai'; import type { Response, ResponseOutputText, WebSearchTool } from 'openai/resources/responses/responses.js'; -import type { ToolDefinition, ToolResult } from '../../../types.js'; +import type { ToolDefinition, ToolExecutionContext, ToolResult } from '../../../types.js'; import { LlmAdapterService } from '../../../llm/index.js'; import { OpenAiCodexSseService, @@ -26,11 +26,15 @@ import { type ProviderCredentialSource, type ResolvedProviderCredential, } from '../../../runtime/credentials/index.js'; - -type WebSearchInput = { - query: string; - contextSize?: 'low' | 'medium' | 'high'; -}; +import { + MAX_WEB_SEARCH_CITATIONS, + WebSearchCitationSchema, + WebSearchInputSchema, + WebSearchOutputSchema, + type WebSearchCitation, + type WebSearchInput, + type WebSearchOutput, +} from './schemas.js'; export type WebSearchToolOptions = { model?: string; @@ -41,9 +45,11 @@ export type WebSearchToolOptions = { credentialStorePath?: string; }; -export const webSearchTool: ToolDefinition = createWebSearchTool(); +export type WebSearchToolDefinition = ToolDefinition; -export function createWebSearchTool(options: WebSearchToolOptions = {}): ToolDefinition { +export const webSearchTool: WebSearchToolDefinition = createWebSearchTool(); + +export function createWebSearchTool(options: WebSearchToolOptions = {}): WebSearchToolDefinition { return { name: 'web_search', description: @@ -64,23 +70,26 @@ export function createWebSearchTool(options: WebSearchToolOptions = {}): ToolDef }, required: ['query'], }, - async execute(raw: unknown): Promise { - if (!isWebSearchInput(raw)) { + inputSchema: WebSearchInputSchema, + outputSchema: WebSearchOutputSchema, + async execute(raw: unknown, context?: ToolExecutionContext): Promise> { + const parsed = WebSearchInputSchema.safeParse(raw); + if (!parsed.success) { return { ok: false, error: 'Invalid input for web_search. Required field: query. Optional field: contextSize ("low", "medium", or "high").', }; } - const input = raw as WebSearchInput; + const input = parsed.data; const provider = options.provider ?? LlmAdapterService.inferProvider(options.model ?? DEFAULT_OPENAI_MODEL); try { switch (provider) { case 'openai': - return await executeOpenAiWebSearch(input, options); + return await executeOpenAiWebSearch(input, options, context?.signal); case 'anthropic': - return await executeAnthropicWebSearch(input, options); + return await executeAnthropicWebSearch(input, options, context?.signal); case 'google': return { ok: false, @@ -110,7 +119,11 @@ export function createWebSearchTool(options: WebSearchToolOptions = {}): ToolDef }; } -async function executeOpenAiWebSearch(input: WebSearchInput, options: WebSearchToolOptions): Promise { +async function executeOpenAiWebSearch( + input: WebSearchInput, + options: WebSearchToolOptions, + signal?: AbortSignal, +): Promise> { const model = options.model ?? process.env.OPENAI_WEB_SEARCH_MODEL ?? DEFAULT_OPENAI_MODEL; const oauthCredential = OpenAiOAuthFetchService.isAccountCredential(options.credential) ? options.credential @@ -143,7 +156,7 @@ async function executeOpenAiWebSearch(input: WebSearchInput, options: WebSearchT } if (oauthCredential) { - return await executeOpenAiOAuthWebSearch(input, { ...options, model }, oauthCredential); + return await executeOpenAiOAuthWebSearch(input, { ...options, model }, oauthCredential, signal); } const apiKey = firstDefinedNonEmpty(options.apiKey, process.env.OPENAI_API_KEY, process.env.PERSONAL_OPENAI_API_KEY); @@ -162,11 +175,11 @@ async function executeOpenAiWebSearch(input: WebSearchInput, options: WebSearchT type: 'web_search', search_context_size: input.contextSize ?? 'medium', } satisfies WebSearchTool], - }); + }, { signal }); return { ok: true, - output: formatOpenAiWebSearchResult(response), + output: WebSearchOutputSchema.parse(formatOpenAiWebSearchResult(response)), }; } @@ -174,7 +187,8 @@ async function executeOpenAiOAuthWebSearch( input: WebSearchInput, options: WebSearchToolOptions & { model: string }, oauthCredential: Parameters[0], -): Promise { + signal?: AbortSignal, +): Promise> { const oauthFetch = OpenAiOAuthFetchService.create(oauthCredential, { storePath: options.credentialStorePath }); const sseText = await OpenAiCodexSseService.execute({ oauthFetch, @@ -191,14 +205,19 @@ async function executeOpenAiOAuthWebSearch( search_context_size: input.contextSize ?? 'medium', }], }, + signal, }); return { ok: true, - output: formatOpenAiOAuthWebSearchSseResult(sseText, options.model), + output: WebSearchOutputSchema.parse(formatOpenAiOAuthWebSearchSseResult(sseText, options.model)), }; } -async function executeAnthropicWebSearch(input: WebSearchInput, options: WebSearchToolOptions): Promise { +async function executeAnthropicWebSearch( + input: WebSearchInput, + options: WebSearchToolOptions, + signal?: AbortSignal, +): Promise> { const apiKey = firstDefinedNonEmpty(options.apiKey, process.env.ANTHROPIC_API_KEY, process.env.PERSONAL_ANTHROPIC_API_KEY); if (!apiKey) { return { @@ -223,11 +242,11 @@ async function executeAnthropicWebSearch(input: WebSearchInput, options: WebSear role: 'user', content: `Search the web for the following query and answer concisely with citations when available:\n\n${input.query}`, }], - }); + }, { signal }); return { ok: true, - output: formatAnthropicWebSearchResult(response), + output: WebSearchOutputSchema.parse(formatAnthropicWebSearchResult(response)), }; } @@ -235,24 +254,6 @@ function firstDefinedNonEmpty(...values: Array): string | un return values.find((value) => typeof value === 'string' && value.trim().length > 0); } -function isWebSearchInput(raw: unknown): raw is WebSearchInput { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { - return false; - } - - const input = raw as Record; - const keys = Object.keys(input); - if (keys.some((key) => key !== 'query' && key !== 'contextSize')) { - return false; - } - - if (typeof input.query !== 'string' || input.query.trim().length === 0) { - return false; - } - - return input.contextSize === undefined || input.contextSize === 'low' || input.contextSize === 'medium' || input.contextSize === 'high'; -} - function formatOpenAiWebSearchResult(response: Response): { provider: 'openai'; model: string; @@ -266,7 +267,7 @@ function formatOpenAiWebSearchResult(response: Response): { provider: 'openai', model: response.model, summary, - citations, + citations: normalizeWebSearchCitations(citations), }; } @@ -316,7 +317,7 @@ function formatOpenAiOAuthWebSearchSseResult(sseText: string, model: string): { provider: 'openai', model, summary: OpenAiCodexSseService.extractOutputText(sseText).trim() || 'No summary returned.', - citations: extractSseWebSearchSources(sseText), + citations: normalizeWebSearchCitations(extractSseWebSearchSources(sseText)), }; } @@ -366,10 +367,21 @@ function formatAnthropicWebSearchResult(response: Message): { provider: 'anthropic', model: response.model, summary, - citations, + citations: normalizeWebSearchCitations(citations), }; } +function normalizeWebSearchCitations( + citations: Array<{ title: string; url: string }>, +): WebSearchCitation[] { + return citations + .flatMap((citation) => { + const parsed = WebSearchCitationSchema.safeParse(citation); + return parsed.success ? [parsed.data] : []; + }) + .slice(0, MAX_WEB_SEARCH_CITATIONS); +} + function extractAnthropicUrlCitations(citationsInput: TextCitation[]): Array<{ title: string; url: string }> { const citations: Array<{ title: string; url: string }> = []; const seen = new Set(); diff --git a/src/core/types.ts b/src/core/types.ts index fe7ca460..93dce2d9 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -30,7 +30,13 @@ export type ToolExecutionContext = { signal?: AbortSignal; }; -export type ToolDefinition = { +export type ToolValidationSchema = { + safeParse(input: unknown): + | { success: true; data: Value } + | { success: false; error: unknown }; +}; + +export type ToolDefinition = { name: string; description: string; requiresApproval?: boolean; @@ -48,13 +54,23 @@ export type ToolDefinition = { */ concurrency?: ToolConcurrencyMode; parameters: Record; // JSON Schema object + /** Canonical host-side validation schema for the model-provided input. */ + inputSchema?: ToolValidationSchema; + /** Canonical host-side validation schema for successful output. */ + outputSchema?: ToolValidationSchema; /** Immutable execution provenance owned by the host, never by the model. */ hostPolicy?: ToolPolicyHostContext; /** Resolve host provenance for broker tools whose authority is input-selected. */ resolveHostPolicy?: (input: unknown) => ToolPolicyHostContext | undefined; - execute: (input: unknown, context?: ToolExecutionContext) => Promise; + execute: (input: unknown, context?: ToolExecutionContext) => Promise>; }; +export type ToolInput = + Definition extends ToolDefinition ? Input : never; + +export type ToolOutput = + Definition extends ToolDefinition ? Output : never; + /** * What the model asked the runtime to do. */ @@ -77,9 +93,9 @@ export type AssistantDiagnostics = { /** * What came back from executing a tool. */ -export type ToolResult = { +export type ToolResult = { ok: boolean; - output?: unknown; + output?: Output; error?: string; }; diff --git a/src/index.ts b/src/index.ts index 7920f53c..69cc8660 100644 --- a/src/index.ts +++ b/src/index.ts @@ -70,8 +70,11 @@ export type { RunResult, ModelRunFailureCode, ToolDefinition, + ToolInput, + ToolOutput, ToolConcurrencyMode, ToolExecutionContext, + ToolValidationSchema, ToolCall, ToolResult, TraceEvent,