From 953bc3047079b55b5500b88735e5e96414411a9e Mon Sep 17 00:00:00 2001 From: phfarath Date: Fri, 4 Sep 2026 12:22:12 -0300 Subject: [PATCH] Add Anthropic tool search bridge for deferred tool definitions Add RequestParams.tool_search policy (off/auto/always, default off) and bridge it to Anthropic's native tool search: regular tool definitions are marked defer_loading and a server-side BM25 tool_search tool is exposed so the model retrieves definitions on demand. Disabled on Vertex, suppressed for structured tool_use requests and sampling tool_choice pins. Addresses #766. --- .../_generated/request_params_reference.md | 1 + docs/docs/models/providers/anthropic.md | 17 ++ src/fast_agent/llm/fastagent_llm.py | 2 + .../llm/provider/anthropic/llm_anthropic.py | 51 ++++++ src/fast_agent/llm/request_params.py | 15 ++ .../provider/anthropic/test_tool_search.py | 145 ++++++++++++++++++ 6 files changed, 231 insertions(+) create mode 100644 tests/unit/fast_agent/llm/provider/anthropic/test_tool_search.py diff --git a/docs/docs/_generated/request_params_reference.md b/docs/docs/_generated/request_params_reference.md index 39fd9fe25..4e596a786 100644 --- a/docs/docs/_generated/request_params_reference.md +++ b/docs/docs/_generated/request_params_reference.md @@ -27,6 +27,7 @@ | `structured_schema` | `dict[str, Any] | None` | `None` | | | `structured_tool_policy` | `Literal['auto', 'always', 'defer', 'no_tools']` | `'auto'` | | | `sampling_tool_choice` | `Literal['auto', 'required', 'none'] | None` | `None` | | +| `tool_search` | `Literal['off', 'auto', 'always']` | `'off'` | | | `template_vars` | `dict[str, Any]` | `{}` | | | `mcp_metadata` | `dict[str, Any] | None` | `None` | | | `tool_execution_handler` | `Any | None` | `None` | | diff --git a/docs/docs/models/providers/anthropic.md b/docs/docs/models/providers/anthropic.md index 4c729d454..3861a5139 100644 --- a/docs/docs/models/providers/anthropic.md +++ b/docs/docs/models/providers/anthropic.md @@ -184,6 +184,23 @@ Version policy is model-aware: - Other supported Anthropic models use legacy versions (`web_search_20250305`, `web_fetch_20250910`). +**Tool search (deferred tool definitions):** + +With many MCP tools, sending every definition on each request costs tokens and +degrades selection accuracy. Anthropic models support deferring tool +definitions: regular tools are marked `defer_loading: true` and a server-side +BM25 `tool_search` tool is added, so the model retrieves relevant definitions +on demand. Tool search is controlled per agent/request with +`RequestParams(tool_search=...)`: + +- `off` (default): always send full tool definitions. +- `auto`: defer definitions when more than 16 tools are available. +- `always`: defer definitions on every request. + +Tool search is not available on `anthropic-vertex`, is suppressed for +structured `tool_use` requests, and is disabled when a sampling +`tool_choice` pins specific tools. + **Provider-managed remote MCP:** The direct `anthropic` provider supports provider-managed remote MCP servers diff --git a/src/fast_agent/llm/fastagent_llm.py b/src/fast_agent/llm/fastagent_llm.py index 67dece3e7..4169d2d7a 100644 --- a/src/fast_agent/llm/fastagent_llm.py +++ b/src/fast_agent/llm/fastagent_llm.py @@ -129,6 +129,7 @@ class FastAgentLLM(ContextDependent, FastAgentLLMProtocol, Generic[MessageParamT PARAM_STRUCTURED_SCHEMA = "structured_schema" PARAM_STRUCTURED_TOOL_POLICY = "structured_tool_policy" PARAM_SAMPLING_TOOL_CHOICE = "sampling_tool_choice" + PARAM_TOOL_SEARCH = "tool_search" PARAM_MCP_TOOLS = "tools" PARAM_MCP_TOOL_CHOICE = "tool_choice" @@ -144,6 +145,7 @@ class FastAgentLLM(ContextDependent, FastAgentLLMProtocol, Generic[MessageParamT PARAM_STRUCTURED_SCHEMA, PARAM_STRUCTURED_TOOL_POLICY, PARAM_SAMPLING_TOOL_CHOICE, + PARAM_TOOL_SEARCH, PARAM_MCP_TOOLS, PARAM_MCP_TOOL_CHOICE, } diff --git a/src/fast_agent/llm/provider/anthropic/llm_anthropic.py b/src/fast_agent/llm/provider/anthropic/llm_anthropic.py index 8d3c91428..a796c10dd 100644 --- a/src/fast_agent/llm/provider/anthropic/llm_anthropic.py +++ b/src/fast_agent/llm/provider/anthropic/llm_anthropic.py @@ -37,6 +37,7 @@ BetaThinkingBlock, BetaThinkingDelta, BetaToolParam, + BetaToolSearchToolBm25_20251119Param, BetaToolUseBlock, BetaToolUseBlockParam, ) @@ -142,6 +143,11 @@ FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14" MCP_CLIENT_BETA = "mcp-client-2025-11-20" +# Tool search defers tool definitions from the initial prompt; the model +# retrieves relevant definitions on demand via a server-side BM25 search tool. +ANTHROPIC_TOOL_SEARCH_TYPE = "tool_search_tool_bm25_20251119" +TOOL_SEARCH_AUTO_TOOL_THRESHOLD = 16 + # Stream capture mode - when enabled, saves all streaming chunks to files for debugging # Set FAST_AGENT_LLM_TRACE=1 (or any non-empty value) to enable STREAM_CAPTURE_ENABLED = bool(os.environ.get("FAST_AGENT_LLM_TRACE")) @@ -1114,6 +1120,40 @@ def _build_output_format( schema = _transform_anthropic_schema({"type": "object"}) return {"type": "json_schema", "schema": schema} + @staticmethod + def _defer_tool_definitions(tools: list[BetaToolParam]) -> list[BetaToolParam]: + """Mark tool definitions as deferred and expose the server-side search tool.""" + deferred_tools: list[BetaToolParam] = [{**tool, "defer_loading": True} for tool in tools] + deferred_tools.append( + cast( + "BetaToolParam", + BetaToolSearchToolBm25_20251119Param( + type=ANTHROPIC_TOOL_SEARCH_TYPE, + name="tool_search_tool_bm25", + ), + ) + ) + return deferred_tools + + def _resolve_tool_search_active( + self, + request_params: RequestParams | None, + tool_count: int, + ) -> bool: + params = request_params or self.default_request_params + policy = params.tool_search + if policy == "off": + return False + if not self.supports_direct_anthropic_beta("tool_search"): + self.logger.warning( + "Anthropic tool_search was requested but is unsupported on this " + "provider; sending full tool definitions instead." + ) + return False + if policy == "always": + return True + return tool_count > TOOL_SEARCH_AUTO_TOOL_THRESHOLD + async def _prepare_tools( self, model: str, @@ -1122,6 +1162,7 @@ async def _prepare_tools( tools: list[Tool] | None = None, structured_mode: StructuredOutputMode | None = None, auto_tool_use_fallback: bool = False, + tool_search_active: bool = False, ) -> list[BetaToolParam]: """Prepare tools based on whether we're in structured output mode.""" regular_tools = [ @@ -1159,6 +1200,8 @@ async def _prepare_tools( strict=True, ) ] + if tool_search_active and regular_tools: + regular_tools = self._defer_tool_definitions(regular_tools) if structured_model or structured_schema: return regular_tools return regular_tools @@ -2789,6 +2832,7 @@ async def _anthropic_request_tools( tools: list[Tool] | None, *, include_provider_tools: bool, + tool_search_active: bool = False, ) -> tuple[list[BetaToolParam], list[str], Any]: available_tools = await self._prepare_tools( model, @@ -2797,6 +2841,7 @@ async def _anthropic_request_tools( tools, structured_mode=structured.mode, auto_tool_use_fallback=structured.auto_tool_use_fallback, + tool_search_active=tool_search_active, ) if include_provider_tools: web_tools, web_tool_betas = self._prepare_web_tools(model) @@ -2871,12 +2916,18 @@ async def _anthropic_completion( structured_model, structured_schema, ) + # Sampling tool-choice pins specific tools; deferred definitions would + # conflict with a forced choice, so tool search is disabled there. + tool_search_active = request.params.sampling_tool_choice is None and ( + self._resolve_tool_search_active(request.params, len(tools or [])) + ) request_tools, web_tool_betas, provider_mcp_payload = await self._anthropic_request_tools( model, structured_model, structured, tools, include_provider_tools=request.params.sampling_tool_choice is None, + tool_search_active=tool_search_active, ) base_args, thinking_enabled = self._build_anthropic_base_args( diff --git a/src/fast_agent/llm/request_params.py b/src/fast_agent/llm/request_params.py index c10a30df4..feaeaeecb 100644 --- a/src/fast_agent/llm/request_params.py +++ b/src/fast_agent/llm/request_params.py @@ -19,6 +19,7 @@ ToolResultMode: TypeAlias = Literal["postprocess", "passthrough", "selectable"] StructuredToolPolicy: TypeAlias = Literal["auto", "always", "defer", "no_tools"] SamplingToolChoicePolicy: TypeAlias = Literal["auto", "required", "none"] +ToolSearchPolicy: TypeAlias = Literal["off", "auto", "always"] _RESPONSE_MODE_TOOL_RESULT_MODES: dict[ResponseMode, ToolResultMode | None] = { "inherit": None, "postprocess": "postprocess", @@ -130,6 +131,20 @@ class RequestParams(CreateMessageRequestParams): inputs only and must not be forwarded as provider arguments. """ + tool_search: ToolSearchPolicy = "off" + """ + Policy for deferring tool definitions when many tools are available. + + Providers with a native tool-search mechanism (Anthropic ``tool_search`` + with ``defer_loading``) mark regular tool definitions as deferred and + expose a search tool so the model retrieves definitions on demand. + + - ``off``: send full tool definitions on every request (default). + - ``auto``: defer definitions when the provider supports tool search and + the toolset is large enough to benefit from it. + - ``always``: defer definitions whenever the provider supports tool search. + """ + template_vars: dict[str, Any] = Field(default_factory=dict) """ Optional dictionary of template variables for dynamic templates. Currently only works for TensorZero inference backend diff --git a/tests/unit/fast_agent/llm/provider/anthropic/test_tool_search.py b/tests/unit/fast_agent/llm/provider/anthropic/test_tool_search.py new file mode 100644 index 000000000..5fcb57c7f --- /dev/null +++ b/tests/unit/fast_agent/llm/provider/anthropic/test_tool_search.py @@ -0,0 +1,145 @@ +"""Tests for Anthropic tool search (deferred tool definitions).""" + +import pytest +from mcp import Tool + +from fast_agent.config import AnthropicSettings, Settings +from fast_agent.context import Context +from fast_agent.llm.provider.anthropic.llm_anthropic import ( + ANTHROPIC_TOOL_SEARCH_TYPE, + TOOL_SEARCH_AUTO_TOOL_THRESHOLD, + AnthropicLLM, +) +from fast_agent.llm.provider.anthropic.llm_anthropic_vertex import AnthropicVertexLLM +from fast_agent.llm.request_params import RequestParams + + +def _make_llm() -> AnthropicLLM: + settings = Settings() + settings.anthropic = AnthropicSettings(api_key="test-key") + context = Context(config=settings) + return AnthropicLLM(context=context, model="claude-sonnet-4-6", name="test-agent") + + +def _make_vertex_llm() -> AnthropicVertexLLM: + settings = Settings() + settings.anthropic = AnthropicSettings(api_key="test-key") + context = Context(config=settings) + return AnthropicVertexLLM(context=context, model="claude-sonnet-4-6", name="test-agent") + + +def _tool(name: str) -> Tool: + return Tool( + name=name, + description=f"Test tool {name}.", + input_schema={"type": "object", "properties": {}}, + ) + + +@pytest.mark.asyncio +async def test_default_policy_sends_full_definitions() -> None: + llm = _make_llm() + tools = [_tool("alpha"), _tool("beta")] + + prepared = await llm._prepare_tools("claude-sonnet-4-6", tools=tools) + + assert len(prepared) == 2 + assert all("defer_loading" not in tool for tool in prepared) + assert [tool["name"] for tool in prepared] == ["alpha", "beta"] + + +@pytest.mark.asyncio +async def test_tool_search_defers_definitions_and_appends_search_tool() -> None: + llm = _make_llm() + tools = [_tool("alpha"), _tool("beta")] + + prepared = await llm._prepare_tools("claude-sonnet-4-6", tools=tools, tool_search_active=True) + + assert len(prepared) == 3 + assert all(tool.get("defer_loading") is True for tool in prepared[:2]) + assert prepared[0]["name"] == "alpha" + assert prepared[0]["description"] == "Test tool alpha." + assert prepared[0]["input_schema"] == {"type": "object", "properties": {}} + search_tool = prepared[-1] + assert search_tool["type"] == ANTHROPIC_TOOL_SEARCH_TYPE + assert search_tool["name"] == "tool_search_tool_bm25" + + +@pytest.mark.asyncio +async def test_structured_tool_use_suppresses_tool_search() -> None: + llm = _make_llm() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + } + + prepared = await llm._prepare_tools( + "claude-sonnet-4-6", + structured_schema=schema, + tools=[_tool("alpha")], + structured_mode="tool_use", + tool_search_active=True, + ) + + assert len(prepared) == 1 + assert prepared[0]["name"] == "return_structured_output" + assert "defer_loading" not in prepared[0] + + +@pytest.mark.asyncio +async def test_json_structured_mode_keeps_deferred_tools() -> None: + llm = _make_llm() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + } + + prepared = await llm._prepare_tools( + "claude-sonnet-4-6", + structured_schema=schema, + tools=[_tool("alpha")], + structured_mode="json", + tool_search_active=True, + ) + + assert len(prepared) == 2 + assert prepared[0]["name"] == "alpha" + assert prepared[0].get("defer_loading") is True + + +def test_resolve_policy_off_by_default() -> None: + llm = _make_llm() + + assert RequestParams().tool_search == "off" + assert llm._resolve_tool_search_active(None, 100) is False + + +def test_resolve_policy_always_active_at_any_tool_count() -> None: + llm = _make_llm() + + assert llm._resolve_tool_search_active(RequestParams(tool_search="always"), 1) is True + + +def test_resolve_policy_auto_uses_tool_threshold() -> None: + llm = _make_llm() + params = RequestParams(tool_search="auto") + tools = [_tool(f"tool_{i}") for i in range(TOOL_SEARCH_AUTO_TOOL_THRESHOLD)] + + assert llm._resolve_tool_search_active(params, len(tools) - 1) is False + assert llm._resolve_tool_search_active(params, len(tools)) is False + assert llm._resolve_tool_search_active(params, len(tools) + 1) is True + + +def test_resolve_policy_uses_default_request_params() -> None: + llm = _make_llm() + llm.default_request_params = RequestParams(tool_search="always") + + assert llm._resolve_tool_search_active(None, 1) is True + + +def test_vertex_does_not_support_tool_search() -> None: + llm = _make_vertex_llm() + + assert llm._resolve_tool_search_active(RequestParams(tool_search="always"), 50) is False